From 6308c59e8321a09a3bbe099a9685d515a1a659a2 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Mon, 6 Jul 2026 01:16:46 -0400 Subject: [PATCH 1/8] feat: backup command (vault + mysql) Signed-off-by: Deezzir --- sunbeam-python/sunbeam/commands/backup.py | 147 ++++++++ sunbeam-python/sunbeam/main.py | 3 + sunbeam-python/sunbeam/steps/backup.py | 404 ++++++++++++++++++++++ 3 files changed, 554 insertions(+) create mode 100644 sunbeam-python/sunbeam/commands/backup.py create mode 100644 sunbeam-python/sunbeam/steps/backup.py diff --git a/sunbeam-python/sunbeam/commands/backup.py b/sunbeam-python/sunbeam/commands/backup.py new file mode 100644 index 000000000..216fd0b7d --- /dev/null +++ b/sunbeam-python/sunbeam/commands/backup.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""``sunbeam backup`` command.""" + +import logging +import sys +from datetime import datetime, timezone + +import click +from rich.console import Console +from rich.table import Table + +from sunbeam.core.common import get_step_message, run_plan +from sunbeam.core.deployment import Deployment +from sunbeam.steps.backup import ( + BACKUP_COMPONENTS, + DEFAULT_BACKUP_TIMEOUT, + VAULT_PREREQUISITE_MSG, + BackupResult, + BackupTarget, + DiscoverBackupApplicationsStep, + ResolveBackupTargetsStep, + RunBackupsStep, + WriteBackupManifestStep, +) + +LOG = logging.getLogger(__name__) +console = Console() + +EXIT_SUCCESS = 0 +EXIT_PARTIAL = 1 +EXIT_FAILURE = 2 + + +def _print_summary(results: list[BackupResult]) -> None: + table = Table() + table.add_column("Application") + table.add_column("Unit") + table.add_column("Component") + table.add_column("Status") + table.add_column("Backup ID") + for result in results: + status = "[green]done[/green]" if result.success else "[red]failed[/red]" + table.add_row( + result.app, + result.unit, + result.component, + status, + result.backup_id or "-", + ) + console.print(table) + + +@click.command() +@click.option( + "--force", + is_flag=True, + default=False, + show_default=True, + help=( + "Back up applications whose cluster health cannot be verified, using the" + " leader unit. May capture stale data; use with caution." + ), +) +@click.option( + "--timeout", + default=DEFAULT_BACKUP_TIMEOUT, + show_default=True, + help="Time in seconds to wait for each backup to complete.", +) +@click.option( + "--no-prompt", is_flag=True, default=False, help="Do not prompt for confirmation." +) +@click.pass_context +def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> None: + """Create backups of stateful Sunbeam applications (MySQL and Vault).""" + if not no_prompt: + click.confirm(VAULT_PREREQUISITE_MSG, abort=True) + + deployment: Deployment = ctx.obj + jhelper = deployment.get_juju_helper() + model = deployment.openstack_machines_model + + console.print( + f"[bold]Backing up [{','.join(c.name for c in BACKUP_COMPONENTS)}]" + f" in model '{model}'...[/bold]" + ) + + discover_step = DiscoverBackupApplicationsStep(jhelper, model, BACKUP_COMPONENTS) + resolve_results = run_plan([discover_step], console) + discovered = get_step_message(resolve_results, DiscoverBackupApplicationsStep) + + if not any(discovered.values()): + console.print( + "No MySQL or Vault applications found. Nothing to back up. Check that" + " the applications are deployed in the model." + ) + sys.exit(EXIT_FAILURE) + + resolve_step = ResolveBackupTargetsStep( + jhelper, discovered, model=model, force=force + ) + targets_results = run_plan([resolve_step], console) + targets: list[BackupTarget] = get_step_message( + targets_results, ResolveBackupTargetsStep + ) + + if not targets: + console.print( + "Could not resolve a backup target for any application. Re-run with" + " --force to back up on leader units regardless of cluster health." + ) + sys.exit(EXIT_FAILURE) + + dispatched_at = datetime.now(timezone.utc).isoformat() + console.print(f"Dispatching backups at {dispatched_at}...") + run_step = RunBackupsStep( + jhelper, targets, force=force, timeout=timeout, model=model + ) + backup_results = run_plan([run_step], console) + results: list[BackupResult] = get_step_message(backup_results, RunBackupsStep) + + _print_summary(results) + + manifest_step = WriteBackupManifestStep(results, dispatched_at) + manifest_results = run_plan([manifest_step], console) + manifest_path = get_step_message(manifest_results, WriteBackupManifestStep) + if manifest_path: + console.print(f"Backup manifest written to: {manifest_path}") + + succeeded = sum(1 for r in results if r.success) + failed = len(results) - succeeded + console.print(f"Backup summary: {succeeded} succeeded, {failed} failed.") + + if failed == 0: + sys.exit(EXIT_SUCCESS) + + if any(r.component == "mysql" and not r.success for r in results): + console.print( + "[yellow]Warning:[/yellow] one or more MySQL backups failed. A partial" + " restore from this set may result in dangling OpenStack objects." + ) + + if succeeded == 0: + sys.exit(EXIT_FAILURE) + sys.exit(EXIT_PARTIAL) diff --git a/sunbeam-python/sunbeam/main.py b/sunbeam-python/sunbeam/main.py index 69efba9d3..e6509cde1 100644 --- a/sunbeam-python/sunbeam/main.py +++ b/sunbeam-python/sunbeam/main.py @@ -9,6 +9,7 @@ from snaphelpers import Snap from sunbeam import log +from sunbeam.commands import backup as backup_cmds from sunbeam.commands import configure as configure_cmds from sunbeam.commands import dashboard as dashboard_cmds from sunbeam.commands import generate_cloud_config as generate_cloud_config_cmds @@ -123,6 +124,8 @@ def main(): cli.add_command(launch_cmds.launch) cli.add_command(openrc_cmds.openrc) cli.add_command(dashboard_cmds.dashboard) + cli.add_command(backup_cmds.backup) + cli.add_command(restore_cmds.restore) # Add identity group cli.add_command(identity_group) diff --git a/sunbeam-python/sunbeam/steps/backup.py b/sunbeam-python/sunbeam/steps/backup.py new file mode 100644 index 000000000..197706d40 --- /dev/null +++ b/sunbeam-python/sunbeam/steps/backup.py @@ -0,0 +1,404 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Steps and component registry for ``sunbeam backup``. + +Backup is organized around a pluggable :class:`BackupComponent` registry so that +discovery, target resolution, concurrent dispatch, and manifest writing stay +component-agnostic. +""" + +import json +import logging +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path + +import yaml +from snaphelpers import Snap + +from sunbeam.core.common import SHARE_PATH, BaseStep, Result, ResultType, StepContext +from sunbeam.core.juju import ( + ActionFailedException, + ApplicationNotFoundException, + JujuException, + JujuHelper, + LeaderNotFoundException, + ModelNotFoundException, +) +from sunbeam.core.openstack import OPENSTACK_MODEL + +LOG = logging.getLogger(__name__) + +MYSQL_CHARM = "mysql-k8s" +VAULT_CHARM = "vault-k8s" +BACKUP_ACTION = "create-backup" +CLUSTER_STATUS_ACTION = "get-cluster-status" +DEFAULT_BACKUP_TIMEOUT = 300 +BACKUP_MANIFEST_DIR = SHARE_PATH / "backups" +VAULT_PREREQUISITE_MSG = ( + "Vault backup/restore requires the unseal keys and root token that were in effect " + "when the backup was created. Are you sure you want to continue?" +) + + +@dataclass +class BackupTarget: + """An application and the unit chosen to run its backup action against.""" + + app: str + unit: str + component: str + action: str + scale: int + is_replica: bool = False + + +@dataclass +class BackupResult: + """The outcome of attempting a backup for a single application.""" + + app: str + unit: str + component: str + success: bool + backup_id: str | None = None + error: str | None = None + + +@dataclass +class BackupComponent: + """Descriptor for a kind of stateful application that can be backed up.""" + + name: str + charm_names: list[str] + action: str + resolve_target: Callable[[JujuHelper, str, str, bool], BackupTarget | None] + backup_id_key: str = "backup-id" + supports_force: bool = False + + +def _secondary_unit_from_status(units: list[str], action_result: dict) -> str | None: + """Map a SECONDARY cluster member to a Juju unit name. + + The cluster status reports members by address/label rather than Juju unit + name, so match members flagged SECONDARY back to one of the application's + Juju units by ordinal. + """ + try: + status = json.loads(action_result.get("status", "{}")) + except (json.JSONDecodeError, TypeError): + return None + + topology = status.get("defaultreplicaset", {}).get("topology", {}) + secondary_labels = [ + label + for label, info in topology.items() + if isinstance(info, dict) and info.get("memberrole", "").upper() == "SECONDARY" + ] + if not secondary_labels: + return None + + for label in secondary_labels: + ordinal = label.split(".")[0].rsplit("-", 1)[-1] + for unit in units: + if unit.rsplit("/", 1)[-1] == ordinal: + return unit + return None + + +def _resolve_mysql_target( + jhelper: JujuHelper, app: str, model: str, force: bool +) -> BackupTarget | None: + """Resolve a MySQL backup target, preferring a secondary (replica) unit. + + Falls back to the leader when no secondary is available. When cluster status + cannot be read, the application is skipped unless ``force`` is set, in which + case the leader is used as a best-effort target. + """ + try: + leader = jhelper.get_leader_unit(app, model) + units = list(jhelper.get_application(app, model).units) + except (LeaderNotFoundException, ApplicationNotFoundException): + LOG.warning("Could not resolve %s, skipping", app) + return None + + scale = len(units) + try: + result = jhelper.run_action(leader, model, CLUSTER_STATUS_ACTION) + secondary = _secondary_unit_from_status(units, result) + if secondary is not None: + return BackupTarget( + app=app, + unit=secondary, + component="mysql", + action=BACKUP_ACTION, + scale=scale, + is_replica=True, + ) + except ActionFailedException as e: + if not force: + LOG.warning("Could not resolve backup target for %s, skipping: %s", app, e) + return None + LOG.warning( + "Could not resolve backup target for %s, using leader (--force): %s", + app, + e, + ) + + return BackupTarget( + app=app, + unit=leader, + component="mysql", + action=BACKUP_ACTION, + scale=scale, + is_replica=False, + ) + + +def _resolve_vault_target( + jhelper: JujuHelper, app: str, model: str, force: bool +) -> BackupTarget | None: + """Resolve the Vault backup target to the leader unit.""" + try: + leader = jhelper.get_leader_unit(app, model) + units = list(jhelper.get_application(app, model).units) + except (LeaderNotFoundException, ApplicationNotFoundException): + LOG.warning("Could not resolve %s, skipping", app) + return None + + return BackupTarget( + app=app, + unit=leader, + component="vault", + action=BACKUP_ACTION, + scale=len(units), + is_replica=False, + ) + + +BACKUP_COMPONENTS: list[BackupComponent] = [ + BackupComponent( + name="mysql", + charm_names=[MYSQL_CHARM], + action=BACKUP_ACTION, + resolve_target=_resolve_mysql_target, + supports_force=True, + ), + BackupComponent( + name="vault", + charm_names=[VAULT_CHARM], + action=BACKUP_ACTION, + resolve_target=_resolve_vault_target, + ), +] + + +def _component_for(name: str) -> BackupComponent | None: + return next((c for c in BACKUP_COMPONENTS if c.name == name), None) + + +class DiscoverBackupApplicationsStep(BaseStep): + """Discover applications of every registered backup component in the model.""" + + def __init__( + self, + jhelper: JujuHelper, + model: str = OPENSTACK_MODEL, + components: list[BackupComponent] = BACKUP_COMPONENTS, + ): + super().__init__( + "Discover backup applications", + "Discovering stateful applications to back up", + ) + self.jhelper = jhelper + self.components = components + self.model = model + + def run(self, context: StepContext) -> Result: + """Return a mapping of component name to discovered application names.""" + try: + status = self.jhelper.get_model_status(self.model) + except (ModelNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + discovered: dict[str, list[str]] = {c.name: [] for c in self.components} + for app_name, app_status in status.apps.items(): + charm_name = app_status.charm_name or "" + for component in self.components: + if charm_name in component.charm_names: + discovered[component.name].append(app_name) + + return Result(ResultType.COMPLETED, discovered) + + +class ResolveBackupTargetsStep(BaseStep): + """Resolve the unit to back up for every discovered application.""" + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + model: str = OPENSTACK_MODEL, + force: bool = False, + ): + super().__init__( + "Resolve backup targets", + "Resolving units for consistent backup targets", + ) + self.jhelper = jhelper + self.discovered = discovered + self.force = force + self.model = model + + def run(self, context: StepContext) -> Result: + """Return a flat list of resolved backup targets across components.""" + targets: list[BackupTarget] = [] + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + if component is None: + continue + for app in apps: + target = component.resolve_target( + self.jhelper, app, self.model, self.force + ) + if target is None: + self.update_status(context, f"skipped {app}") + continue + targets.append(target) + + return Result(ResultType.COMPLETED, targets) + + +def _run_single_backup( + jhelper: JujuHelper, + target: BackupTarget, + backup_id_key: str, + model: str, + timeout: int, + action_params: dict | None = None, +) -> BackupResult: + """Dispatch a single backup action and capture the result. + + Never raises: any failure is encoded in the returned :class:`BackupResult`. + """ + try: + result = jhelper.run_action( + target.unit, model, target.action, action_params, timeout=timeout + ) + return BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + success=True, + backup_id=result.get(backup_id_key), + ) + except Exception as e: + return BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + success=False, + error=str(e), + ) + + +class RunBackupsStep(BaseStep): + """Dispatch every target's backup action concurrently and collect results.""" + + def __init__( + self, + jhelper: JujuHelper, + targets: list[BackupTarget], + force: bool = False, + timeout: int = DEFAULT_BACKUP_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Run backups", "Dispatching backups across applications") + self.jhelper = jhelper + self.targets = targets + self.force = force + self.timeout = timeout + self.model = model + + def is_skip(self, context: StepContext) -> Result: + """Skip when there are no resolved targets to back up.""" + if not self.targets: + return Result(ResultType.SKIPPED, "No backup targets resolved") + return Result(ResultType.COMPLETED) + + def _action_params(self, component: BackupComponent | None) -> dict | None: + if self.force and component is not None and component.supports_force: + return {"force": True} + return None + + def run(self, context: StepContext) -> Result: + """Dispatch backups concurrently, returning a list of results.""" + components = { + target.component: _component_for(target.component) + for target in self.targets + } + results: list[BackupResult] = [] + with ThreadPoolExecutor(max_workers=len(self.targets)) as executor: + futures = [] + for target in self.targets: + component = components[target.component] + backup_id_key = component.backup_id_key if component else "backup-id" + futures.append( + executor.submit( + _run_single_backup, + self.jhelper, + target, + backup_id_key, + self.model, + self.timeout, + self._action_params(component), + ) + ) + for future in as_completed(futures): + results.append(future.result()) + + return Result(ResultType.COMPLETED, results) + + +class WriteBackupManifestStep(BaseStep): + """Write a timestamped manifest of the backup run to the snap share path.""" + + def __init__( + self, + results: list[BackupResult], + dispatched_at: str, + manifest_dir: Path | None = None, + ): + super().__init__("Write backup manifest", "Writing backup manifest") + self.results = results + self.dispatched_at = dispatched_at + self.manifest_dir = manifest_dir + + def run(self, context: StepContext) -> Result: + """Write the manifest and return its path.""" + if self.manifest_dir is not None: + directory = self.manifest_dir + else: + directory = Snap().paths.user_common / BACKUP_MANIFEST_DIR + directory.mkdir(parents=True, exist_ok=True) + + succeeded = sum(1 for r in self.results if r.success) + failed = len(self.results) - succeeded + manifest = { + "dispatched_at": self.dispatched_at, + "summary": {"succeeded": succeeded, "failed": failed}, + "results": [asdict(r) for r in self.results], + } + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + path = directory / f"backup-manifest-{timestamp}.yaml" + try: + with path.open("w") as manifest_file: + yaml.safe_dump(manifest, manifest_file, sort_keys=False) + except OSError as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED, str(path)) From 41cc8f5cf0f825e234e25d447b8ef97029345ca2 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Mon, 6 Jul 2026 01:17:50 -0400 Subject: [PATCH 2/8] tests(backup): add unit tests Signed-off-by: Deezzir --- .../tests/functional/local/test_backup.py | 26 ++ .../unit/sunbeam/commands/test_backup.py | 140 +++++++ .../tests/unit/sunbeam/steps/test_backup.py | 342 ++++++++++++++++++ 3 files changed, 508 insertions(+) create mode 100644 sunbeam-python/tests/functional/local/test_backup.py create mode 100644 sunbeam-python/tests/unit/sunbeam/commands/test_backup.py create mode 100644 sunbeam-python/tests/unit/sunbeam/steps/test_backup.py diff --git a/sunbeam-python/tests/functional/local/test_backup.py b/sunbeam-python/tests/functional/local/test_backup.py new file mode 100644 index 000000000..19f92a2fb --- /dev/null +++ b/sunbeam-python/tests/functional/local/test_backup.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Functional smoke test for ``sunbeam backup``. + +Skipped by default: requires a bootstrapped cloud with the ``openstack`` snap +installed and the object-storage backup prerequisite configured. Consistent with +the rest of the hardware/environment-gated functional suite. +""" + +import os + +import pytest + +from .utils import sunbeam_command + +pytestmark = pytest.mark.skipif( + not os.environ.get("SUNBEAM_FUNCTIONAL_BACKUP"), + reason="requires a bootstrapped cloud; set SUNBEAM_FUNCTIONAL_BACKUP to enable", +) + + +def test_backup_smoke(): + """``sunbeam backup --help`` is available and the command runs.""" + output = sunbeam_command("backup --help", capture_output=True) + assert "Create backups" in output diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py new file mode 100644 index 000000000..1af0154ca --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import json +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from sunbeam.commands.backup import backup + + +def _app_status(charm_name): + app = Mock() + app.charm_name = charm_name + app.units = {} + return app + + +def _model_status(apps): + status = Mock() + status.apps = apps + return status + + +@pytest.fixture +def deployment(): + return Mock() + + +@pytest.fixture +def jhelper(deployment): + jhelper = Mock() + deployment.get_juju_helper.return_value = jhelper + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + return jhelper + + +def _leader_only_cluster_status(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + return {"backup-id": f"backup-{unit.replace('/', '-')}"} + + +class TestBackupCommand: + def test_prompt_abort_stops_before_work(self, deployment, jhelper): + result = CliRunner().invoke(backup, obj=deployment, input="n\n") + + assert result.exit_code == 1, result.output + assert "Aborted" in result.output + jhelper.get_model_status.assert_not_called() + + def test_no_applications_exits_2(self, deployment, jhelper): + jhelper.get_model_status.return_value = _model_status({}) + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "Nothing to back up" in result.output + + def test_all_success_exits_0(self, deployment, jhelper): + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": _app_status("mysql-k8s"), + "vault": _app_status("vault-k8s"), + } + ) + jhelper.get_application.return_value = _app_status("mysql-k8s") + jhelper.run_action.side_effect = _leader_only_cluster_status + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "2 succeeded, 0 failed" in result.output + + def test_partial_failure_exits_1_with_warning(self, deployment, jhelper): + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": _app_status("mysql-k8s"), + "vault": _app_status("vault-k8s"), + } + ) + jhelper.get_application.return_value = _app_status("mysql-k8s") + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + if unit == "keystone-mysql/0": + raise Exception("backup failed") + return {"backup-id": "backup-vault-0"} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "1 succeeded, 1 failed" in result.output + assert "Warning" in result.output + + def test_all_failed_exits_2(self, deployment, jhelper): + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": _app_status("mysql-k8s")} + ) + jhelper.get_application.return_value = _app_status("mysql-k8s") + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + raise Exception("backup failed") + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "0 succeeded, 1 failed" in result.output diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py new file mode 100644 index 000000000..936700885 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py @@ -0,0 +1,342 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import json +from unittest.mock import Mock + +from sunbeam.core.common import ResultType +from sunbeam.core.juju import ( + ActionFailedException, + LeaderNotFoundException, + ModelNotFoundException, +) +from sunbeam.steps.backup import ( + BACKUP_COMPONENTS, + BackupComponent, + BackupResult, + BackupTarget, + DiscoverBackupApplicationsStep, + ResolveBackupTargetsStep, + RunBackupsStep, + WriteBackupManifestStep, + _resolve_mysql_target, + _resolve_vault_target, +) + + +def _app_status(charm_name, units=None): + app = Mock() + app.charm_name = charm_name + app.units = units or {} + return app + + +def _model_status(apps): + status = Mock() + status.apps = apps + return status + + +def _cluster_status(secondary_ordinal): + topology = { + "mysql-0.mysql-endpoints": {"memberrole": "PRIMARY"}, + f"mysql-{secondary_ordinal}.mysql-endpoints": {"memberrole": "SECONDARY"}, + } + return {"status": json.dumps({"defaultreplicaset": {"topology": topology}})} + + +class TestBackupResult: + def test_construction_with_error_does_not_raise(self): + result = BackupResult( + app="keystone-mysql", + unit="keystone-mysql/0", + component="mysql", + success=False, + error="boom", + ) + assert result.success is False + assert result.backup_id is None + assert result.error == "boom" + + +class TestRegistry: + def test_registry_contains_mysql_and_vault(self): + names = {c.name for c in BACKUP_COMPONENTS} + assert names == {"mysql", "vault"} + + +class TestResolveMySQLTarget: + def test_picks_secondary_unit(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.run_action.return_value = _cluster_status(1) + app = _app_status( + "mysql-k8s", units={"keystone-mysql/0": Mock(), "keystone-mysql/1": Mock()} + ) + jhelper.get_application.return_value = app + + target = _resolve_mysql_target( + jhelper, "keystone-mysql", "openstack", force=False + ) + + assert target is not None + assert target.unit == "keystone-mysql/1" + assert target.is_replica is True + assert target.scale == 2 + + def test_falls_back_to_leader_when_no_secondary(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "cinder-mysql/0" + jhelper.run_action.return_value = { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + jhelper.get_application.return_value = _app_status( + "mysql-k8s", units={"cinder-mysql/0": Mock()} + ) + + target = _resolve_mysql_target( + jhelper, "cinder-mysql", "openstack", force=False + ) + + assert target is not None + assert target.unit == "cinder-mysql/0" + assert target.is_replica is False + assert target.scale == 1 + + def test_skips_on_action_failure_without_force(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.get_application.return_value = _app_status( + "mysql-k8s", units={"keystone-mysql/0": Mock()} + ) + jhelper.run_action.side_effect = ActionFailedException("failed") + + target = _resolve_mysql_target( + jhelper, "keystone-mysql", "openstack", force=False + ) + + assert target is None + + def test_uses_leader_on_action_failure_with_force(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.get_application.return_value = _app_status( + "mysql-k8s", units={"keystone-mysql/0": Mock()} + ) + jhelper.run_action.side_effect = ActionFailedException("failed") + + target = _resolve_mysql_target( + jhelper, "keystone-mysql", "openstack", force=True + ) + + assert target is not None + assert target.unit == "keystone-mysql/0" + assert target.is_replica is False + assert target.scale == 1 + + def test_skips_when_no_leader(self): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = LeaderNotFoundException("no leader") + + target = _resolve_mysql_target( + jhelper, "keystone-mysql", "openstack", force=True + ) + + assert target is None + + +class TestResolveVaultTarget: + def test_resolves_leader(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "vault/0" + jhelper.get_application.return_value = _app_status( + "vault-k8s", units={"vault/0": Mock()} + ) + + target = _resolve_vault_target(jhelper, "vault", "openstack", force=False) + + assert target is not None + assert target.unit == "vault/0" + assert target.component == "vault" + assert target.is_replica is False + assert target.scale == 1 + + def test_skips_when_no_leader(self): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = LeaderNotFoundException("no leader") + + target = _resolve_vault_target(jhelper, "vault", "openstack", force=False) + + assert target is None + + +class TestDiscoverBackupApplicationsStep: + def test_discovers_by_charm_name(self, step_context): + jhelper = Mock() + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": _app_status("mysql-k8s"), + "nova-mysql": _app_status("mysql-k8s"), + "vault": _app_status("vault-k8s"), + "keystone": _app_status("keystone-k8s"), + } + ) + + result = DiscoverBackupApplicationsStep(jhelper).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert sorted(result.message["mysql"]) == ["keystone-mysql", "nova-mysql"] + assert result.message["vault"] == ["vault"] + + def test_fails_on_model_error(self, step_context): + jhelper = Mock() + jhelper.get_model_status.side_effect = ModelNotFoundException("missing") + + result = DiscoverBackupApplicationsStep(jhelper).run(step_context) + + assert result.result_type == ResultType.FAILED + + +class TestResolveBackupTargetsStep: + def test_flattens_targets_and_skips_unresolvable(self, step_context): + jhelper = Mock() + + def _leader(app, model): + if app == "broken-mysql": + raise LeaderNotFoundException("no leader") + return f"{app}/0" + + jhelper.get_leader_unit.side_effect = _leader + jhelper.run_action.return_value = _cluster_status(1) + jhelper.get_application.side_effect = lambda app, model: _app_status( + "mysql-k8s", units={f"{app}/0": Mock(), f"{app}/1": Mock()} + ) + + discovered = {"mysql": ["keystone-mysql", "broken-mysql"], "vault": ["vault"]} + result = ResolveBackupTargetsStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + apps = {t.app for t in result.message} + assert apps == {"keystone-mysql", "vault"} + + +class TestRunBackupsStep: + def test_skips_when_no_targets(self, step_context): + step = RunBackupsStep(Mock(), []) + assert step.is_skip(step_context).result_type == ResultType.SKIPPED + + def test_aggregates_mixed_results(self, step_context): + jhelper = Mock() + + def _run_action(unit, model, action, params=None, timeout=None): + if unit == "glance-mysql/1": + raise ActionFailedException("backup failed") + return {"backup-id": f"backup-{unit.replace('/', '-')}"} + + jhelper.run_action.side_effect = _run_action + targets = [ + BackupTarget( + "keystone-mysql", "keystone-mysql/1", "mysql", "create-backup", 2 + ), + BackupTarget("glance-mysql", "glance-mysql/1", "mysql", "create-backup", 2), + BackupTarget("vault", "vault/0", "vault", "create-backup", 1), + ] + + result = RunBackupsStep(jhelper, targets).run(step_context) + + assert result.result_type == ResultType.COMPLETED + by_app = {r.app: r for r in result.message} + assert by_app["keystone-mysql"].success is True + assert by_app["keystone-mysql"].backup_id == "backup-keystone-mysql-1" + assert by_app["glance-mysql"].success is False + assert by_app["glance-mysql"].error is not None + assert by_app["vault"].success is True + + def test_force_passes_force_param_only_to_mysql(self, step_context): + jhelper = Mock() + jhelper.run_action.return_value = {"backup-id": "id"} + targets = [ + BackupTarget( + "keystone-mysql", "keystone-mysql/1", "mysql", "create-backup", 2 + ), + BackupTarget("vault", "vault/0", "vault", "create-backup", 1), + ] + + RunBackupsStep(jhelper, targets, force=True).run(step_context) + + params_by_unit = { + call.args[0]: call.args[3] for call in jhelper.run_action.call_args_list + } + assert params_by_unit["keystone-mysql/1"] == {"force": True} + assert params_by_unit["vault/0"] is None + + +class TestWriteBackupManifestStep: + def test_writes_manifest(self, step_context, tmp_path): + results = [ + BackupResult("keystone-mysql", "keystone-mysql/1", "mysql", True, "id-1"), + BackupResult("glance-mysql", "glance-mysql/1", "mysql", False, None, "err"), + ] + step = WriteBackupManifestStep( + results, "2026-04-09T14:22:01+00:00", manifest_dir=tmp_path + ) + + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + written = list(tmp_path.glob("backup-manifest-*.yaml")) + assert len(written) == 1 + import yaml + + data = yaml.safe_load(written[0].read_text()) + assert data["summary"] == {"succeeded": 1, "failed": 1} + assert data["dispatched_at"] == "2026-04-09T14:22:01+00:00" + assert {r["app"] for r in data["results"]} == { + "keystone-mysql", + "glance-mysql", + } + + +class TestExtensibility: + """Adding a component is a registration change, not a code change (FR-021).""" + + def test_new_component_flows_through_generic_pipeline( + self, step_context, monkeypatch + ): + def _resolve_fake(jhelper, app, model, force): + return BackupTarget(app, f"{app}/0", "fake", "create-backup", 1) + + fake = BackupComponent( + name="fake", + charm_names=["fake-charm"], + action="create-backup", + resolve_target=_resolve_fake, + ) + components = BACKUP_COMPONENTS + [fake] + monkeypatch.setattr("sunbeam.steps.backup.BACKUP_COMPONENTS", components) + + jhelper = Mock() + jhelper.get_model_status.return_value = _model_status( + {"my-fake": _app_status("fake-charm")} + ) + jhelper.run_action.return_value = {"backup-id": "fake-backup"} + + discover = DiscoverBackupApplicationsStep(jhelper, components=components).run( + step_context + ) + assert discover.message["fake"] == ["my-fake"] + + resolve = ResolveBackupTargetsStep(jhelper, discover.message).run(step_context) + fake_targets = [t for t in resolve.message if t.component == "fake"] + assert len(fake_targets) == 1 + + run = RunBackupsStep(jhelper, fake_targets).run(step_context) + assert run.message[0].component == "fake" + assert run.message[0].success is True + assert run.message[0].backup_id == "fake-backup" From 361e9a61fd846f0c4f3a4c866a7efc033b7b2ea9 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Mon, 6 Jul 2026 01:18:10 -0400 Subject: [PATCH 3/8] feat: restore command (vault + mysql) Signed-off-by: Deezzir --- sunbeam-python/sunbeam/commands/restore.py | 143 +++++++++++++++++++++ sunbeam-python/sunbeam/main.py | 1 + sunbeam-python/sunbeam/steps/restore.py | 121 +++++++++++++++++ 3 files changed, 265 insertions(+) create mode 100644 sunbeam-python/sunbeam/commands/restore.py create mode 100644 sunbeam-python/sunbeam/steps/restore.py diff --git a/sunbeam-python/sunbeam/commands/restore.py b/sunbeam-python/sunbeam/commands/restore.py new file mode 100644 index 000000000..9e3e9f2ba --- /dev/null +++ b/sunbeam-python/sunbeam/commands/restore.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""``sunbeam restore`` command. + +This command lays out the in-place restore sequence. The steps that depend on the +control-plane pause/resume charm actions are guarded, so invoking restore today +stops before any destructive action and reports a clear failure. +""" + +import logging +from datetime import datetime + +import click +from rich.console import Console + +from sunbeam.core.common import BaseStep, get_step_message, run_plan +from sunbeam.core.deployment import Deployment +from sunbeam.steps.backup import ( + BACKUP_COMPONENTS, + VAULT_PREREQUISITE_MSG, + BackupTarget, + DiscoverBackupApplicationsStep, + ResolveBackupTargetsStep, +) +from sunbeam.steps.restore import ( + DEFAULT_RESTORE_TIMEOUT, + DEFAULT_SCALE_TIMEOUT, + PauseControlPlaneStep, + RestoreMySQLStep, + RestoreVaultStep, + ResumeControlPlaneStep, + ScaleMySQLStep, +) + +LOG = logging.getLogger(__name__) +console = Console() + +RESTORE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S" + + +def _validate_restore_to_time( + ctx: click.Context, param: click.Parameter, value: str | None +) -> str | None: + if value is None: + return None + try: + datetime.strptime(value, RESTORE_TIME_FORMAT) + except ValueError: + raise click.BadParameter( + f"expected format 'YYYY-MM-DD HH:MM:SS', got {value!r}" + ) + return value + + +@click.command() +@click.option( + "--restore-to-time", + default=None, + callback=_validate_restore_to_time, + help="Point-in-time to restore to, formatted 'YYYY-MM-DD HH:MM:SS'.", +) +@click.option( + "--force", + is_flag=True, + default=False, + show_default=True, + help="Proceed with restore despite cluster health concerns.", +) +@click.option( + "--timeout", + default=DEFAULT_RESTORE_TIMEOUT, + show_default=True, + help="Time in seconds to wait for restore operations to complete.", +) +@click.option( + "--no-prompt", is_flag=True, default=False, help="Do not prompt for confirmation." +) +@click.pass_context +def restore( + ctx: click.Context, + restore_to_time: str | None, + force: bool, + timeout: int, + no_prompt: bool, +) -> None: + """Restore stateful Sunbeam applications from a backup.""" + if not no_prompt: + click.confirm(VAULT_PREREQUISITE_MSG, abort=True) + + deployment: Deployment = ctx.obj + jhelper = deployment.get_juju_helper() + model = deployment.openstack_machines_model + + console.print( + f"[bold]Restoring [{','.join(c.name for c in BACKUP_COMPONENTS)}]" + f" in model '{model}' from backup...[/bold]" + ) + + discover_results = run_plan( + [DiscoverBackupApplicationsStep(jhelper, model, BACKUP_COMPONENTS)], console + ) + discovered = get_step_message(discover_results, DiscoverBackupApplicationsStep) + + resolve_results = run_plan( + [ResolveBackupTargetsStep(jhelper, discovered, model=model, force=force)], + console, + ) + targets: list[BackupTarget] = get_step_message( + resolve_results, ResolveBackupTargetsStep + ) + mysql_targets = [t for t in targets if t.component == "mysql"] + + # Restore plan + plan: list[BaseStep] = [PauseControlPlaneStep(jhelper, model=model)] + for target in mysql_targets: + plan.append( + ScaleMySQLStep( + jhelper, target.app, 1, timeout=DEFAULT_SCALE_TIMEOUT, model=model + ) + ) + plan.append( + RestoreMySQLStep( + jhelper, + target, + restore_to_time=restore_to_time, + timeout=timeout, + model=model, + ) + ) + plan.append( + ScaleMySQLStep( + jhelper, + target.app, + target.scale, + timeout=DEFAULT_SCALE_TIMEOUT, + model=model, + ) + ) + plan.append(RestoreVaultStep(jhelper, timeout=timeout, model=model)) + plan.append(ResumeControlPlaneStep(jhelper, model=model)) + + run_plan(plan, console) diff --git a/sunbeam-python/sunbeam/main.py b/sunbeam-python/sunbeam/main.py index e6509cde1..6517880a8 100644 --- a/sunbeam-python/sunbeam/main.py +++ b/sunbeam-python/sunbeam/main.py @@ -20,6 +20,7 @@ from sunbeam.commands import plans as plans_cmd from sunbeam.commands import prepare_node as prepare_node_cmds from sunbeam.commands import proxy as proxy_cmds +from sunbeam.commands import restore as restore_cmds from sunbeam.commands import sso as sso_cmd from sunbeam.commands import utils as utils_cmds from sunbeam.core import deployments as deployments_jobs diff --git a/sunbeam-python/sunbeam/steps/restore.py b/sunbeam-python/sunbeam/steps/restore.py new file mode 100644 index 000000000..67eed5110 --- /dev/null +++ b/sunbeam-python/sunbeam/steps/restore.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Steps for ``sunbeam restore``.""" + +import logging + +from sunbeam.core.common import BaseStep, Result, ResultType, StepContext +from sunbeam.core.juju import ( + JujuException, + JujuHelper, + JujuWaitException, +) +from sunbeam.core.openstack import OPENSTACK_MODEL +from sunbeam.steps.backup import BackupTarget + +LOG = logging.getLogger(__name__) + +DEFAULT_SCALE_TIMEOUT = 1800 +DEFAULT_RESTORE_TIMEOUT = 1800 + + +class PauseControlPlaneStep(BaseStep): + """Pause OpenStack control-plane API services before a MySQL restore.""" + + def __init__(self, jhelper: JujuHelper, model: str = OPENSTACK_MODEL): + super().__init__("Pause control-plane", "Pausing control-plane services") + self.jhelper = jhelper + self.model = model + + def run(self, context: StepContext) -> Result: + """Fail cleanly: pause is not yet supported by the charms.""" + return Result(ResultType.FAILED, "Unimplemented") + + +class ResumeControlPlaneStep(BaseStep): + """Resume OpenStack control-plane API services after a MySQL restore.""" + + def __init__(self, jhelper: JujuHelper, model: str = OPENSTACK_MODEL): + super().__init__("Resume control-plane", "Resuming control-plane services") + self.jhelper = jhelper + self.model = model + + def run(self, context: StepContext) -> Result: + """Fail cleanly: resume is not yet supported by the charms.""" + return Result(ResultType.FAILED, "Unimplemented") + + +class ScaleMySQLStep(BaseStep): + """Scale a MySQL application to a target number of units.""" + + def __init__( + self, + jhelper: JujuHelper, + application: str, + scale: int, + timeout: int = DEFAULT_SCALE_TIMEOUT, + model: str = OPENSTACK_MODEL, + ) -> None: + super().__init__("Scale MySQL", f"Scaling {application} to {scale} unit(s)") + self.jhelper = jhelper + self.application = application + self.scale = scale + self.timeout = timeout + self.model = model + + def run(self, context: StepContext) -> Result: + """Scale the application and wait for it to settle.""" + try: + self.jhelper.scale_application(self.model, self.application, self.scale) + self.jhelper.wait_until_active( + self.model, apps=[self.application], timeout=self.timeout + ) + except (JujuException, JujuWaitException, TimeoutError) as e: + return Result(ResultType.FAILED, str(e)) + return Result(ResultType.COMPLETED) + + +class RestoreMySQLStep(BaseStep): + """Restore a MySQL application from a backup.""" + + def __init__( + self, + jhelper: JujuHelper, + target: BackupTarget, + restore_to_time: str | None = None, + timeout: int = DEFAULT_RESTORE_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Restore MySQL", f"Restoring {target.app}") + self.jhelper = jhelper + self.target = target + self.restore_to_time = restore_to_time + self.model = model + self.timeout = timeout + + def run(self, context: StepContext) -> Result: + """Fail cleanly: restore is gated on control-plane pause support.""" + return Result(ResultType.FAILED, "Unimplemented") + + +class RestoreVaultStep(BaseStep): + """Restore a Vault application from a backup. + + Skeleton: surfaces the unseal-keys/root-token prerequisite and does not + perform the restore in this iteration. + """ + + def __init__( + self, + jhelper: JujuHelper, + timeout: int = DEFAULT_RESTORE_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Restore Vault", "Restoring Vault") + self.jhelper = jhelper + self.timeout = timeout + + def run(self, context: StepContext) -> Result: + """Fail cleanly, surfacing the Vault restore prerequisite.""" + return Result(ResultType.FAILED, "Unimplemented") From fc7864ffd3a7786f1ae75ee4e9e7335d9756a006 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Mon, 6 Jul 2026 01:18:33 -0400 Subject: [PATCH 4/8] tests(restore): add unit tests Signed-off-by: Deezzir --- .../unit/sunbeam/commands/test_restore.py | 73 +++++++++++++++++++ .../tests/unit/sunbeam/steps/test_restore.py | 57 +++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 sunbeam-python/tests/unit/sunbeam/commands/test_restore.py create mode 100644 sunbeam-python/tests/unit/sunbeam/steps/test_restore.py diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py new file mode 100644 index 000000000..186893c4b --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from sunbeam.commands.restore import restore + + +def _app_status(charm_name): + app = Mock() + app.charm_name = charm_name + app.units = {} + return app + + +def _model_status(apps): + status = Mock() + status.apps = apps + return status + + +@pytest.fixture +def deployment(): + return Mock() + + +@pytest.fixture +def jhelper(deployment): + jhelper = Mock() + deployment.get_juju_helper.return_value = jhelper + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": _app_status("mysql-k8s"), "vault": _app_status("vault-k8s")} + ) + jhelper.get_application.return_value = _app_status("mysql-k8s") + return jhelper + + +class TestRestoreCommand: + def test_prompt_abort_stops_before_work(self, deployment, jhelper): + result = CliRunner().invoke(restore, obj=deployment, input="n\n") + + assert result.exit_code == 1, result.output + assert "Aborted" in result.output + jhelper.get_model_status.assert_not_called() + jhelper.scale_application.assert_not_called() + + def test_stops_at_pause_guard_and_is_non_destructive(self, deployment, jhelper): + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "Unimplemented" in result.output + jhelper.scale_application.assert_not_called() + + def test_no_restore_or_resume_action_dispatched(self, deployment, jhelper): + CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + for call in jhelper.run_action.call_args_list: + action = call.args[2] if len(call.args) > 2 else "" + assert action not in ("restore-backup", "resume") + + def test_invalid_restore_to_time_fails_fast(self, deployment, jhelper): + result = CliRunner().invoke( + restore, ["--restore-to-time", "not-a-date"], obj=deployment + ) + + assert result.exit_code != 0 + assert "YYYY-MM-DD HH:MM:SS" in result.output + jhelper.get_model_status.assert_not_called() + jhelper.scale_application.assert_not_called() diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py new file mode 100644 index 000000000..34576b878 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import Mock + +from sunbeam.core.common import ResultType +from sunbeam.core.juju import JujuException +from sunbeam.steps.backup import BackupTarget +from sunbeam.steps.restore import ( + PauseControlPlaneStep, + RestoreMySQLStep, + RestoreVaultStep, + ResumeControlPlaneStep, + ScaleMySQLStep, +) + + +class TestGuardedSteps: + def test_pause_returns_failed(self, step_context): + result = PauseControlPlaneStep(Mock()).run(step_context) + assert result.result_type == ResultType.FAILED + assert "Unimplemented" in result.message + + def test_resume_returns_failed(self, step_context): + result = ResumeControlPlaneStep(Mock()).run(step_context) + assert result.result_type == ResultType.FAILED + + def test_restore_mysql_returns_failed_and_does_not_dispatch(self, step_context): + jhelper = Mock() + target = BackupTarget( + "keystone-mysql", "keystone-mysql/0", "mysql", "restore", 3 + ) + result = RestoreMySQLStep(jhelper, target).run(step_context) + assert result.result_type == ResultType.FAILED + jhelper.run_action.assert_not_called() + + def test_restore_vault_surfaces_prerequisite(self, step_context): + result = RestoreVaultStep(Mock()).run(step_context) + assert result.result_type == ResultType.FAILED + assert "Unimplemented" in result.message + + +class TestScaleMySQLStep: + def test_scales_and_waits(self, step_context): + jhelper = Mock() + result = ScaleMySQLStep(jhelper, "keystone-mysql", 1).run(step_context) + assert result.result_type == ResultType.COMPLETED + jhelper.scale_application.assert_called_once_with( + "openstack", "keystone-mysql", 1 + ) + jhelper.wait_until_active.assert_called_once() + + def test_returns_failed_on_juju_error(self, step_context): + jhelper = Mock() + jhelper.scale_application.side_effect = JujuException("boom") + result = ScaleMySQLStep(jhelper, "keystone-mysql", 1).run(step_context) + assert result.result_type == ResultType.FAILED From 8c4cbe51cebd34633390f19a55ce4ecd40c478b6 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Wed, 15 Jul 2026 00:25:54 -0400 Subject: [PATCH 5/8] refactor: single module for backup/restore Signed-off-by: Deezzir --- sunbeam-python/sunbeam/commands/backup.py | 147 --- .../sunbeam/commands/backup_restore.py | 548 +++++++++++ sunbeam-python/sunbeam/commands/restore.py | 143 --- sunbeam-python/sunbeam/core/juju.py | 18 + sunbeam-python/sunbeam/main.py | 8 +- sunbeam-python/sunbeam/steps/backup.py | 404 -------- .../sunbeam/steps/backup_restore.py | 931 ++++++++++++++++++ sunbeam-python/sunbeam/steps/restore.py | 121 --- .../unit/sunbeam/commands/test_backup.py | 140 ++- .../sunbeam/commands/test_list_backups.py | 164 +++ .../unit/sunbeam/commands/test_restore.py | 326 +++++- .../tests/unit/sunbeam/steps/test_backup.py | 200 +++- .../tests/unit/sunbeam/steps/test_restore.py | 103 +- 13 files changed, 2385 insertions(+), 868 deletions(-) delete mode 100644 sunbeam-python/sunbeam/commands/backup.py create mode 100644 sunbeam-python/sunbeam/commands/backup_restore.py delete mode 100644 sunbeam-python/sunbeam/commands/restore.py delete mode 100644 sunbeam-python/sunbeam/steps/backup.py create mode 100644 sunbeam-python/sunbeam/steps/backup_restore.py delete mode 100644 sunbeam-python/sunbeam/steps/restore.py create mode 100644 sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py diff --git a/sunbeam-python/sunbeam/commands/backup.py b/sunbeam-python/sunbeam/commands/backup.py deleted file mode 100644 index 216fd0b7d..000000000 --- a/sunbeam-python/sunbeam/commands/backup.py +++ /dev/null @@ -1,147 +0,0 @@ -# SPDX-FileCopyrightText: 2026 - Canonical Ltd -# SPDX-License-Identifier: Apache-2.0 - -"""``sunbeam backup`` command.""" - -import logging -import sys -from datetime import datetime, timezone - -import click -from rich.console import Console -from rich.table import Table - -from sunbeam.core.common import get_step_message, run_plan -from sunbeam.core.deployment import Deployment -from sunbeam.steps.backup import ( - BACKUP_COMPONENTS, - DEFAULT_BACKUP_TIMEOUT, - VAULT_PREREQUISITE_MSG, - BackupResult, - BackupTarget, - DiscoverBackupApplicationsStep, - ResolveBackupTargetsStep, - RunBackupsStep, - WriteBackupManifestStep, -) - -LOG = logging.getLogger(__name__) -console = Console() - -EXIT_SUCCESS = 0 -EXIT_PARTIAL = 1 -EXIT_FAILURE = 2 - - -def _print_summary(results: list[BackupResult]) -> None: - table = Table() - table.add_column("Application") - table.add_column("Unit") - table.add_column("Component") - table.add_column("Status") - table.add_column("Backup ID") - for result in results: - status = "[green]done[/green]" if result.success else "[red]failed[/red]" - table.add_row( - result.app, - result.unit, - result.component, - status, - result.backup_id or "-", - ) - console.print(table) - - -@click.command() -@click.option( - "--force", - is_flag=True, - default=False, - show_default=True, - help=( - "Back up applications whose cluster health cannot be verified, using the" - " leader unit. May capture stale data; use with caution." - ), -) -@click.option( - "--timeout", - default=DEFAULT_BACKUP_TIMEOUT, - show_default=True, - help="Time in seconds to wait for each backup to complete.", -) -@click.option( - "--no-prompt", is_flag=True, default=False, help="Do not prompt for confirmation." -) -@click.pass_context -def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> None: - """Create backups of stateful Sunbeam applications (MySQL and Vault).""" - if not no_prompt: - click.confirm(VAULT_PREREQUISITE_MSG, abort=True) - - deployment: Deployment = ctx.obj - jhelper = deployment.get_juju_helper() - model = deployment.openstack_machines_model - - console.print( - f"[bold]Backing up [{','.join(c.name for c in BACKUP_COMPONENTS)}]" - f" in model '{model}'...[/bold]" - ) - - discover_step = DiscoverBackupApplicationsStep(jhelper, model, BACKUP_COMPONENTS) - resolve_results = run_plan([discover_step], console) - discovered = get_step_message(resolve_results, DiscoverBackupApplicationsStep) - - if not any(discovered.values()): - console.print( - "No MySQL or Vault applications found. Nothing to back up. Check that" - " the applications are deployed in the model." - ) - sys.exit(EXIT_FAILURE) - - resolve_step = ResolveBackupTargetsStep( - jhelper, discovered, model=model, force=force - ) - targets_results = run_plan([resolve_step], console) - targets: list[BackupTarget] = get_step_message( - targets_results, ResolveBackupTargetsStep - ) - - if not targets: - console.print( - "Could not resolve a backup target for any application. Re-run with" - " --force to back up on leader units regardless of cluster health." - ) - sys.exit(EXIT_FAILURE) - - dispatched_at = datetime.now(timezone.utc).isoformat() - console.print(f"Dispatching backups at {dispatched_at}...") - run_step = RunBackupsStep( - jhelper, targets, force=force, timeout=timeout, model=model - ) - backup_results = run_plan([run_step], console) - results: list[BackupResult] = get_step_message(backup_results, RunBackupsStep) - - _print_summary(results) - - manifest_step = WriteBackupManifestStep(results, dispatched_at) - manifest_results = run_plan([manifest_step], console) - manifest_path = get_step_message(manifest_results, WriteBackupManifestStep) - if manifest_path: - console.print(f"Backup manifest written to: {manifest_path}") - - succeeded = sum(1 for r in results if r.success) - failed = len(results) - succeeded - console.print(f"Backup summary: {succeeded} succeeded, {failed} failed.") - - if failed == 0: - sys.exit(EXIT_SUCCESS) - - if any(r.component == "mysql" and not r.success for r in results): - console.print( - "[yellow]Warning:[/yellow] one or more MySQL backups failed. A partial" - " restore from this set may result in dangling OpenStack objects." - ) - - if succeeded == 0: - sys.exit(EXIT_FAILURE) - sys.exit(EXIT_PARTIAL) diff --git a/sunbeam-python/sunbeam/commands/backup_restore.py b/sunbeam-python/sunbeam/commands/backup_restore.py new file mode 100644 index 000000000..98728639a --- /dev/null +++ b/sunbeam-python/sunbeam/commands/backup_restore.py @@ -0,0 +1,548 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""``sunbeam backup``, ``sunbeam restore`` and ``sunbeam list-backups`` commands.""" + +import logging +import sys +from datetime import datetime, timezone +from typing import Tuple + +import click +from rich.console import Console +from rich.table import Table + +from sunbeam.core.common import BaseStep, get_step_message, run_plan +from sunbeam.core.deployment import Deployment +from sunbeam.steps.backup_restore import ( + BACKUP_COMPONENTS, + DEFAULT_BACKUP_TIMEOUT, + DEFAULT_RESTORE_TIMEOUT, + DEFAULT_SCALE_TIMEOUT, + MYSQL_S3_RELATION, + RESTORE_TIME_FORMAT, + VAULT_PREREQUISITE_MSG, + VAULT_S3_RELATION, + BackupInventory, + BackupResult, + BackupTarget, + CheckAppPauseResumeSupportStep, + CheckS3RelationsStep, + DiscoverBackupApplicationsStep, + ListBackupsStep, + PauseAppStep, + ResolveBackupTargetsStep, + RestoreMySQLStep, + RestoreVaultStep, + ResumeAppStep, + RunBackupsStep, + ScaleAppStep, + WriteBackupInventoryManifestStep, + WriteBackupManifestStep, +) + +LOG = logging.getLogger(__name__) +console = Console() + +EXIT_SUCCESS = 0 +EXIT_PARTIAL = 1 +EXIT_FAILURE = 2 + +COMPONENT_LABELS = {"mysql": "MySQL", "vault": "Vault"} + +S3_RELATIONS = ( + ("mysql", MYSQL_S3_RELATION), + ("vault", VAULT_S3_RELATION), +) + + +def _discover_applications( + console_: Console, jhelper, model: str, components +) -> dict[str, list[str]]: + """Discover applications for every registered backup component.""" + results = run_plan( + [DiscoverBackupApplicationsStep(jhelper, model, components)], console_ + ) + return get_step_message(results, DiscoverBackupApplicationsStep) + + +def _filter_s3_related_apps( + console_: Console, + jhelper, + discovered: dict[str, list[str]], + model: str, +) -> Tuple[dict[str, list[str]], bool]: + """Filter discovered apps to those related to the required S3 endpoints.""" + was_filtered = False + filtered: dict[str, list[str]] = {} + + for component_name, endpoint_name in S3_RELATIONS: + applications = discovered.get(component_name, []) + if not applications: + continue + + s3_check_results = run_plan( + [ + CheckS3RelationsStep( + jhelper, + applications, + endpoint_name=endpoint_name, + model=model, + ) + ], + console_, + ) + s3_relations = get_step_message(s3_check_results, CheckS3RelationsStep) + unrelated_apps = s3_relations["unrelated"] + filtered[component_name] = s3_relations["related"] + + if unrelated_apps: + was_filtered = True + label = COMPONENT_LABELS.get(component_name, component_name) + console_.print( + f"[yellow]Warning:[/yellow] the following {label} applications are" + f" not related via '{endpoint_name}' and will be skipped:" + f" {', '.join(unrelated_apps)}" + ) + + return filtered, was_filtered + + +def _resolve_targets( + console_: Console, + jhelper, + discovered: dict[str, list[str]], + model: str, + force: bool, +) -> list[BackupTarget]: + """Resolve the unit to act on for every discovered application.""" + results = run_plan( + [ResolveBackupTargetsStep(jhelper, discovered, model=model, force=force)], + console_, + ) + return get_step_message(results, ResolveBackupTargetsStep) + + +def _list_inventory( + console_: Console, + jhelper, + targets: list[BackupTarget], + model: str, + timeout: int, +) -> list[BackupInventory]: + """List available backups for the given targets.""" + results = run_plan( + [ListBackupsStep(jhelper, targets, timeout=timeout, model=model)], console_ + ) + return get_step_message(results, ListBackupsStep) + + +def _print_inventory(results: list[BackupInventory]) -> None: + table = Table() + table.add_column("Application") + table.add_column("Unit") + table.add_column("Component") + table.add_column("Backup IDs") + table.add_column("Status") + + for result in sorted(results, key=lambda inventory: inventory.app): + status = "[green]done[/green]" if result.success else "[red]failed[/red]" + backup_ids = "-" + if result.success and result.backup_ids: + backup_ids = "\n".join(result.backup_ids) + table.add_row(result.app, result.unit, result.component, backup_ids, status) + + console.print(table) + + +def _print_summary(results: list[BackupResult]) -> None: + table = Table() + table.add_column("Application") + table.add_column("Unit") + table.add_column("Component") + table.add_column("Status") + table.add_column("Backup ID") + for result in results: + status = "[green]done[/green]" if result.success else "[red]failed[/red]" + table.add_row( + result.app, + result.unit, + result.component, + status, + result.backup_id or "-", + ) + console.print(table) + + +def _filter_restore_targets( + targets: list[BackupTarget], + inventory: list[BackupInventory], +) -> list[BackupTarget]: + """Warn on missing inventory and keep only restorable targets.""" + inventory_by_app = {entry.app: entry for entry in inventory} + failed_inventory = sorted( + (entry for entry in inventory if not entry.success or not entry.backup_ids), + key=lambda entry: entry.app, + ) + for entry in failed_inventory: + if not entry.success: + details = f": {entry.error}" if entry.error else "" + console.print( + "[yellow]Warning:[/yellow] Failed to list backups for " + f"{entry.app}{details}" + ) + else: + console.print( + f"[yellow]Warning:[/yellow] No backups available for {entry.app}." + ) + + return [ + target + for target in targets + if (inventory_by_app.get(target.app) is not None) + and inventory_by_app[target.app].success + and inventory_by_app[target.app].backup_ids + ] + + +def _run_mysql_restore( + jhelper, + target: BackupTarget, + model: str, + restore_to_time: str | None, + timeout: int, +) -> None: + """Run the restore plan for a MySQL target.""" + app_plan = [ + PauseAppStep(jhelper, target.app, model=model), + ScaleAppStep( + jhelper, + target.app, + 1, + timeout=DEFAULT_SCALE_TIMEOUT, + model=model, + ), + RestoreMySQLStep( + jhelper, + target, + restore_to_time=restore_to_time, + timeout=timeout, + model=model, + ), + ScaleAppStep( + jhelper, + target.app, + target.scale, + timeout=DEFAULT_SCALE_TIMEOUT, + model=model, + ), + ResumeAppStep(jhelper, target.app, model=model), + ] + + try: + run_plan(app_plan, console) + except click.ClickException as e: + revert_plan = [ + ScaleAppStep( + jhelper, + target.app, + target.scale, + timeout=DEFAULT_SCALE_TIMEOUT, + model=model, + ), + ResumeAppStep(jhelper, target.app, model=model), + ] + try: + run_plan(revert_plan, console) + except click.ClickException: + pass + + raise click.ClickException( + f"{target.app} restore failed: {e.message or str(e)}. Attempted to revert." + ) + + +def _validate_restore_to_time( + ctx: click.Context, param: click.Parameter, value: str | None +) -> str | None: + if value is None: + return None + try: + datetime.strptime(value, RESTORE_TIME_FORMAT) + except ValueError: + raise click.BadParameter( + f"expected format 'YYYY-MM-DD HH:MM:SS', got {value!r}" + ) + return value + + +@click.command() +@click.option( + "--force", + is_flag=True, + default=False, + show_default=True, + help=( + "Back up applications whose cluster health cannot be verified, using the" + " leader unit. May capture stale data; use with caution." + ), +) +@click.option( + "--timeout", + default=DEFAULT_BACKUP_TIMEOUT, + show_default=True, + help="Time in seconds to wait for each backup to complete.", +) +@click.option( + "--no-prompt", is_flag=True, default=False, help="Do not prompt for confirmation." +) +@click.pass_context +def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> None: + """Create backups of stateful Sunbeam applications (MySQL and Vault).""" + if not no_prompt: + click.confirm(VAULT_PREREQUISITE_MSG, abort=True) + + deployment: Deployment = ctx.obj + jhelper = deployment.get_juju_helper() + model = deployment.openstack_machines_model + + console.print( + f"[bold]Backing up [{','.join(c.name for c in BACKUP_COMPONENTS)}]" + f" in model '{model}'...[/bold]" + ) + + discovered = _discover_applications(console, jhelper, model, BACKUP_COMPONENTS) + if not any(discovered.values()): + console.print("No applications found to back up. Exiting.") + sys.exit(EXIT_FAILURE) + + discovered, was_filtered = _filter_s3_related_apps( + console, jhelper, discovered, model + ) + if was_filtered and not no_prompt: + click.confirm( + "Continue and back up the remaining components?", + abort=True, + ) + + targets: list[BackupTarget] = _resolve_targets( + console, jhelper, discovered, model, force=force + ) + + if not targets: + console.print( + "Could not resolve a backup target for any application. Re-run with" + " --force to back up on leader units regardless of cluster health." + ) + sys.exit(EXIT_FAILURE) + + dispatched_at = datetime.now(timezone.utc).isoformat() + console.print(f"Dispatching backups at {dispatched_at}...") + run_step = RunBackupsStep( + jhelper, targets, force=force, timeout=timeout, model=model + ) + backup_results = run_plan([run_step], console) + results: list[BackupResult] = get_step_message(backup_results, RunBackupsStep) + + _print_summary(results) + + manifest_step = WriteBackupManifestStep(results, dispatched_at) + manifest_results = run_plan([manifest_step], console) + manifest_path = get_step_message(manifest_results, WriteBackupManifestStep) + if manifest_path: + console.print(f"Backup manifest written to: {manifest_path}") + + succeeded = sum(1 for r in results if r.success) + failed = len(results) - succeeded + console.print(f"Backup summary: {succeeded} succeeded, {failed} failed.") + + if failed == 0: + sys.exit(EXIT_SUCCESS) + + if any(r.component == "mysql" and not r.success for r in results): + console.print( + "[yellow]Warning:[/yellow] one or more MySQL backups failed. A partial" + " restore from this set may result in dangling OpenStack objects." + ) + + if succeeded == 0: + sys.exit(EXIT_FAILURE) + sys.exit(EXIT_PARTIAL) + + +@click.command("list-backups") +@click.option( + "--timeout", + default=DEFAULT_BACKUP_TIMEOUT, + show_default=True, + help="Time in seconds to wait for each list action to complete.", +) +@click.pass_context +def list_backups(ctx: click.Context, timeout: int) -> None: + """List backup IDs from stateful Sunbeam applications.""" + deployment: Deployment = ctx.obj + jhelper = deployment.get_juju_helper() + model = deployment.openstack_machines_model + + console.print( + f"[bold]Listing backups for [{','.join(c.name for c in BACKUP_COMPONENTS)}]" + f" in model '{model}'...[/bold]" + ) + + discovered = _discover_applications(console, jhelper, model, BACKUP_COMPONENTS) + discovered, _ = _filter_s3_related_apps(console, jhelper, discovered, model) + + if not any(discovered.values()): + console.print("No applications found to list backups from. Exiting.") + sys.exit(EXIT_FAILURE) + + targets: list[BackupTarget] = _resolve_targets( + console, jhelper, discovered, model, force=False + ) + + listed_at = datetime.now(timezone.utc).isoformat() + inventory: list[BackupInventory] = _list_inventory( + console, jhelper, targets, model, timeout + ) + + failed_inventory = sorted( + (entry for entry in inventory if not entry.success), + key=lambda entry: entry.app, + ) + for entry in failed_inventory: + details = f": {entry.error}" if entry.error else "" + console.print( + f"[yellow]Warning:[/yellow] Failed to list backups for {entry.app}{details}" + ) + + _print_inventory(inventory) + + manifest_step = WriteBackupInventoryManifestStep(inventory, listed_at) + manifest_results = run_plan([manifest_step], console) + manifest_path = get_step_message( + manifest_results, + WriteBackupInventoryManifestStep, + ) + if manifest_path: + console.print(f"Backup inventory manifest written to: {manifest_path}") + + if failed_inventory: + sys.exit(EXIT_FAILURE) + + sys.exit(EXIT_SUCCESS) + + +@click.command() +@click.option( + "--restore-to-time", + default=None, + callback=_validate_restore_to_time, + help="Point-in-time to restore to, formatted 'YYYY-MM-DD HH:MM:SS'.", +) +@click.option( + "--force", + is_flag=True, + default=False, + show_default=True, + help="Proceed with restore despite cluster health concerns.", +) +@click.option( + "--timeout", + default=DEFAULT_RESTORE_TIMEOUT, + show_default=True, + help="Time in seconds to wait for restore operations to complete.", +) +@click.option( + "--no-prompt", is_flag=True, default=False, help="Do not prompt for confirmation." +) +@click.pass_context +def restore( + ctx: click.Context, + restore_to_time: str | None, + force: bool, + timeout: int, + no_prompt: bool, +) -> None: + """Restore stateful Sunbeam applications from a backup.""" + deployment: Deployment = ctx.obj + jhelper = deployment.get_juju_helper() + model = deployment.openstack_machines_model + + console.print( + f"[bold]Restoring [{','.join(c.name for c in BACKUP_COMPONENTS)}]" + f" in model '{model}' from backup...[/bold]" + ) + + discovered = _discover_applications(console, jhelper, model, BACKUP_COMPONENTS) + if not any(discovered.values()): + console.print("No applications found to restore. Exiting.") + sys.exit(EXIT_FAILURE) + + discovered, was_filtered = _filter_s3_related_apps( + console, jhelper, discovered, model + ) + if was_filtered and not no_prompt: + click.confirm( + "Continue and restore the remaining components?", + abort=True, + ) + + targets: list[BackupTarget] = _resolve_targets( + console, jhelper, discovered, model, force=force + ) + if not targets: + console.print("No restore targets could be resolved. Exiting.") + sys.exit(EXIT_FAILURE) + + inventory: list[BackupInventory] = _list_inventory( + console, jhelper, targets, model, timeout + ) + targets = _filter_restore_targets(targets, inventory) + + if not targets: + console.print("No backups were found to restore from. Exiting.") + sys.exit(EXIT_FAILURE) + + if restore_to_time is not None and any(t.component == "vault" for t in targets): + console.print( + "[yellow]Warning:[/yellow] Vault does not support point-in-time restore;" + " the latest Vault backup will be used." + ) + + if any(t.component == "vault" for t in targets) and not no_prompt: + click.confirm(VAULT_PREREQUISITE_MSG, abort=True) + + mysql_targets = [t for t in targets if t.component == "mysql"] + vault_targets = [t for t in targets if t.component == "vault"] + + precheck_plan: list[BaseStep] = [] + apps_to_pause_resume = sorted({target.app for target in mysql_targets}) + if apps_to_pause_resume: + precheck_plan.append( + CheckAppPauseResumeSupportStep( + jhelper, + apps_to_pause_resume, + model=model, + ) + ) + + if precheck_plan: + run_plan(precheck_plan, console) + + for target in mysql_targets: + _run_mysql_restore( + jhelper, + target, + model, + restore_to_time, + timeout, + ) + + vault_plan: list[BaseStep] = [] + for target in vault_targets: + vault_plan.append( + RestoreVaultStep(jhelper, target, timeout=timeout, model=model) + ) + + if vault_targets: + run_plan(vault_plan, console) diff --git a/sunbeam-python/sunbeam/commands/restore.py b/sunbeam-python/sunbeam/commands/restore.py deleted file mode 100644 index 9e3e9f2ba..000000000 --- a/sunbeam-python/sunbeam/commands/restore.py +++ /dev/null @@ -1,143 +0,0 @@ -# SPDX-FileCopyrightText: 2026 - Canonical Ltd -# SPDX-License-Identifier: Apache-2.0 - -"""``sunbeam restore`` command. - -This command lays out the in-place restore sequence. The steps that depend on the -control-plane pause/resume charm actions are guarded, so invoking restore today -stops before any destructive action and reports a clear failure. -""" - -import logging -from datetime import datetime - -import click -from rich.console import Console - -from sunbeam.core.common import BaseStep, get_step_message, run_plan -from sunbeam.core.deployment import Deployment -from sunbeam.steps.backup import ( - BACKUP_COMPONENTS, - VAULT_PREREQUISITE_MSG, - BackupTarget, - DiscoverBackupApplicationsStep, - ResolveBackupTargetsStep, -) -from sunbeam.steps.restore import ( - DEFAULT_RESTORE_TIMEOUT, - DEFAULT_SCALE_TIMEOUT, - PauseControlPlaneStep, - RestoreMySQLStep, - RestoreVaultStep, - ResumeControlPlaneStep, - ScaleMySQLStep, -) - -LOG = logging.getLogger(__name__) -console = Console() - -RESTORE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S" - - -def _validate_restore_to_time( - ctx: click.Context, param: click.Parameter, value: str | None -) -> str | None: - if value is None: - return None - try: - datetime.strptime(value, RESTORE_TIME_FORMAT) - except ValueError: - raise click.BadParameter( - f"expected format 'YYYY-MM-DD HH:MM:SS', got {value!r}" - ) - return value - - -@click.command() -@click.option( - "--restore-to-time", - default=None, - callback=_validate_restore_to_time, - help="Point-in-time to restore to, formatted 'YYYY-MM-DD HH:MM:SS'.", -) -@click.option( - "--force", - is_flag=True, - default=False, - show_default=True, - help="Proceed with restore despite cluster health concerns.", -) -@click.option( - "--timeout", - default=DEFAULT_RESTORE_TIMEOUT, - show_default=True, - help="Time in seconds to wait for restore operations to complete.", -) -@click.option( - "--no-prompt", is_flag=True, default=False, help="Do not prompt for confirmation." -) -@click.pass_context -def restore( - ctx: click.Context, - restore_to_time: str | None, - force: bool, - timeout: int, - no_prompt: bool, -) -> None: - """Restore stateful Sunbeam applications from a backup.""" - if not no_prompt: - click.confirm(VAULT_PREREQUISITE_MSG, abort=True) - - deployment: Deployment = ctx.obj - jhelper = deployment.get_juju_helper() - model = deployment.openstack_machines_model - - console.print( - f"[bold]Restoring [{','.join(c.name for c in BACKUP_COMPONENTS)}]" - f" in model '{model}' from backup...[/bold]" - ) - - discover_results = run_plan( - [DiscoverBackupApplicationsStep(jhelper, model, BACKUP_COMPONENTS)], console - ) - discovered = get_step_message(discover_results, DiscoverBackupApplicationsStep) - - resolve_results = run_plan( - [ResolveBackupTargetsStep(jhelper, discovered, model=model, force=force)], - console, - ) - targets: list[BackupTarget] = get_step_message( - resolve_results, ResolveBackupTargetsStep - ) - mysql_targets = [t for t in targets if t.component == "mysql"] - - # Restore plan - plan: list[BaseStep] = [PauseControlPlaneStep(jhelper, model=model)] - for target in mysql_targets: - plan.append( - ScaleMySQLStep( - jhelper, target.app, 1, timeout=DEFAULT_SCALE_TIMEOUT, model=model - ) - ) - plan.append( - RestoreMySQLStep( - jhelper, - target, - restore_to_time=restore_to_time, - timeout=timeout, - model=model, - ) - ) - plan.append( - ScaleMySQLStep( - jhelper, - target.app, - target.scale, - timeout=DEFAULT_SCALE_TIMEOUT, - model=model, - ) - ) - plan.append(RestoreVaultStep(jhelper, timeout=timeout, model=model)) - plan.append(ResumeControlPlaneStep(jhelper, model=model)) - - run_plan(plan, console) diff --git a/sunbeam-python/sunbeam/core/juju.py b/sunbeam-python/sunbeam/core/juju.py index e1f095f8d..e875923af 100644 --- a/sunbeam-python/sunbeam/core/juju.py +++ b/sunbeam-python/sunbeam/core/juju.py @@ -857,6 +857,24 @@ def run_action( raise ActionFailedException(str(task)) return task.results + def get_application_actions(self, application: str, model: str) -> list[str]: + """Return action names available for an application in a model.""" + with self._model(model) as juju: + try: + response = self.cli( + "actions", + application, + include_controller=False, + juju=juju, + ) + except jubilant.CLIError as e: + raise JujuException(str(e)) from e + + actions = response.get(application) + if not isinstance(actions, dict): + return [] + return sorted(actions.keys()) + def add_secret(self, model: str, name: str, data: dict, info: str) -> str: """Add secret to the model. diff --git a/sunbeam-python/sunbeam/main.py b/sunbeam-python/sunbeam/main.py index 6517880a8..96fc35833 100644 --- a/sunbeam-python/sunbeam/main.py +++ b/sunbeam-python/sunbeam/main.py @@ -9,7 +9,7 @@ from snaphelpers import Snap from sunbeam import log -from sunbeam.commands import backup as backup_cmds +from sunbeam.commands import backup_restore as backup_restore_cmds from sunbeam.commands import configure as configure_cmds from sunbeam.commands import dashboard as dashboard_cmds from sunbeam.commands import generate_cloud_config as generate_cloud_config_cmds @@ -20,7 +20,6 @@ from sunbeam.commands import plans as plans_cmd from sunbeam.commands import prepare_node as prepare_node_cmds from sunbeam.commands import proxy as proxy_cmds -from sunbeam.commands import restore as restore_cmds from sunbeam.commands import sso as sso_cmd from sunbeam.commands import utils as utils_cmds from sunbeam.core import deployments as deployments_jobs @@ -125,8 +124,9 @@ def main(): cli.add_command(launch_cmds.launch) cli.add_command(openrc_cmds.openrc) cli.add_command(dashboard_cmds.dashboard) - cli.add_command(backup_cmds.backup) - cli.add_command(restore_cmds.restore) + cli.add_command(backup_restore_cmds.backup) + cli.add_command(backup_restore_cmds.list_backups) + cli.add_command(backup_restore_cmds.restore) # Add identity group cli.add_command(identity_group) diff --git a/sunbeam-python/sunbeam/steps/backup.py b/sunbeam-python/sunbeam/steps/backup.py deleted file mode 100644 index 197706d40..000000000 --- a/sunbeam-python/sunbeam/steps/backup.py +++ /dev/null @@ -1,404 +0,0 @@ -# SPDX-FileCopyrightText: 2026 - Canonical Ltd -# SPDX-License-Identifier: Apache-2.0 - -"""Steps and component registry for ``sunbeam backup``. - -Backup is organized around a pluggable :class:`BackupComponent` registry so that -discovery, target resolution, concurrent dispatch, and manifest writing stay -component-agnostic. -""" - -import json -import logging -from collections.abc import Callable -from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import asdict, dataclass -from datetime import datetime, timezone -from pathlib import Path - -import yaml -from snaphelpers import Snap - -from sunbeam.core.common import SHARE_PATH, BaseStep, Result, ResultType, StepContext -from sunbeam.core.juju import ( - ActionFailedException, - ApplicationNotFoundException, - JujuException, - JujuHelper, - LeaderNotFoundException, - ModelNotFoundException, -) -from sunbeam.core.openstack import OPENSTACK_MODEL - -LOG = logging.getLogger(__name__) - -MYSQL_CHARM = "mysql-k8s" -VAULT_CHARM = "vault-k8s" -BACKUP_ACTION = "create-backup" -CLUSTER_STATUS_ACTION = "get-cluster-status" -DEFAULT_BACKUP_TIMEOUT = 300 -BACKUP_MANIFEST_DIR = SHARE_PATH / "backups" -VAULT_PREREQUISITE_MSG = ( - "Vault backup/restore requires the unseal keys and root token that were in effect " - "when the backup was created. Are you sure you want to continue?" -) - - -@dataclass -class BackupTarget: - """An application and the unit chosen to run its backup action against.""" - - app: str - unit: str - component: str - action: str - scale: int - is_replica: bool = False - - -@dataclass -class BackupResult: - """The outcome of attempting a backup for a single application.""" - - app: str - unit: str - component: str - success: bool - backup_id: str | None = None - error: str | None = None - - -@dataclass -class BackupComponent: - """Descriptor for a kind of stateful application that can be backed up.""" - - name: str - charm_names: list[str] - action: str - resolve_target: Callable[[JujuHelper, str, str, bool], BackupTarget | None] - backup_id_key: str = "backup-id" - supports_force: bool = False - - -def _secondary_unit_from_status(units: list[str], action_result: dict) -> str | None: - """Map a SECONDARY cluster member to a Juju unit name. - - The cluster status reports members by address/label rather than Juju unit - name, so match members flagged SECONDARY back to one of the application's - Juju units by ordinal. - """ - try: - status = json.loads(action_result.get("status", "{}")) - except (json.JSONDecodeError, TypeError): - return None - - topology = status.get("defaultreplicaset", {}).get("topology", {}) - secondary_labels = [ - label - for label, info in topology.items() - if isinstance(info, dict) and info.get("memberrole", "").upper() == "SECONDARY" - ] - if not secondary_labels: - return None - - for label in secondary_labels: - ordinal = label.split(".")[0].rsplit("-", 1)[-1] - for unit in units: - if unit.rsplit("/", 1)[-1] == ordinal: - return unit - return None - - -def _resolve_mysql_target( - jhelper: JujuHelper, app: str, model: str, force: bool -) -> BackupTarget | None: - """Resolve a MySQL backup target, preferring a secondary (replica) unit. - - Falls back to the leader when no secondary is available. When cluster status - cannot be read, the application is skipped unless ``force`` is set, in which - case the leader is used as a best-effort target. - """ - try: - leader = jhelper.get_leader_unit(app, model) - units = list(jhelper.get_application(app, model).units) - except (LeaderNotFoundException, ApplicationNotFoundException): - LOG.warning("Could not resolve %s, skipping", app) - return None - - scale = len(units) - try: - result = jhelper.run_action(leader, model, CLUSTER_STATUS_ACTION) - secondary = _secondary_unit_from_status(units, result) - if secondary is not None: - return BackupTarget( - app=app, - unit=secondary, - component="mysql", - action=BACKUP_ACTION, - scale=scale, - is_replica=True, - ) - except ActionFailedException as e: - if not force: - LOG.warning("Could not resolve backup target for %s, skipping: %s", app, e) - return None - LOG.warning( - "Could not resolve backup target for %s, using leader (--force): %s", - app, - e, - ) - - return BackupTarget( - app=app, - unit=leader, - component="mysql", - action=BACKUP_ACTION, - scale=scale, - is_replica=False, - ) - - -def _resolve_vault_target( - jhelper: JujuHelper, app: str, model: str, force: bool -) -> BackupTarget | None: - """Resolve the Vault backup target to the leader unit.""" - try: - leader = jhelper.get_leader_unit(app, model) - units = list(jhelper.get_application(app, model).units) - except (LeaderNotFoundException, ApplicationNotFoundException): - LOG.warning("Could not resolve %s, skipping", app) - return None - - return BackupTarget( - app=app, - unit=leader, - component="vault", - action=BACKUP_ACTION, - scale=len(units), - is_replica=False, - ) - - -BACKUP_COMPONENTS: list[BackupComponent] = [ - BackupComponent( - name="mysql", - charm_names=[MYSQL_CHARM], - action=BACKUP_ACTION, - resolve_target=_resolve_mysql_target, - supports_force=True, - ), - BackupComponent( - name="vault", - charm_names=[VAULT_CHARM], - action=BACKUP_ACTION, - resolve_target=_resolve_vault_target, - ), -] - - -def _component_for(name: str) -> BackupComponent | None: - return next((c for c in BACKUP_COMPONENTS if c.name == name), None) - - -class DiscoverBackupApplicationsStep(BaseStep): - """Discover applications of every registered backup component in the model.""" - - def __init__( - self, - jhelper: JujuHelper, - model: str = OPENSTACK_MODEL, - components: list[BackupComponent] = BACKUP_COMPONENTS, - ): - super().__init__( - "Discover backup applications", - "Discovering stateful applications to back up", - ) - self.jhelper = jhelper - self.components = components - self.model = model - - def run(self, context: StepContext) -> Result: - """Return a mapping of component name to discovered application names.""" - try: - status = self.jhelper.get_model_status(self.model) - except (ModelNotFoundException, JujuException) as e: - return Result(ResultType.FAILED, str(e)) - - discovered: dict[str, list[str]] = {c.name: [] for c in self.components} - for app_name, app_status in status.apps.items(): - charm_name = app_status.charm_name or "" - for component in self.components: - if charm_name in component.charm_names: - discovered[component.name].append(app_name) - - return Result(ResultType.COMPLETED, discovered) - - -class ResolveBackupTargetsStep(BaseStep): - """Resolve the unit to back up for every discovered application.""" - - def __init__( - self, - jhelper: JujuHelper, - discovered: dict[str, list[str]], - model: str = OPENSTACK_MODEL, - force: bool = False, - ): - super().__init__( - "Resolve backup targets", - "Resolving units for consistent backup targets", - ) - self.jhelper = jhelper - self.discovered = discovered - self.force = force - self.model = model - - def run(self, context: StepContext) -> Result: - """Return a flat list of resolved backup targets across components.""" - targets: list[BackupTarget] = [] - for component_name, apps in self.discovered.items(): - component = _component_for(component_name) - if component is None: - continue - for app in apps: - target = component.resolve_target( - self.jhelper, app, self.model, self.force - ) - if target is None: - self.update_status(context, f"skipped {app}") - continue - targets.append(target) - - return Result(ResultType.COMPLETED, targets) - - -def _run_single_backup( - jhelper: JujuHelper, - target: BackupTarget, - backup_id_key: str, - model: str, - timeout: int, - action_params: dict | None = None, -) -> BackupResult: - """Dispatch a single backup action and capture the result. - - Never raises: any failure is encoded in the returned :class:`BackupResult`. - """ - try: - result = jhelper.run_action( - target.unit, model, target.action, action_params, timeout=timeout - ) - return BackupResult( - app=target.app, - unit=target.unit, - component=target.component, - success=True, - backup_id=result.get(backup_id_key), - ) - except Exception as e: - return BackupResult( - app=target.app, - unit=target.unit, - component=target.component, - success=False, - error=str(e), - ) - - -class RunBackupsStep(BaseStep): - """Dispatch every target's backup action concurrently and collect results.""" - - def __init__( - self, - jhelper: JujuHelper, - targets: list[BackupTarget], - force: bool = False, - timeout: int = DEFAULT_BACKUP_TIMEOUT, - model: str = OPENSTACK_MODEL, - ): - super().__init__("Run backups", "Dispatching backups across applications") - self.jhelper = jhelper - self.targets = targets - self.force = force - self.timeout = timeout - self.model = model - - def is_skip(self, context: StepContext) -> Result: - """Skip when there are no resolved targets to back up.""" - if not self.targets: - return Result(ResultType.SKIPPED, "No backup targets resolved") - return Result(ResultType.COMPLETED) - - def _action_params(self, component: BackupComponent | None) -> dict | None: - if self.force and component is not None and component.supports_force: - return {"force": True} - return None - - def run(self, context: StepContext) -> Result: - """Dispatch backups concurrently, returning a list of results.""" - components = { - target.component: _component_for(target.component) - for target in self.targets - } - results: list[BackupResult] = [] - with ThreadPoolExecutor(max_workers=len(self.targets)) as executor: - futures = [] - for target in self.targets: - component = components[target.component] - backup_id_key = component.backup_id_key if component else "backup-id" - futures.append( - executor.submit( - _run_single_backup, - self.jhelper, - target, - backup_id_key, - self.model, - self.timeout, - self._action_params(component), - ) - ) - for future in as_completed(futures): - results.append(future.result()) - - return Result(ResultType.COMPLETED, results) - - -class WriteBackupManifestStep(BaseStep): - """Write a timestamped manifest of the backup run to the snap share path.""" - - def __init__( - self, - results: list[BackupResult], - dispatched_at: str, - manifest_dir: Path | None = None, - ): - super().__init__("Write backup manifest", "Writing backup manifest") - self.results = results - self.dispatched_at = dispatched_at - self.manifest_dir = manifest_dir - - def run(self, context: StepContext) -> Result: - """Write the manifest and return its path.""" - if self.manifest_dir is not None: - directory = self.manifest_dir - else: - directory = Snap().paths.user_common / BACKUP_MANIFEST_DIR - directory.mkdir(parents=True, exist_ok=True) - - succeeded = sum(1 for r in self.results if r.success) - failed = len(self.results) - succeeded - manifest = { - "dispatched_at": self.dispatched_at, - "summary": {"succeeded": succeeded, "failed": failed}, - "results": [asdict(r) for r in self.results], - } - - timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") - path = directory / f"backup-manifest-{timestamp}.yaml" - try: - with path.open("w") as manifest_file: - yaml.safe_dump(manifest, manifest_file, sort_keys=False) - except OSError as e: - return Result(ResultType.FAILED, str(e)) - - return Result(ResultType.COMPLETED, str(path)) diff --git a/sunbeam-python/sunbeam/steps/backup_restore.py b/sunbeam-python/sunbeam/steps/backup_restore.py new file mode 100644 index 000000000..d6c9dc8e4 --- /dev/null +++ b/sunbeam-python/sunbeam/steps/backup_restore.py @@ -0,0 +1,931 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Steps and component registry for ``sunbeam backup/restore``.""" + +import json +import logging +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path + +import yaml +from snaphelpers import Snap + +from sunbeam.core.common import SHARE_PATH, BaseStep, Result, ResultType, StepContext +from sunbeam.core.juju import ( + ActionFailedException, + ApplicationNotFoundException, + JujuException, + JujuHelper, + JujuWaitException, + LeaderNotFoundException, + ModelNotFoundException, +) +from sunbeam.core.openstack import OPENSTACK_MODEL + +LOG = logging.getLogger(__name__) + +MYSQL_CHARM = "mysql-k8s" +VAULT_CHARM = "vault-k8s" +BACKUP_ACTION = "create-backup" +LIST_BACKUPS_ACTION = "list-backups" +CLUSTER_STATUS_ACTION = "get-cluster-status" +MYSQL_S3_RELATION = "s3-parameters" +VAULT_S3_RELATION = "s3-parameters" +S3_INTERFACE = "s3" +DEFAULT_BACKUP_TIMEOUT = 300 +BACKUP_MANIFEST_DIR = SHARE_PATH / "backups" +VAULT_PREREQUISITE_MSG = ( + "Vault backup/restore requires the unseal keys and root token that were in effect " + "when the backup was created. Are you sure you want to continue?" +) +DEFAULT_SCALE_TIMEOUT = 1800 +DEFAULT_RESTORE_TIMEOUT = 1800 +MYSQL_RESTORE_ACTION = "restore" +VAULT_RESTORE_ACTION = "restore-backup" +PAUSE_ACTION = "pause" +RESUME_ACTION = "resume" +RESTORE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S" + + +@dataclass +class BackupTarget: + """An application and the unit chosen to run its backup action against.""" + + app: str + unit: str + component: str + action: str + scale: int + is_replica: bool = False + + +@dataclass +class BackupResult: + """The outcome of attempting a backup for a single application.""" + + app: str + unit: str + component: str + success: bool + backup_id: str | None = None + error: str | None = None + + +@dataclass +class BackupInventory: + """The result of listing backup IDs for a single application target.""" + + app: str + unit: str + component: str + success: bool + backup_ids: list[str] | None = None + error: str | None = None + + +@dataclass +class BackupComponent: + """Descriptor for a kind of stateful application that can be backed up.""" + + name: str + charm_names: list[str] + action: str + resolve_target: Callable[[JujuHelper, str, str, bool], BackupTarget | None] + list_action: str = LIST_BACKUPS_ACTION + parse_backup_ids: Callable[[dict], list[str]] | None = None + backup_id_key: str = "backup-id" + supports_force: bool = False + + +def _component_for(name: str) -> BackupComponent | None: + return next((c for c in BACKUP_COMPONENTS if c.name == name), None) + + +def _is_app_active(app_status: object) -> bool: + """Return whether application workload status is active.""" + status = getattr(app_status, "app_status", None) + current = getattr(status, "current", None) + if not isinstance(current, str): + return True + return current == "active" + + +def _app_status_value(app_status: object) -> str: + """Return application workload status string for display/logging.""" + status = getattr(app_status, "app_status", None) + current = getattr(status, "current", None) + if isinstance(current, str): + return current + return "unknown" + + +def _parse_mysql_backup_ids(action_result: dict) -> list[str]: + """Parse MySQL list-backups output table and return finished backup IDs.""" + backups_text = action_result.get("backups") + if backups_text is None: + backups_text = (action_result.get("results") or {}).get("backups") + if not isinstance(backups_text, str): + return [] + + backup_ids: list[str] = [] + for raw_line in backups_text.splitlines(): + line = raw_line.strip() + if not line or line.startswith("backup-id") or set(line) == {"-"}: + continue + columns = [column.strip() for column in line.split("|")] + if len(columns) < 3: + continue + if columns[2].lower() != "finished": + continue + backup_ids.append(columns[0]) + return backup_ids + + +def _parse_vault_backup_ids(action_result: dict) -> list[str]: + """Parse Vault list-backups output and return backup IDs.""" + raw_backup_ids = action_result.get("backup-ids") + if raw_backup_ids is None: + raw_backup_ids = (action_result.get("results") or {}).get("backup-ids") + + if isinstance(raw_backup_ids, list): + return [str(backup_id) for backup_id in raw_backup_ids] + if not isinstance(raw_backup_ids, str): + return [] + + try: + parsed = json.loads(raw_backup_ids) + except json.JSONDecodeError: + return [] + if not isinstance(parsed, list): + return [] + return [str(backup_id) for backup_id in parsed] + + +def _secondary_unit_from_status(units: list[str], action_result: dict) -> str | None: + """Map a SECONDARY cluster member to a Juju unit name. + + The cluster status reports members by address/label rather than Juju unit + name, so match members flagged SECONDARY back to one of the application's + Juju units by ordinal. + """ + try: + status = json.loads(action_result.get("status", "{}")) + except (json.JSONDecodeError, TypeError): + return None + + topology = status.get("defaultreplicaset", {}).get("topology", {}) + secondary_labels = [ + label + for label, info in topology.items() + if isinstance(info, dict) and info.get("memberrole", "").upper() == "SECONDARY" + ] + if not secondary_labels: + return None + + for label in secondary_labels: + ordinal = label.split(".")[0].rsplit("-", 1)[-1] + for unit in units: + if unit.rsplit("/", 1)[-1] == ordinal: + return unit + return None + + +def _latest_backup_id(backup_ids: list[str]) -> str | None: + """Return the lexicographically latest backup ID, if present.""" + if not backup_ids: + return None + return sorted(backup_ids)[-1] + + +def _resolve_mysql_target( + jhelper: JujuHelper, app: str, model: str, force: bool +) -> BackupTarget | None: + """Resolve a MySQL backup target, preferring a secondary (replica) unit. + + Falls back to the leader when no secondary is available. When cluster status + cannot be read, the application is skipped unless ``force`` is set, in which + case the leader is used as a best-effort target. + """ + try: + leader = jhelper.get_leader_unit(app, model) + units = list(jhelper.get_application(app, model).units) + except (LeaderNotFoundException, ApplicationNotFoundException): + LOG.warning("Could not resolve %s, skipping", app) + return None + + scale = len(units) + try: + result = jhelper.run_action(leader, model, CLUSTER_STATUS_ACTION) + secondary = _secondary_unit_from_status(units, result) + if secondary is not None: + return BackupTarget( + app=app, + unit=secondary, + component="mysql", + action=BACKUP_ACTION, + scale=scale, + is_replica=True, + ) + except ActionFailedException as e: + if not force: + LOG.warning("Could not resolve backup target for %s, skipping: %s", app, e) + return None + LOG.warning( + "Could not resolve backup target for %s, using leader (--force): %s", + app, + e, + ) + + return BackupTarget( + app=app, + unit=leader, + component="mysql", + action=BACKUP_ACTION, + scale=scale, + is_replica=False, + ) + + +def _resolve_vault_target( + jhelper: JujuHelper, app: str, model: str, force: bool +) -> BackupTarget | None: + """Resolve the Vault backup target to the leader unit.""" + try: + leader = jhelper.get_leader_unit(app, model) + units = list(jhelper.get_application(app, model).units) + except (LeaderNotFoundException, ApplicationNotFoundException): + LOG.warning("Could not resolve %s, skipping", app) + return None + + return BackupTarget( + app=app, + unit=leader, + component="vault", + action=BACKUP_ACTION, + scale=len(units), + is_replica=False, + ) + + +def _run_single_backup( + jhelper: JujuHelper, + target: BackupTarget, + backup_id_key: str, + model: str, + timeout: int, + action_params: dict | None = None, +) -> BackupResult: + """Dispatch a single backup action and capture the result. + + Any failure is encoded in the returned :class:`BackupResult`. + """ + try: + result = jhelper.run_action( + target.unit, model, target.action, action_params, timeout=timeout + ) + return BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + success=True, + backup_id=result.get(backup_id_key), + ) + except Exception as e: + return BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + success=False, + error=str(e), + ) + + +def _run_single_list_backups( + jhelper: JujuHelper, + target: BackupTarget, + action_name: str, + parse_backup_ids: Callable[[dict], list[str]] | None, + model: str, + timeout: int, +) -> BackupInventory: + """Dispatch a single list-backups action and parse backup IDs.""" + try: + result = jhelper.run_action(target.unit, model, action_name, timeout=timeout) + backup_ids = parse_backup_ids(result) if parse_backup_ids is not None else [] + return BackupInventory( + app=target.app, + unit=target.unit, + component=target.component, + success=True, + backup_ids=backup_ids, + ) + except Exception as e: + return BackupInventory( + app=target.app, + unit=target.unit, + component=target.component, + success=False, + error=str(e), + ) + + +BACKUP_COMPONENTS: list[BackupComponent] = [ + BackupComponent( + name="mysql", + charm_names=[MYSQL_CHARM], + action=BACKUP_ACTION, + resolve_target=_resolve_mysql_target, + parse_backup_ids=_parse_mysql_backup_ids, + supports_force=True, + ), + BackupComponent( + name="vault", + charm_names=[VAULT_CHARM], + action=BACKUP_ACTION, + resolve_target=_resolve_vault_target, + parse_backup_ids=_parse_vault_backup_ids, + ), +] + + +class DiscoverBackupApplicationsStep(BaseStep): + """Discover applications of every registered backup component in the model.""" + + def __init__( + self, + jhelper: JujuHelper, + model: str = OPENSTACK_MODEL, + components: list[BackupComponent] = BACKUP_COMPONENTS, + ): + super().__init__( + "Discover backup applications", + "Discovering stateful applications to back up", + ) + self.jhelper = jhelper + self.components = components + self.model = model + + def run(self, context: StepContext) -> Result: + """Return a mapping of component name to discovered application names.""" + try: + status = self.jhelper.get_model_status(self.model) + except (ModelNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + discovered: dict[str, list[str]] = {c.name: [] for c in self.components} + non_active_targets: list[tuple[str, str]] = [] + for app_name, app_status in status.apps.items(): + charm_name = app_status.charm_name or "" + for component in self.components: + if charm_name in component.charm_names: + if not _is_app_active(app_status): + non_active_targets.append( + (app_name, _app_status_value(app_status)) + ) + break + discovered[component.name].append(app_name) + + if non_active_targets: + details = ", ".join( + f"{app}({state})" for app, state in sorted(non_active_targets) + ) + return Result( + ResultType.FAILED, + "Target applications must be active before backup/restore operations: " + + details, + ) + + return Result(ResultType.COMPLETED, discovered) + + +class ResolveBackupTargetsStep(BaseStep): + """Resolve the unit to back up for every discovered application.""" + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + model: str = OPENSTACK_MODEL, + force: bool = False, + ): + super().__init__( + "Resolve backup targets", + "Resolving units for consistent backup targets", + ) + self.jhelper = jhelper + self.discovered = discovered + self.force = force + self.model = model + + def run(self, context: StepContext) -> Result: + """Return a flat list of resolved backup targets across components.""" + targets: list[BackupTarget] = [] + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + if component is None: + continue + for app in apps: + target = component.resolve_target( + self.jhelper, app, self.model, self.force + ) + if target is None: + self.update_status(context, f"skipped {app}") + continue + targets.append(target) + + return Result(ResultType.COMPLETED, targets) + + +class CheckS3RelationsStep(BaseStep): + """Partition applications by required relation name and S3 interface.""" + + def __init__( + self, + jhelper: JujuHelper, + applications: list[str], + endpoint_name: str, + model: str = OPENSTACK_MODEL, + ): + super().__init__( + "Check S3 relations", + "Checking applications related to S3", + ) + self.jhelper = jhelper + self.applications = applications + self.endpoint_name = endpoint_name + self.model = model + + def run(self, context: StepContext) -> Result: + """Return related/unrelated applications by relation name + interface.""" + try: + status = self.jhelper.get_model_status(self.model) + except (ModelNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + related: list[str] = [] + unrelated: list[str] = [] + for app_name in self.applications: + app_status = status.apps.get(app_name) + if app_status is None: + unrelated.append(app_name) + continue + relations = app_status.relations or {} + endpoint_relations = relations.get(self.endpoint_name, []) + if any(rel.interface == S3_INTERFACE for rel in endpoint_relations): + related.append(app_name) + else: + unrelated.append(app_name) + + return Result( + ResultType.COMPLETED, + { + "related": sorted(related), + "unrelated": sorted(unrelated), + }, + ) + + +class RunBackupsStep(BaseStep): + """Dispatch every target's backup action concurrently and collect results.""" + + def __init__( + self, + jhelper: JujuHelper, + targets: list[BackupTarget], + force: bool = False, + timeout: int = DEFAULT_BACKUP_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Run backups", "Dispatching backups across applications") + self.jhelper = jhelper + self.targets = targets + self.force = force + self.timeout = timeout + self.model = model + + def _action_params(self, component: BackupComponent | None) -> dict | None: + if self.force and component is not None and component.supports_force: + return {"force": True} + return None + + def run(self, context: StepContext) -> Result: + """Dispatch backups concurrently, returning a list of results.""" + components = { + target.component: _component_for(target.component) + for target in self.targets + } + results: list[BackupResult] = [] + with ThreadPoolExecutor(max_workers=len(self.targets)) as executor: + futures = [] + for target in self.targets: + component = components[target.component] + backup_id_key = component.backup_id_key if component else "backup-id" + futures.append( + executor.submit( + _run_single_backup, + self.jhelper, + target, + backup_id_key, + self.model, + self.timeout, + self._action_params(component), + ) + ) + for future in as_completed(futures): + results.append(future.result()) + + return Result(ResultType.COMPLETED, results) + + +class ListBackupsStep(BaseStep): + """Dispatch list-backups actions concurrently and collect inventories.""" + + def __init__( + self, + jhelper: JujuHelper, + targets: list[BackupTarget], + timeout: int = DEFAULT_BACKUP_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("List backups", "Listing backups across applications") + self.jhelper = jhelper + self.targets = targets + self.timeout = timeout + self.model = model + + def run(self, context: StepContext) -> Result: + """Dispatch list-backups concurrently, returning inventory entries.""" + if not self.targets: + return Result(ResultType.COMPLETED, []) + + components = { + target.component: _component_for(target.component) + for target in self.targets + } + inventories: list[BackupInventory] = [] + with ThreadPoolExecutor(max_workers=len(self.targets)) as executor: + futures = [] + for target in self.targets: + component = components[target.component] + action_name = ( + component.list_action + if component is not None + else LIST_BACKUPS_ACTION + ) + parser = component.parse_backup_ids if component is not None else None + futures.append( + executor.submit( + _run_single_list_backups, + self.jhelper, + target, + action_name, + parser, + self.model, + self.timeout, + ) + ) + for future in as_completed(futures): + inventories.append(future.result()) + + return Result(ResultType.COMPLETED, inventories) + + +class WriteBackupManifestStep(BaseStep): + """Write a timestamped manifest of the backup run to the snap share path.""" + + def __init__( + self, + results: list[BackupResult], + dispatched_at: str, + manifest_dir: Path | None = None, + ): + super().__init__("Write backup manifest", "Writing backup manifest") + self.results = results + self.dispatched_at = dispatched_at + self.manifest_dir = manifest_dir + + def run(self, context: StepContext) -> Result: + """Write the manifest and return its path.""" + if self.manifest_dir is not None: + directory = self.manifest_dir + else: + directory = Snap().paths.user_common / BACKUP_MANIFEST_DIR + directory.mkdir(parents=True, exist_ok=True) + + succeeded = sum(1 for r in self.results if r.success) + failed = len(self.results) - succeeded + manifest = { + "dispatched_at": self.dispatched_at, + "summary": {"succeeded": succeeded, "failed": failed}, + "results": [asdict(r) for r in self.results], + } + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + path = directory / f"backup-manifest-{timestamp}.yaml" + try: + with path.open("w") as manifest_file: + yaml.safe_dump(manifest, manifest_file, sort_keys=False) + except OSError as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED, str(path)) + + +class WriteBackupInventoryManifestStep(BaseStep): + """Write a timestamped manifest for backup inventory results.""" + + def __init__( + self, + results: list[BackupInventory], + listed_at: str, + manifest_dir: Path | None = None, + ): + super().__init__("Write backup inventory manifest", "Writing backup inventory") + self.results = results + self.listed_at = listed_at + self.manifest_dir = manifest_dir + + def run(self, context: StepContext) -> Result: + """Write the inventory manifest and return its path.""" + if self.manifest_dir is not None: + directory = self.manifest_dir + else: + directory = Snap().paths.user_common / BACKUP_MANIFEST_DIR + directory.mkdir(parents=True, exist_ok=True) + + succeeded = sum(1 for r in self.results if r.success) + failed = len(self.results) - succeeded + manifest = { + "listed_at": self.listed_at, + "summary": {"succeeded": succeeded, "failed": failed}, + "results": [asdict(r) for r in self.results], + } + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + path = directory / f"backup-inventory-{timestamp}.yaml" + try: + with path.open("w") as manifest_file: + yaml.safe_dump(manifest, manifest_file, sort_keys=False) + except OSError as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED, str(path)) + + +class _ApiAppActionStep(BaseStep): + """Dispatch an action on an application's leader.""" + + def __init__( + self, + jhelper: JujuHelper, + name: str, + description: str, + app: str, + action_name: str, + model: str = OPENSTACK_MODEL, + ): + super().__init__(name, description) + self.jhelper = jhelper + self.model = model + self.app = app + self.action_name = action_name + + def run(self, context: StepContext) -> Result: + """Run action on the application's leader.""" + try: + leader = self.jhelper.get_leader_unit(self.app, self.model) + self.jhelper.run_action( + leader, + self.model, + self.action_name, + timeout=DEFAULT_RESTORE_TIMEOUT, + ) + except ( + ActionFailedException, + LeaderNotFoundException, + ModelNotFoundException, + JujuException, + ) as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED) + + +class PauseAppStep(_ApiAppActionStep): + """Pause application API services.""" + + def __init__(self, jhelper: JujuHelper, app: str, model: str = OPENSTACK_MODEL): + super().__init__( + jhelper, + "Pause App", + "Pause API container services", + app, + PAUSE_ACTION, + model, + ) + + +class ResumeAppStep(_ApiAppActionStep): + """Resume application API services.""" + + def __init__(self, jhelper: JujuHelper, app: str, model: str = OPENSTACK_MODEL): + super().__init__( + jhelper, + "Resume App", + "Resume API container services", + app, + RESUME_ACTION, + model, + ) + + +class ScaleAppStep(BaseStep): + """Scale an application to a target number of units.""" + + def __init__( + self, + jhelper: JujuHelper, + application: str, + scale: int, + timeout: int = DEFAULT_SCALE_TIMEOUT, + model: str = OPENSTACK_MODEL, + ) -> None: + super().__init__("Scale App", f"Scaling {application} to {scale} unit(s)") + self.jhelper = jhelper + self.application = application + self.scale = scale + self.timeout = timeout + self.model = model + + def run(self, context: StepContext) -> Result: + """Scale the application and wait for it to settle.""" + try: + self.jhelper.scale_application(self.model, self.application, self.scale) + self.jhelper.wait_until_active( + self.model, apps=[self.application], timeout=self.timeout + ) + except (JujuException, JujuWaitException, TimeoutError) as e: + return Result(ResultType.FAILED, str(e)) + return Result(ResultType.COMPLETED) + + +class CheckAppPauseResumeSupportStep(BaseStep): + """Validate pause/resume action support for all target applications.""" + + def __init__( + self, + jhelper: JujuHelper, + apps: list[str], + model: str = OPENSTACK_MODEL, + ): + super().__init__( + "Check pause/resume support", + "Validating pause/resume support before restore", + ) + self.jhelper = jhelper + self.apps = apps + self.model = model + + def run(self, context: StepContext) -> Result: + """Fail fast if any app does not support pause/resume actions.""" + unsupported_apps: list[str] = [] + for app in self.apps: + try: + actions = self.jhelper.get_application_actions(app, self.model) + except (JujuException, ModelNotFoundException): + return Result( + ResultType.FAILED, + f"Unable to query actions for {app}. No changes have been made.", + ) + + if PAUSE_ACTION not in actions or RESUME_ACTION not in actions: + unsupported_apps.append(app) + + if unsupported_apps: + details = ", ".join(sorted(unsupported_apps)) + return Result( + ResultType.FAILED, + "The following applications do not support the 'pause/resume' " + f"action required for restore: {details}. No changes have been made.", + ) + + return Result(ResultType.COMPLETED) + + +class RestoreMySQLStep(BaseStep): + """Restore a MySQL application from a backup.""" + + def __init__( + self, + jhelper: JujuHelper, + target: BackupTarget, + restore_to_time: str | None = None, + timeout: int = DEFAULT_RESTORE_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Restore MySQL", f"Restoring {target.app}") + self.jhelper = jhelper + self.target = target + self.restore_to_time = restore_to_time + self.model = model + self.timeout = timeout + + def run(self, context: StepContext) -> Result: + """Restore MySQL using latest backup or restore-to-time.""" + try: + leader = self.jhelper.get_leader_unit(self.target.app, self.model) + except (LeaderNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + params: dict[str, str] = {} + if self.restore_to_time is not None: + params["restore-to-time"] = self.restore_to_time + else: + try: + list_result = self.jhelper.run_action( + leader, + self.model, + LIST_BACKUPS_ACTION, + timeout=self.timeout, + ) + except (ActionFailedException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + latest = _latest_backup_id(_parse_mysql_backup_ids(list_result)) + if latest is None: + return Result( + ResultType.FAILED, + f"No finished MySQL backups found for {self.target.app}.", + ) + params["backup-id"] = latest + + try: + self.jhelper.run_action( + leader, + self.model, + MYSQL_RESTORE_ACTION, + params, + timeout=self.timeout, + ) + except (ActionFailedException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED) + + +class RestoreVaultStep(BaseStep): + """Restore a Vault application from a backup.""" + + def __init__( + self, + jhelper: JujuHelper, + target: BackupTarget, + timeout: int = DEFAULT_RESTORE_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Restore Vault", f"Restoring {target.app}") + self.jhelper = jhelper + self.target = target + self.model = model + self.timeout = timeout + + def run(self, context: StepContext) -> Result: + """Restore Vault from the latest available backup.""" + try: + leader = self.jhelper.get_leader_unit(self.target.app, self.model) + except (LeaderNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + try: + list_result = self.jhelper.run_action( + leader, + self.model, + LIST_BACKUPS_ACTION, + timeout=self.timeout, + ) + except (ActionFailedException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + latest = _latest_backup_id(_parse_vault_backup_ids(list_result)) + if latest is None: + return Result( + ResultType.FAILED, + f"No Vault backups found for {self.target.app}.", + ) + + try: + self.jhelper.run_action( + leader, + self.model, + VAULT_RESTORE_ACTION, + {"backup-id": latest}, + timeout=self.timeout, + ) + except (ActionFailedException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED) diff --git a/sunbeam-python/sunbeam/steps/restore.py b/sunbeam-python/sunbeam/steps/restore.py deleted file mode 100644 index 67eed5110..000000000 --- a/sunbeam-python/sunbeam/steps/restore.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2026 - Canonical Ltd -# SPDX-License-Identifier: Apache-2.0 - -"""Steps for ``sunbeam restore``.""" - -import logging - -from sunbeam.core.common import BaseStep, Result, ResultType, StepContext -from sunbeam.core.juju import ( - JujuException, - JujuHelper, - JujuWaitException, -) -from sunbeam.core.openstack import OPENSTACK_MODEL -from sunbeam.steps.backup import BackupTarget - -LOG = logging.getLogger(__name__) - -DEFAULT_SCALE_TIMEOUT = 1800 -DEFAULT_RESTORE_TIMEOUT = 1800 - - -class PauseControlPlaneStep(BaseStep): - """Pause OpenStack control-plane API services before a MySQL restore.""" - - def __init__(self, jhelper: JujuHelper, model: str = OPENSTACK_MODEL): - super().__init__("Pause control-plane", "Pausing control-plane services") - self.jhelper = jhelper - self.model = model - - def run(self, context: StepContext) -> Result: - """Fail cleanly: pause is not yet supported by the charms.""" - return Result(ResultType.FAILED, "Unimplemented") - - -class ResumeControlPlaneStep(BaseStep): - """Resume OpenStack control-plane API services after a MySQL restore.""" - - def __init__(self, jhelper: JujuHelper, model: str = OPENSTACK_MODEL): - super().__init__("Resume control-plane", "Resuming control-plane services") - self.jhelper = jhelper - self.model = model - - def run(self, context: StepContext) -> Result: - """Fail cleanly: resume is not yet supported by the charms.""" - return Result(ResultType.FAILED, "Unimplemented") - - -class ScaleMySQLStep(BaseStep): - """Scale a MySQL application to a target number of units.""" - - def __init__( - self, - jhelper: JujuHelper, - application: str, - scale: int, - timeout: int = DEFAULT_SCALE_TIMEOUT, - model: str = OPENSTACK_MODEL, - ) -> None: - super().__init__("Scale MySQL", f"Scaling {application} to {scale} unit(s)") - self.jhelper = jhelper - self.application = application - self.scale = scale - self.timeout = timeout - self.model = model - - def run(self, context: StepContext) -> Result: - """Scale the application and wait for it to settle.""" - try: - self.jhelper.scale_application(self.model, self.application, self.scale) - self.jhelper.wait_until_active( - self.model, apps=[self.application], timeout=self.timeout - ) - except (JujuException, JujuWaitException, TimeoutError) as e: - return Result(ResultType.FAILED, str(e)) - return Result(ResultType.COMPLETED) - - -class RestoreMySQLStep(BaseStep): - """Restore a MySQL application from a backup.""" - - def __init__( - self, - jhelper: JujuHelper, - target: BackupTarget, - restore_to_time: str | None = None, - timeout: int = DEFAULT_RESTORE_TIMEOUT, - model: str = OPENSTACK_MODEL, - ): - super().__init__("Restore MySQL", f"Restoring {target.app}") - self.jhelper = jhelper - self.target = target - self.restore_to_time = restore_to_time - self.model = model - self.timeout = timeout - - def run(self, context: StepContext) -> Result: - """Fail cleanly: restore is gated on control-plane pause support.""" - return Result(ResultType.FAILED, "Unimplemented") - - -class RestoreVaultStep(BaseStep): - """Restore a Vault application from a backup. - - Skeleton: surfaces the unseal-keys/root-token prerequisite and does not - perform the restore in this iteration. - """ - - def __init__( - self, - jhelper: JujuHelper, - timeout: int = DEFAULT_RESTORE_TIMEOUT, - model: str = OPENSTACK_MODEL, - ): - super().__init__("Restore Vault", "Restoring Vault") - self.jhelper = jhelper - self.timeout = timeout - - def run(self, context: StepContext) -> Result: - """Fail cleanly, surfacing the Vault restore prerequisite.""" - return Result(ResultType.FAILED, "Unimplemented") diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py index 1af0154ca..3e671af89 100644 --- a/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py @@ -7,13 +7,15 @@ import pytest from click.testing import CliRunner -from sunbeam.commands.backup import backup +from sunbeam.commands.backup_restore import backup def _app_status(charm_name): app = Mock() app.charm_name = charm_name app.units = {} + app.relations = {} + app.app_status.current = "active" return app @@ -58,19 +60,27 @@ def test_prompt_abort_stops_before_work(self, deployment, jhelper): assert "Aborted" in result.output jhelper.get_model_status.assert_not_called() - def test_no_applications_exits_2(self, deployment, jhelper): + def test_no_applications(self, deployment, jhelper): jhelper.get_model_status.return_value = _model_status({}) result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) assert result.exit_code == 2, result.output - assert "Nothing to back up" in result.output - - def test_all_success_exits_0(self, deployment, jhelper): + assert "No applications found to back up. Exiting." in result.output + + def test_all_success(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql_s3_relation = Mock() + mysql_s3_relation.interface = "s3" + mysql.relations = {"s3-parameters": [mysql_s3_relation]} + vault = _app_status("vault-k8s") + vault_s3_relation = Mock() + vault_s3_relation.interface = "s3" + vault.relations = {"s3-parameters": [vault_s3_relation]} jhelper.get_model_status.return_value = _model_status( { - "keystone-mysql": _app_status("mysql-k8s"), - "vault": _app_status("vault-k8s"), + "keystone-mysql": mysql, + "vault": vault, } ) jhelper.get_application.return_value = _app_status("mysql-k8s") @@ -81,11 +91,19 @@ def test_all_success_exits_0(self, deployment, jhelper): assert result.exit_code == 0, result.output assert "2 succeeded, 0 failed" in result.output - def test_partial_failure_exits_1_with_warning(self, deployment, jhelper): + def test_partial_failure(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql_s3_relation = Mock() + mysql_s3_relation.interface = "s3" + mysql.relations = {"s3-parameters": [mysql_s3_relation]} + vault = _app_status("vault-k8s") + vault_s3_relation = Mock() + vault_s3_relation.interface = "s3" + vault.relations = {"s3-parameters": [vault_s3_relation]} jhelper.get_model_status.return_value = _model_status( { - "keystone-mysql": _app_status("mysql-k8s"), - "vault": _app_status("vault-k8s"), + "keystone-mysql": mysql, + "vault": vault, } ) jhelper.get_application.return_value = _app_status("mysql-k8s") @@ -113,10 +131,12 @@ def _run_action(unit, model, action, params=None, timeout=None): assert "1 succeeded, 1 failed" in result.output assert "Warning" in result.output - def test_all_failed_exits_2(self, deployment, jhelper): - jhelper.get_model_status.return_value = _model_status( - {"keystone-mysql": _app_status("mysql-k8s")} - ) + def test_all_failed(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql_s3_relation = Mock() + mysql_s3_relation.interface = "s3" + mysql.relations = {"s3-parameters": [mysql_s3_relation]} + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) jhelper.get_application.return_value = _app_status("mysql-k8s") def _run_action(unit, model, action, params=None, timeout=None): @@ -138,3 +158,95 @@ def _run_action(unit, model, action, params=None, timeout=None): assert result.exit_code == 2, result.output assert "0 succeeded, 1 failed" in result.output + + def test_unrelated_mysql_is_skipped_and_backup_continues(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql.relations = {} + vault = _app_status("vault-k8s") + vault_s3_relation = Mock() + vault_s3_relation.interface = "s3" + vault.relations = {"s3-parameters": [vault_s3_relation]} + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + } + ) + jhelper.get_application.return_value = vault + jhelper.run_action.return_value = {"backup-id": "backup-vault-0"} + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "will be skipped" in result.output + called_units = [call.args[0] for call in jhelper.run_action.call_args_list] + assert called_units == ["vault/0"] + + def test_no_supported_apps_left(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql.relations = {} + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "the following MySQL applications are not related" in result.output + assert "Could not resolve a backup target for any application" in result.output + + def test_unrelated_vault_is_skipped_and_mysql_backup_continues( + self, deployment, jhelper + ): + mysql = _app_status("mysql-k8s") + mysql_s3_relation = Mock() + mysql_s3_relation.interface = "s3" + mysql.relations = {"s3-parameters": [mysql_s3_relation]} + vault = _app_status("vault-k8s") + vault.relations = {} + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + } + ) + jhelper.get_application.return_value = mysql + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + return {"backup-id": "backup-keystone-mysql-0"} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "the following Vault applications are not related" in result.output + + def test_no_supported_apps_left_when_only_vault(self, deployment, jhelper): + vault = _app_status("vault-k8s") + vault.relations = {} + jhelper.get_model_status.return_value = _model_status({"vault": vault}) + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "Could not resolve a backup target for any application" in result.output + + def test_non_active_target_app_fails(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql.app_status.current = "blocked" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "keystone-mysql" in result.output + assert "blocked" in result.output diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py new file mode 100644 index 000000000..886476a3b --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py @@ -0,0 +1,164 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import json +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from sunbeam.commands.backup_restore import list_backups + + +def _app_status(charm_name): + app = Mock() + app.charm_name = charm_name + app.units = {} + app.relations = {} + app.app_status.current = "active" + return app + + +def _model_status(apps): + status = Mock() + status.apps = apps + return status + + +@pytest.fixture +def deployment(): + return Mock() + + +@pytest.fixture +def jhelper(deployment): + jhelper = Mock() + deployment.get_juju_helper.return_value = jhelper + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + return jhelper + + +class TestListBackupsCommand: + def test_no_applications(self, deployment, jhelper): + jhelper.get_model_status.return_value = _model_status({}) + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert "No applications found" in result.output + + def test_no_supported_apps_left(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql.relations = {} + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert "the following MySQL applications are not related" in result.output + assert "No applications found to list backups from. Exiting." in result.output + + def test_lists_backups_and_writes_manifest(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql_s3_relation = Mock() + mysql_s3_relation.interface = "s3" + mysql.relations = {"s3-parameters": [mysql_s3_relation]} + vault = _app_status("vault-k8s") + vault_s3_relation = Mock() + vault_s3_relation.interface = "s3" + vault.relations = {"s3-parameters": [vault_s3_relation]} + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + } + ) + jhelper.get_application.side_effect = lambda app, model: ( + mysql if app == "keystone-mysql" else vault + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + if unit == "keystone-mysql/0": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return { + "backup-ids": json.dumps(["vault-backup-openstack-2026-07-15-00-03-28"]) + } + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 0, result.output + assert "keystone-mysql" in result.output + assert "vault" in result.output + assert "Backup inventory manifest written to" in result.output + + def test_non_active_target_app_fails(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql.app_status.current = "waiting" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 1, result.output + assert "keystone-mysql" in result.output + assert "waiting" in result.output + + def test_list_action_failures_exit_2_with_details(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql_s3_relation = Mock() + mysql_s3_relation.interface = "s3" + mysql.relations = {"s3-parameters": [mysql_s3_relation]} + vault = _app_status("vault-k8s") + vault_s3_relation = Mock() + vault_s3_relation.interface = "s3" + vault.relations = {"s3-parameters": [vault_s3_relation]} + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + } + ) + jhelper.get_application.side_effect = lambda app, model: ( + mysql if app == "keystone-mysql" else vault + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + if action == "list-backups": + raise Exception("list failed") + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert "Failed to list backups for" in result.output + assert "keystone-mysql" in result.output + assert "vault" in result.output diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py index 186893c4b..900c33886 100644 --- a/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py @@ -1,18 +1,22 @@ # SPDX-FileCopyrightText: 2026 - Canonical Ltd # SPDX-License-Identifier: Apache-2.0 +import json from unittest.mock import Mock import pytest from click.testing import CliRunner -from sunbeam.commands.restore import restore +from sunbeam.commands.backup_restore import restore +from sunbeam.core.juju import JujuException def _app_status(charm_name): app = Mock() app.charm_name = charm_name app.units = {} + app.relations = {} + app.app_status.current = "active" return app @@ -22,6 +26,32 @@ def _model_status(apps): return status +def _default_list_backups(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return { + "backup-ids": json.dumps(["vault-backup-openstack-2026-07-15-00-03-28"]) + } + return {} + + @pytest.fixture def deployment(): return Mock() @@ -32,10 +62,24 @@ def jhelper(deployment): jhelper = Mock() deployment.get_juju_helper.return_value = jhelper jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + mysql = _app_status("mysql-k8s") + mysql_s3_relation = Mock() + mysql_s3_relation.interface = "s3" + mysql.relations = {"s3-parameters": [mysql_s3_relation]} + vault = _app_status("vault-k8s") + vault_s3_relation = Mock() + vault_s3_relation.interface = "s3" + vault.relations = {"s3-parameters": [vault_s3_relation]} jhelper.get_model_status.return_value = _model_status( - {"keystone-mysql": _app_status("mysql-k8s"), "vault": _app_status("vault-k8s")} + { + "keystone-mysql": mysql, + "vault": vault, + "keystone-k8s": _app_status("keystone-k8s"), + } ) jhelper.get_application.return_value = _app_status("mysql-k8s") + jhelper.get_application_actions.return_value = ["pause", "resume"] + jhelper.run_action.side_effect = _default_list_backups return jhelper @@ -45,22 +89,100 @@ def test_prompt_abort_stops_before_work(self, deployment, jhelper): assert result.exit_code == 1, result.output assert "Aborted" in result.output - jhelper.get_model_status.assert_not_called() jhelper.scale_application.assert_not_called() + restore_actions = { + call.args[2] + for call in jhelper.run_action.call_args_list + if len(call.args) > 2 + } + assert "restore-backup" not in restore_actions + assert "restore" not in restore_actions def test_stops_at_pause_guard_and_is_non_destructive(self, deployment, jhelper): + jhelper.get_application_actions.return_value = [] + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "pause/resume" in result.output + jhelper.scale_application.assert_not_called() + + def test_prechecks_pause_resume_for_all_apps_before_any_restore_work( + self, deployment, jhelper + ): + mysql_a = _app_status("mysql-k8s") + mysql_a_s3 = Mock() + mysql_a_s3.interface = "s3" + mysql_a.relations = {"s3-parameters": [mysql_a_s3]} + + mysql_b = _app_status("mysql-k8s") + mysql_b_s3 = Mock() + mysql_b_s3.interface = "s3" + mysql_b.relations = {"s3-parameters": [mysql_b_s3]} + + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql_a, + "nova-mysql": mysql_b, + } + ) + + def _get_actions(app, model): + if app == "nova-mysql": + return ["pause"] + return ["pause", "resume"] + + jhelper.get_application_actions.side_effect = _get_actions + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) assert result.exit_code == 1, result.output - assert "Unimplemented" in result.output + assert "pause/resume" in result.output + assert "nova-mysql" in result.output jhelper.scale_application.assert_not_called() + restore_actions = { + call.args[2] + for call in jhelper.run_action.call_args_list + if len(call.args) > 2 + } + assert "pause" not in restore_actions + assert "resume" not in restore_actions + assert "restore" not in restore_actions + def test_no_restore_or_resume_action_dispatched(self, deployment, jhelper): + jhelper.get_application_actions.return_value = [] CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) - for call in jhelper.run_action.call_args_list: - action = call.args[2] if len(call.args) > 2 else "" - assert action not in ("restore-backup", "resume") + restore_actions = { + call.args[2] + for call in jhelper.run_action.call_args_list + if len(call.args) > 2 + } + assert "restore-backup" not in restore_actions + assert "restore" not in restore_actions def test_invalid_restore_to_time_fails_fast(self, deployment, jhelper): result = CliRunner().invoke( @@ -71,3 +193,193 @@ def test_invalid_restore_to_time_fails_fast(self, deployment, jhelper): assert "YYYY-MM-DD HH:MM:SS" in result.output jhelper.get_model_status.assert_not_called() jhelper.scale_application.assert_not_called() + + def test_unrelated_mysql_is_skipped_and_restore_continues( + self, deployment, jhelper + ): + mysql = _app_status("mysql-k8s") + mysql.relations = {} + vault = _app_status("vault-k8s") + vault_s3_relation = Mock() + vault_s3_relation.interface = "s3" + vault.relations = {"s3-parameters": [vault_s3_relation]} + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + "keystone-k8s": _app_status("keystone-k8s"), + } + ) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "will be skipped" in result.output + jhelper.scale_application.assert_not_called() + + def test_no_supported_apps_left(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql.relations = {} + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "No restore targets could be resolved. Exiting." in result.output + + def test_unrelated_vault_is_skipped_and_restore_continues( + self, deployment, jhelper + ): + mysql = _app_status("mysql-k8s") + mysql_s3_relation = Mock() + mysql_s3_relation.interface = "s3" + mysql.relations = {"s3-parameters": [mysql_s3_relation]} + vault = _app_status("vault-k8s") + vault.relations = {} + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + "keystone-k8s": _app_status("keystone-k8s"), + } + ) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "the following Vault applications are not related" in result.output + + def test_warns_on_pitr_for_vault(self, deployment, jhelper): + result = CliRunner().invoke( + restore, + ["--restore-to-time", "2026-07-15 00:00:00", "--no-prompt"], + obj=deployment, + ) + + assert result.exit_code == 0, result.output + assert "Vault does not support point-in-time restore" in result.output + + def test_no_supported_apps_left_when_only_vault(self, deployment, jhelper): + vault = _app_status("vault-k8s") + vault.relations = {} + jhelper.get_model_status.return_value = _model_status({"vault": vault}) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "No restore targets could be resolved. Exiting." in result.output + + def test_no_backups_found(self, deployment, jhelper): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | failed" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return {"backup-ids": json.dumps([])} + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "No backups were found to restore from. Exiting." in result.output + + def test_inventory_lookup_failures_are_reported_and_exit_2( + self, deployment, jhelper + ): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + if action == "list-backups": + raise Exception("list failed") + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "Failed to list backups for" in result.output + assert "keystone-mysql" in result.output + assert "vault" in result.output + + def test_non_active_target_app_fails(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") + mysql.app_status.current = "error" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "keystone-mysql" in result.output + assert "error" in result.output + + def test_mysql_restore_failure_attempts_cleanup_and_reports_app( + self, deployment, jhelper + ): + mysql = _app_status("mysql-k8s") + mysql_s3_relation = Mock() + mysql_s3_relation.interface = "s3" + mysql.relations = {"s3-parameters": [mysql_s3_relation]} + mysql.units = {"keystone-mysql/0": Mock(), "keystone-mysql/1": Mock()} + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + jhelper.get_application.return_value = mysql + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": json.dumps( + { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + ) + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + if action == "restore": + raise JujuException("restore failed") + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "keystone-mysql" in result.output + assert "restore failed" in result.output + assert jhelper.scale_application.call_args_list == [ + ((deployment.openstack_machines_model, "keystone-mysql", 1),), + ((deployment.openstack_machines_model, "keystone-mysql", 2),), + ] diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py index 936700885..49ed50bd9 100644 --- a/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py @@ -10,24 +10,35 @@ LeaderNotFoundException, ModelNotFoundException, ) -from sunbeam.steps.backup import ( +from sunbeam.steps.backup_restore import ( BACKUP_COMPONENTS, + LIST_BACKUPS_ACTION, + MYSQL_S3_RELATION, + S3_INTERFACE, BackupComponent, + BackupInventory, BackupResult, BackupTarget, + CheckS3RelationsStep, DiscoverBackupApplicationsStep, + ListBackupsStep, ResolveBackupTargetsStep, RunBackupsStep, + WriteBackupInventoryManifestStep, WriteBackupManifestStep, + _parse_mysql_backup_ids, + _parse_vault_backup_ids, _resolve_mysql_target, _resolve_vault_target, ) -def _app_status(charm_name, units=None): +def _app_status(charm_name, units=None, relations=None): app = Mock() app.charm_name = charm_name app.units = units or {} + app.relations = relations or {} + app.app_status.current = "active" return app @@ -202,6 +213,23 @@ def test_fails_on_model_error(self, step_context): assert result.result_type == ResultType.FAILED + def test_fails_when_target_app_is_not_active(self, step_context): + jhelper = Mock() + mysql = _app_status("mysql-k8s") + mysql.app_status.current = "blocked" + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": _app_status("vault-k8s"), + } + ) + + result = DiscoverBackupApplicationsStep(jhelper).run(step_context) + + assert result.result_type == ResultType.FAILED + assert "keystone-mysql" in result.message + assert "blocked" in result.message + class TestResolveBackupTargetsStep: def test_flattens_targets_and_skips_unresolvable(self, step_context): @@ -226,11 +254,52 @@ def _leader(app, model): assert apps == {"keystone-mysql", "vault"} -class TestRunBackupsStep: - def test_skips_when_no_targets(self, step_context): - step = RunBackupsStep(Mock(), []) - assert step.is_skip(step_context).result_type == ResultType.SKIPPED +class TestCheckS3RelationsStep: + def test_partitions_apps_by_endpoint_and_s3_interface(self, step_context): + jhelper = Mock() + s3_relation = Mock() + s3_relation.interface = S3_INTERFACE + non_s3_relation = Mock() + non_s3_relation.interface = "other" + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": _app_status( + "mysql-k8s", relations={MYSQL_S3_RELATION: [s3_relation]} + ), + "nova-mysql": _app_status( + "mysql-k8s", relations={MYSQL_S3_RELATION: [non_s3_relation]} + ), + } + ) + result = CheckS3RelationsStep( + jhelper, + ["keystone-mysql", "nova-mysql"], + endpoint_name=MYSQL_S3_RELATION, + ).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message == { + "related": ["keystone-mysql"], + "unrelated": ["nova-mysql"], + } + + def test_missing_app_is_treated_as_unrelated(self, step_context): + jhelper = Mock() + jhelper.get_model_status.return_value = _model_status({}) + + result = CheckS3RelationsStep( + jhelper, ["keystone-mysql"], endpoint_name=MYSQL_S3_RELATION + ).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message == { + "related": [], + "unrelated": ["keystone-mysql"], + } + + +class TestRunBackupsStep: def test_aggregates_mixed_results(self, step_context): jhelper = Mock() @@ -277,6 +346,92 @@ def test_force_passes_force_param_only_to_mysql(self, step_context): assert params_by_unit["vault/0"] is None +class TestListBackupsParsing: + def test_parse_mysql_backup_ids_filters_finished_entries(self): + action_result = { + "backups": ( + "backup-id | backup-type | backup-status\n" + "--------------------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished\n" + "2026-07-14T00:00:00Z | physical | failed" + ) + } + + backup_ids = _parse_mysql_backup_ids(action_result) + + assert backup_ids == ["2026-07-15T00:00:00Z"] + + def test_parse_vault_backup_ids_json_array(self): + action_result = { + "backup-ids": json.dumps( + [ + "vault-backup-openstack-2026-07-15-00-03-28", + "vault-backup-openstack-2026-07-14-00-03-28", + ] + ) + } + + backup_ids = _parse_vault_backup_ids(action_result) + + assert backup_ids == [ + "vault-backup-openstack-2026-07-15-00-03-28", + "vault-backup-openstack-2026-07-14-00-03-28", + ] + + +class TestListBackupsStep: + def test_collects_backup_ids_by_target(self, step_context): + jhelper = Mock() + + def _run_action(unit, model, action, params=None, timeout=None): + assert action == LIST_BACKUPS_ACTION + if unit == "keystone-mysql/1": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return { + "backup-ids": json.dumps(["vault-backup-openstack-2026-07-15-00-03-28"]) + } + + jhelper.run_action.side_effect = _run_action + targets = [ + BackupTarget( + "keystone-mysql", "keystone-mysql/1", "mysql", "create-backup", 2 + ), + BackupTarget("vault", "vault/0", "vault", "create-backup", 1), + ] + + result = ListBackupsStep(jhelper, targets).run(step_context) + + assert result.result_type == ResultType.COMPLETED + by_app = {r.app: r for r in result.message} + assert by_app["keystone-mysql"].success is True + assert by_app["keystone-mysql"].backup_ids == ["2026-07-15T00:00:00Z"] + assert by_app["vault"].success is True + assert by_app["vault"].backup_ids == [ + "vault-backup-openstack-2026-07-15-00-03-28" + ] + + def test_collects_errors_without_raising(self, step_context): + jhelper = Mock() + jhelper.run_action.side_effect = Exception("boom") + targets = [ + BackupTarget( + "keystone-mysql", "keystone-mysql/1", "mysql", "create-backup", 2 + ) + ] + + result = ListBackupsStep(jhelper, targets).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message[0].success is False + assert result.message[0].error == "boom" + + class TestWriteBackupManifestStep: def test_writes_manifest(self, step_context, tmp_path): results = [ @@ -303,6 +458,35 @@ def test_writes_manifest(self, step_context, tmp_path): } +class TestWriteBackupInventoryManifestStep: + def test_writes_inventory_manifest(self, step_context, tmp_path): + results = [ + BackupInventory( + app="keystone-mysql", + unit="keystone-mysql/1", + component="mysql", + success=True, + backup_ids=["2026-07-15T00:00:00Z"], + ), + BackupInventory( + app="vault", + unit="vault/0", + component="vault", + success=False, + error="failed", + ), + ] + step = WriteBackupInventoryManifestStep( + results, "2026-07-15T00:04:28+00:00", manifest_dir=tmp_path + ) + + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + written = list(tmp_path.glob("backup-inventory-*.yaml")) + assert len(written) == 1 + + class TestExtensibility: """Adding a component is a registration change, not a code change (FR-021).""" @@ -319,7 +503,9 @@ def _resolve_fake(jhelper, app, model, force): resolve_target=_resolve_fake, ) components = BACKUP_COMPONENTS + [fake] - monkeypatch.setattr("sunbeam.steps.backup.BACKUP_COMPONENTS", components) + monkeypatch.setattr( + "sunbeam.steps.backup_restore.BACKUP_COMPONENTS", components + ) jhelper = Mock() jhelper.get_model_status.return_value = _model_status( diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py index 34576b878..18a5bceb0 100644 --- a/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py @@ -5,45 +5,106 @@ from sunbeam.core.common import ResultType from sunbeam.core.juju import JujuException -from sunbeam.steps.backup import BackupTarget -from sunbeam.steps.restore import ( - PauseControlPlaneStep, +from sunbeam.steps.backup_restore import ( + BackupTarget, + PauseAppStep, RestoreMySQLStep, RestoreVaultStep, - ResumeControlPlaneStep, - ScaleMySQLStep, + ResumeAppStep, + ScaleAppStep, ) class TestGuardedSteps: - def test_pause_returns_failed(self, step_context): - result = PauseControlPlaneStep(Mock()).run(step_context) - assert result.result_type == ResultType.FAILED - assert "Unimplemented" in result.message + def test_pause_dispatches_action_without_is_skip_check(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-k8s/0" - def test_resume_returns_failed(self, step_context): - result = ResumeControlPlaneStep(Mock()).run(step_context) - assert result.result_type == ResultType.FAILED + step = PauseAppStep(jhelper, app="keystone-k8s") + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + jhelper.get_application_actions.assert_not_called() + jhelper.run_action.assert_called_once() + + def test_resume_dispatches_action(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-k8s/0" - def test_restore_mysql_returns_failed_and_does_not_dispatch(self, step_context): + step = ResumeAppStep(jhelper, app="keystone-k8s") + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + jhelper.run_action.assert_called_once() + + def test_restore_mysql_uses_latest_backup_id(self, step_context): jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.run_action.side_effect = [ + { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + }, + {}, + ] target = BackupTarget( "keystone-mysql", "keystone-mysql/0", "mysql", "restore", 3 ) result = RestoreMySQLStep(jhelper, target).run(step_context) - assert result.result_type == ResultType.FAILED - jhelper.run_action.assert_not_called() - def test_restore_vault_surfaces_prerequisite(self, step_context): - result = RestoreVaultStep(Mock()).run(step_context) - assert result.result_type == ResultType.FAILED - assert "Unimplemented" in result.message + assert result.result_type == ResultType.COMPLETED + assert jhelper.run_action.call_args_list[1].args[2] == "restore" + assert jhelper.run_action.call_args_list[1].args[3] == { + "backup-id": "2026-07-15T00:00:00Z" + } + + def test_restore_mysql_uses_restore_to_time(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + target = BackupTarget( + "keystone-mysql", "keystone-mysql/0", "mysql", "restore", 3 + ) + + result = RestoreMySQLStep( + jhelper, + target, + restore_to_time="2026-07-15 00:00:00", + ).run(step_context) + + assert result.result_type == ResultType.COMPLETED + jhelper.run_action.assert_called_once_with( + "keystone-mysql/0", + "openstack", + "restore", + {"restore-to-time": "2026-07-15 00:00:00"}, + timeout=1800, + ) + + def test_restore_vault_uses_latest_backup(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "vault/0" + jhelper.run_action.side_effect = [ + {"backup-ids": '["vault-backup-openstack-2026-07-15-00-03-28"]'}, + {}, + ] + target = BackupTarget("vault", "vault/0", "vault", "restore-backup", 1) + + result = RestoreVaultStep(jhelper, target).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert jhelper.run_action.call_args_list[1].args[2] == "restore-backup" + assert jhelper.run_action.call_args_list[1].args[3] == { + "backup-id": "vault-backup-openstack-2026-07-15-00-03-28" + } class TestScaleMySQLStep: def test_scales_and_waits(self, step_context): jhelper = Mock() - result = ScaleMySQLStep(jhelper, "keystone-mysql", 1).run(step_context) + result = ScaleAppStep(jhelper, "keystone-mysql", 1).run(step_context) assert result.result_type == ResultType.COMPLETED jhelper.scale_application.assert_called_once_with( "openstack", "keystone-mysql", 1 @@ -53,5 +114,5 @@ def test_scales_and_waits(self, step_context): def test_returns_failed_on_juju_error(self, step_context): jhelper = Mock() jhelper.scale_application.side_effect = JujuException("boom") - result = ScaleMySQLStep(jhelper, "keystone-mysql", 1).run(step_context) + result = ScaleAppStep(jhelper, "keystone-mysql", 1).run(step_context) assert result.result_type == ResultType.FAILED From 68a665b0d69b7387450dfa1c9c77e46d364fe495 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Thu, 16 Jul 2026 15:04:36 -0400 Subject: [PATCH 6/8] fix: use OPENSTACK_MODEL Signed-off-by: Deezzir --- .../sunbeam/commands/backup_restore.py | 133 +++++++++++------- .../sunbeam/steps/backup_restore.py | 94 ++++++------- .../unit/sunbeam/commands/test_backup.py | 48 +++---- .../sunbeam/commands/test_list_backups.py | 20 ++- .../unit/sunbeam/commands/test_restore.py | 75 ++++------ .../tests/unit/sunbeam/steps/test_backup.py | 75 +++++----- 6 files changed, 224 insertions(+), 221 deletions(-) diff --git a/sunbeam-python/sunbeam/commands/backup_restore.py b/sunbeam-python/sunbeam/commands/backup_restore.py index 98728639a..5a2977f91 100644 --- a/sunbeam-python/sunbeam/commands/backup_restore.py +++ b/sunbeam-python/sunbeam/commands/backup_restore.py @@ -14,6 +14,7 @@ from sunbeam.core.common import BaseStep, get_step_message, run_plan from sunbeam.core.deployment import Deployment +from sunbeam.core.openstack import OPENSTACK_MODEL from sunbeam.steps.backup_restore import ( BACKUP_COMPONENTS, DEFAULT_BACKUP_TIMEOUT, @@ -21,7 +22,6 @@ DEFAULT_SCALE_TIMEOUT, MYSQL_S3_RELATION, RESTORE_TIME_FORMAT, - VAULT_PREREQUISITE_MSG, VAULT_S3_RELATION, BackupInventory, BackupResult, @@ -140,17 +140,26 @@ def _list_inventory( def _print_inventory(results: list[BackupInventory]) -> None: table = Table() table.add_column("Application") - table.add_column("Unit") table.add_column("Component") table.add_column("Backup IDs") table.add_column("Status") for result in sorted(results, key=lambda inventory: inventory.app): - status = "[green]done[/green]" if result.success else "[red]failed[/red]" - backup_ids = "-" - if result.success and result.backup_ids: - backup_ids = "\n".join(result.backup_ids) - table.add_row(result.app, result.unit, result.component, backup_ids, status) + if result.backups: + ordered = sorted(result.backups, key=lambda b: b.backup_id, reverse=True) + backup_ids = "\n".join(b.backup_id for b in ordered) + statuses = "\n".join( + "[green]ok[/green]" + if b.success is True + else "[red]failed[/red]" + if b.success is False + else "-" + for b in ordered + ) + else: + backup_ids = "-" + statuses = "[red]failed[/red]" if result.error else "-" + table.add_row(result.app, result.component, backup_ids, statuses) console.print(table) @@ -158,18 +167,22 @@ def _print_inventory(results: list[BackupInventory]) -> None: def _print_summary(results: list[BackupResult]) -> None: table = Table() table.add_column("Application") - table.add_column("Unit") table.add_column("Component") - table.add_column("Status") table.add_column("Backup ID") + table.add_column("Status") for result in results: - status = "[green]done[/green]" if result.success else "[red]failed[/red]" + if result.backup is None: + table.add_row(result.app, result.component, "-", "-") + continue + status = ( + "[green]ok[/green]" + if result.backup.success is True + else "[red]failed[/red]" + if result.backup.success is False + else "-" + ) table.add_row( - result.app, - result.unit, - result.component, - status, - result.backup_id or "-", + result.app, result.component, result.backup.backup_id or "-", status ) console.print(table) @@ -181,28 +194,48 @@ def _filter_restore_targets( """Warn on missing inventory and keep only restorable targets.""" inventory_by_app = {entry.app: entry for entry in inventory} failed_inventory = sorted( - (entry for entry in inventory if not entry.success or not entry.backup_ids), + ( + entry + for entry in inventory + if not entry.backups or not any(b.success is True for b in entry.backups) + ), key=lambda entry: entry.app, ) for entry in failed_inventory: - if not entry.success: + if entry.error: details = f": {entry.error}" if entry.error else "" console.print( "[yellow]Warning:[/yellow] Failed to list backups for " f"{entry.app}{details}" ) - else: + elif not entry.backups: console.print( f"[yellow]Warning:[/yellow] No backups available for {entry.app}." ) + else: + console.print( + f"[yellow]Warning:[/yellow] No successful backups available for" + f" {entry.app}." + ) + + restorable = [] + for target in targets: + target_inventory = inventory_by_app.get(target.app) + if ( + target_inventory is not None + and target_inventory.error is None + and target_inventory.backups + and any(b.success is True for b in target_inventory.backups) + ): + restorable.append(target) + return restorable - return [ - target - for target in targets - if (inventory_by_app.get(target.app) is not None) - and inventory_by_app[target.app].success - and inventory_by_app[target.app].backup_ids - ] + +def _get_api_app_from_mysql_app(app_name: str) -> str: + """Return the API application name for a given MySQL application name.""" + if app_name.endswith("-mysql"): + return app_name.replace("-mysql", "", 1) + return app_name def _run_mysql_restore( @@ -213,8 +246,10 @@ def _run_mysql_restore( timeout: int, ) -> None: """Run the restore plan for a MySQL target.""" + api_app = _get_api_app_from_mysql_app(target.app) + app_plan = [ - PauseAppStep(jhelper, target.app, model=model), + PauseAppStep(jhelper, api_app, model=model), ScaleAppStep( jhelper, target.app, @@ -236,7 +271,7 @@ def _run_mysql_restore( timeout=DEFAULT_SCALE_TIMEOUT, model=model, ), - ResumeAppStep(jhelper, target.app, model=model), + ResumeAppStep(jhelper, api_app, model=model), ] try: @@ -250,7 +285,7 @@ def _run_mysql_restore( timeout=DEFAULT_SCALE_TIMEOUT, model=model, ), - ResumeAppStep(jhelper, target.app, model=model), + ResumeAppStep(jhelper, api_app, model=model), ] try: run_plan(revert_plan, console) @@ -299,15 +334,12 @@ def _validate_restore_to_time( @click.pass_context def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> None: """Create backups of stateful Sunbeam applications (MySQL and Vault).""" - if not no_prompt: - click.confirm(VAULT_PREREQUISITE_MSG, abort=True) - deployment: Deployment = ctx.obj jhelper = deployment.get_juju_helper() - model = deployment.openstack_machines_model + model = OPENSTACK_MODEL console.print( - f"[bold]Backing up [{','.join(c.name for c in BACKUP_COMPONENTS)}]" + f"[bold]Backing up \\[{','.join(c.name for c in BACKUP_COMPONENTS)}]" f" in model '{model}'...[/bold]" ) @@ -322,6 +354,7 @@ def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> No if was_filtered and not no_prompt: click.confirm( "Continue and back up the remaining components?", + default=False, abort=True, ) @@ -336,8 +369,8 @@ def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> No ) sys.exit(EXIT_FAILURE) - dispatched_at = datetime.now(timezone.utc).isoformat() - console.print(f"Dispatching backups at {dispatched_at}...") + dispatched_at = datetime.now(timezone.utc).strftime(RESTORE_TIME_FORMAT) + console.print(f"Dispatching backups at {dispatched_at} UTC...") run_step = RunBackupsStep( jhelper, targets, force=force, timeout=timeout, model=model ) @@ -352,14 +385,17 @@ def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> No if manifest_path: console.print(f"Backup manifest written to: {manifest_path}") - succeeded = sum(1 for r in results if r.success) - failed = len(results) - succeeded + succeeded = sum(1 for r in results if r.backup is not None and r.backup.success) + failed = sum(1 for r in results if r.error is not None) console.print(f"Backup summary: {succeeded} succeeded, {failed} failed.") if failed == 0: sys.exit(EXIT_SUCCESS) - if any(r.component == "mysql" and not r.success for r in results): + if any( + r.component == "mysql" and (r.backup is None or not r.backup.success) + for r in results + ): console.print( "[yellow]Warning:[/yellow] one or more MySQL backups failed. A partial" " restore from this set may result in dangling OpenStack objects." @@ -382,10 +418,10 @@ def list_backups(ctx: click.Context, timeout: int) -> None: """List backup IDs from stateful Sunbeam applications.""" deployment: Deployment = ctx.obj jhelper = deployment.get_juju_helper() - model = deployment.openstack_machines_model + model = OPENSTACK_MODEL console.print( - f"[bold]Listing backups for [{','.join(c.name for c in BACKUP_COMPONENTS)}]" + f"[bold]Listing backups for \\[{','.join(c.name for c in BACKUP_COMPONENTS)}]" f" in model '{model}'...[/bold]" ) @@ -400,13 +436,13 @@ def list_backups(ctx: click.Context, timeout: int) -> None: console, jhelper, discovered, model, force=False ) - listed_at = datetime.now(timezone.utc).isoformat() + listed_at = datetime.now(timezone.utc).strftime(RESTORE_TIME_FORMAT) inventory: list[BackupInventory] = _list_inventory( console, jhelper, targets, model, timeout ) failed_inventory = sorted( - (entry for entry in inventory if not entry.success), + (entry for entry in inventory if entry.error), key=lambda entry: entry.app, ) for entry in failed_inventory: @@ -466,10 +502,10 @@ def restore( """Restore stateful Sunbeam applications from a backup.""" deployment: Deployment = ctx.obj jhelper = deployment.get_juju_helper() - model = deployment.openstack_machines_model + model = OPENSTACK_MODEL console.print( - f"[bold]Restoring [{','.join(c.name for c in BACKUP_COMPONENTS)}]" + f"[bold]Restoring \\[{','.join(c.name for c in BACKUP_COMPONENTS)}]" f" in model '{model}' from backup...[/bold]" ) @@ -484,6 +520,7 @@ def restore( if was_filtered and not no_prompt: click.confirm( "Continue and restore the remaining components?", + default=False, abort=True, ) @@ -509,14 +546,14 @@ def restore( " the latest Vault backup will be used." ) - if any(t.component == "vault" for t in targets) and not no_prompt: - click.confirm(VAULT_PREREQUISITE_MSG, abort=True) - mysql_targets = [t for t in targets if t.component == "mysql"] vault_targets = [t for t in targets if t.component == "vault"] precheck_plan: list[BaseStep] = [] - apps_to_pause_resume = sorted({target.app for target in mysql_targets}) + + apps_to_pause_resume = sorted( + {_get_api_app_from_mysql_app(target.app) for target in mysql_targets} + ) if apps_to_pause_resume: precheck_plan.append( CheckAppPauseResumeSupportStep( diff --git a/sunbeam-python/sunbeam/steps/backup_restore.py b/sunbeam-python/sunbeam/steps/backup_restore.py index d6c9dc8e4..f14acdffb 100644 --- a/sunbeam-python/sunbeam/steps/backup_restore.py +++ b/sunbeam-python/sunbeam/steps/backup_restore.py @@ -38,10 +38,6 @@ S3_INTERFACE = "s3" DEFAULT_BACKUP_TIMEOUT = 300 BACKUP_MANIFEST_DIR = SHARE_PATH / "backups" -VAULT_PREREQUISITE_MSG = ( - "Vault backup/restore requires the unseal keys and root token that were in effect " - "when the backup was created. Are you sure you want to continue?" -) DEFAULT_SCALE_TIMEOUT = 1800 DEFAULT_RESTORE_TIMEOUT = 1800 MYSQL_RESTORE_ACTION = "restore" @@ -63,6 +59,14 @@ class BackupTarget: is_replica: bool = False +@dataclass +class BackupItem: + """A single backup entry in a backup inventory.""" + + backup_id: str + success: bool | None = None + + @dataclass class BackupResult: """The outcome of attempting a backup for a single application.""" @@ -70,8 +74,7 @@ class BackupResult: app: str unit: str component: str - success: bool - backup_id: str | None = None + backup: BackupItem | None = None error: str | None = None @@ -82,8 +85,7 @@ class BackupInventory: app: str unit: str component: str - success: bool - backup_ids: list[str] | None = None + backups: list[BackupItem] | None = None error: str | None = None @@ -96,7 +98,7 @@ class BackupComponent: action: str resolve_target: Callable[[JujuHelper, str, str, bool], BackupTarget | None] list_action: str = LIST_BACKUPS_ACTION - parse_backup_ids: Callable[[dict], list[str]] | None = None + parse_backups: Callable[[dict], list[BackupItem]] | None = None backup_id_key: str = "backup-id" supports_force: bool = False @@ -123,7 +125,7 @@ def _app_status_value(app_status: object) -> str: return "unknown" -def _parse_mysql_backup_ids(action_result: dict) -> list[str]: +def _parse_mysql_backups(action_result: dict) -> list[BackupItem]: """Parse MySQL list-backups output table and return finished backup IDs.""" backups_text = action_result.get("backups") if backups_text is None: @@ -131,7 +133,7 @@ def _parse_mysql_backup_ids(action_result: dict) -> list[str]: if not isinstance(backups_text, str): return [] - backup_ids: list[str] = [] + backups: list[BackupItem] = [] for raw_line in backups_text.splitlines(): line = raw_line.strip() if not line or line.startswith("backup-id") or set(line) == {"-"}: @@ -140,20 +142,16 @@ def _parse_mysql_backup_ids(action_result: dict) -> list[str]: if len(columns) < 3: continue if columns[2].lower() != "finished": + backups.append(BackupItem(backup_id=columns[0], success=False)) continue - backup_ids.append(columns[0]) - return backup_ids + backups.append(BackupItem(columns[0], success=True)) + return backups -def _parse_vault_backup_ids(action_result: dict) -> list[str]: +def _parse_vault_backups(action_result: dict) -> list[BackupItem]: """Parse Vault list-backups output and return backup IDs.""" raw_backup_ids = action_result.get("backup-ids") - if raw_backup_ids is None: - raw_backup_ids = (action_result.get("results") or {}).get("backup-ids") - - if isinstance(raw_backup_ids, list): - return [str(backup_id) for backup_id in raw_backup_ids] - if not isinstance(raw_backup_ids, str): + if raw_backup_ids is None or not isinstance(raw_backup_ids, str): return [] try: @@ -162,7 +160,7 @@ def _parse_vault_backup_ids(action_result: dict) -> list[str]: return [] if not isinstance(parsed, list): return [] - return [str(backup_id) for backup_id in parsed] + return [BackupItem(backup_id=str(backup_id), success=True) for backup_id in parsed] def _secondary_unit_from_status(units: list[str], action_result: dict) -> str | None: @@ -172,16 +170,12 @@ def _secondary_unit_from_status(units: list[str], action_result: dict) -> str | name, so match members flagged SECONDARY back to one of the application's Juju units by ordinal. """ - try: - status = json.loads(action_result.get("status", "{}")) - except (json.JSONDecodeError, TypeError): - return None - + status = action_result.get("status") or {} topology = status.get("defaultreplicaset", {}).get("topology", {}) secondary_labels = [ label for label, info in topology.items() - if isinstance(info, dict) and info.get("memberrole", "").upper() == "SECONDARY" + if isinstance(info, dict) and info.get("memberrole", "").lower() == "secondary" ] if not secondary_labels: return None @@ -194,11 +188,12 @@ def _secondary_unit_from_status(units: list[str], action_result: dict) -> str | return None -def _latest_backup_id(backup_ids: list[str]) -> str | None: - """Return the lexicographically latest backup ID, if present.""" - if not backup_ids: +def _latest_backup(backups: list[BackupItem]) -> str | None: + """Return the lexicographically latest successful backup ID, if present.""" + successful = [b for b in backups if b.success] + if not successful: return None - return sorted(backup_ids)[-1] + return sorted(successful, key=lambda b: b.backup_id)[-1].backup_id def _resolve_mysql_target( @@ -287,19 +282,20 @@ def _run_single_backup( result = jhelper.run_action( target.unit, model, target.action, action_params, timeout=timeout ) + backup_id = result.get(backup_id_key) return BackupResult( app=target.app, unit=target.unit, component=target.component, - success=True, - backup_id=result.get(backup_id_key), + backup=BackupItem(backup_id=backup_id, success=True) + if backup_id is not None + else None, ) except Exception as e: return BackupResult( app=target.app, unit=target.unit, component=target.component, - success=False, error=str(e), ) @@ -308,27 +304,25 @@ def _run_single_list_backups( jhelper: JujuHelper, target: BackupTarget, action_name: str, - parse_backup_ids: Callable[[dict], list[str]] | None, + parse_backups: Callable[[dict], list[BackupItem]] | None, model: str, timeout: int, ) -> BackupInventory: """Dispatch a single list-backups action and parse backup IDs.""" try: result = jhelper.run_action(target.unit, model, action_name, timeout=timeout) - backup_ids = parse_backup_ids(result) if parse_backup_ids is not None else [] + backups = parse_backups(result) if parse_backups is not None else [] return BackupInventory( app=target.app, unit=target.unit, component=target.component, - success=True, - backup_ids=backup_ids, + backups=backups, ) except Exception as e: return BackupInventory( app=target.app, unit=target.unit, component=target.component, - success=False, error=str(e), ) @@ -339,7 +333,7 @@ def _run_single_list_backups( charm_names=[MYSQL_CHARM], action=BACKUP_ACTION, resolve_target=_resolve_mysql_target, - parse_backup_ids=_parse_mysql_backup_ids, + parse_backups=_parse_mysql_backups, supports_force=True, ), BackupComponent( @@ -347,7 +341,7 @@ def _run_single_list_backups( charm_names=[VAULT_CHARM], action=BACKUP_ACTION, resolve_target=_resolve_vault_target, - parse_backup_ids=_parse_vault_backup_ids, + parse_backups=_parse_vault_backups, ), ] @@ -414,7 +408,7 @@ def __init__( ): super().__init__( "Resolve backup targets", - "Resolving units for consistent backup targets", + "Resolving units for backup targets", ) self.jhelper = jhelper self.discovered = discovered @@ -576,7 +570,7 @@ def run(self, context: StepContext) -> Result: if component is not None else LIST_BACKUPS_ACTION ) - parser = component.parse_backup_ids if component is not None else None + parser = component.parse_backups if component is not None else None futures.append( executor.submit( _run_single_list_backups, @@ -616,8 +610,10 @@ def run(self, context: StepContext) -> Result: directory = Snap().paths.user_common / BACKUP_MANIFEST_DIR directory.mkdir(parents=True, exist_ok=True) - succeeded = sum(1 for r in self.results if r.success) - failed = len(self.results) - succeeded + succeeded = sum( + 1 for r in self.results if r.backup is not None and r.backup.success + ) + failed = sum(1 for r in self.results if r.error is not None) manifest = { "dispatched_at": self.dispatched_at, "summary": {"succeeded": succeeded, "failed": failed}, @@ -657,7 +653,9 @@ def run(self, context: StepContext) -> Result: directory = Snap().paths.user_common / BACKUP_MANIFEST_DIR directory.mkdir(parents=True, exist_ok=True) - succeeded = sum(1 for r in self.results if r.success) + succeeded = sum( + 1 for r in self.results if r.backups and any(b.success for b in r.backups) + ) failed = len(self.results) - succeeded manifest = { "listed_at": self.listed_at, @@ -855,7 +853,7 @@ def run(self, context: StepContext) -> Result: except (ActionFailedException, JujuException) as e: return Result(ResultType.FAILED, str(e)) - latest = _latest_backup_id(_parse_mysql_backup_ids(list_result)) + latest = _latest_backup(_parse_mysql_backups(list_result)) if latest is None: return Result( ResultType.FAILED, @@ -910,7 +908,7 @@ def run(self, context: StepContext) -> Result: except (ActionFailedException, JujuException) as e: return Result(ResultType.FAILED, str(e)) - latest = _latest_backup_id(_parse_vault_backup_ids(list_result)) + latest = _latest_backup(_parse_vault_backups(list_result)) if latest is None: return Result( ResultType.FAILED, diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py index 3e671af89..d08fcd30f 100644 --- a/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: 2026 - Canonical Ltd # SPDX-License-Identifier: Apache-2.0 -import json from unittest.mock import Mock import pytest @@ -41,25 +40,16 @@ def jhelper(deployment): def _leader_only_cluster_status(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } return {"backup-id": f"backup-{unit.replace('/', '-')}"} class TestBackupCommand: - def test_prompt_abort_stops_before_work(self, deployment, jhelper): - result = CliRunner().invoke(backup, obj=deployment, input="n\n") - - assert result.exit_code == 1, result.output - assert "Aborted" in result.output - jhelper.get_model_status.assert_not_called() - def test_no_applications(self, deployment, jhelper): jhelper.get_model_status.return_value = _model_status({}) @@ -111,13 +101,11 @@ def test_partial_failure(self, deployment, jhelper): def _run_action(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } if unit == "keystone-mysql/0": raise Exception("backup failed") @@ -142,13 +130,11 @@ def test_all_failed(self, deployment, jhelper): def _run_action(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } raise Exception("backup failed") @@ -213,13 +199,11 @@ def test_unrelated_vault_is_skipped_and_mysql_backup_continues( def _run_action(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } return {"backup-id": "backup-keystone-mysql-0"} diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py index 886476a3b..90486cf9a 100644 --- a/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py @@ -80,13 +80,11 @@ def test_lists_backups_and_writes_manifest(self, deployment, jhelper): def _run_action(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } if unit == "keystone-mysql/0": return { @@ -142,13 +140,11 @@ def test_list_action_failures_exit_2_with_details(self, deployment, jhelper): def _run_action(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } if action == "list-backups": raise Exception("list failed") diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py index 900c33886..51e43de83 100644 --- a/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py @@ -9,6 +9,7 @@ from sunbeam.commands.backup_restore import restore from sunbeam.core.juju import JujuException +from sunbeam.core.openstack import OPENSTACK_MODEL def _app_status(charm_name): @@ -29,13 +30,11 @@ def _model_status(apps): def _default_list_backups(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } if action == "list-backups" and unit.startswith("keystone-mysql"): return { @@ -84,20 +83,6 @@ def jhelper(deployment): class TestRestoreCommand: - def test_prompt_abort_stops_before_work(self, deployment, jhelper): - result = CliRunner().invoke(restore, obj=deployment, input="n\n") - - assert result.exit_code == 1, result.output - assert "Aborted" in result.output - jhelper.scale_application.assert_not_called() - restore_actions = { - call.args[2] - for call in jhelper.run_action.call_args_list - if len(call.args) > 2 - } - assert "restore-backup" not in restore_actions - assert "restore" not in restore_actions - def test_stops_at_pause_guard_and_is_non_destructive(self, deployment, jhelper): jhelper.get_application_actions.return_value = [] result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) @@ -127,8 +112,8 @@ def test_prechecks_pause_resume_for_all_apps_before_any_restore_work( ) def _get_actions(app, model): - if app == "nova-mysql": - return ["pause"] + if app == "nova": + return [] return ["pause", "resume"] jhelper.get_application_actions.side_effect = _get_actions @@ -136,13 +121,11 @@ def _get_actions(app, model): def _run_action(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } if action == "list-backups": return { @@ -160,7 +143,7 @@ def _run_action(unit, model, action, params=None, timeout=None): assert result.exit_code == 1, result.output assert "pause/resume" in result.output - assert "nova-mysql" in result.output + assert "nova" in result.output jhelper.scale_application.assert_not_called() restore_actions = { @@ -273,13 +256,11 @@ def test_no_backups_found(self, deployment, jhelper): def _run_action(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } if action == "list-backups" and unit.startswith("keystone-mysql"): return { @@ -306,13 +287,11 @@ def test_inventory_lookup_failures_are_reported_and_exit_2( def _run_action(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } if action == "list-backups": raise Exception("list failed") @@ -352,13 +331,11 @@ def test_mysql_restore_failure_attempts_cleanup_and_reports_app( def _run_action(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } if action == "list-backups": return { @@ -380,6 +357,6 @@ def _run_action(unit, model, action, params=None, timeout=None): assert "keystone-mysql" in result.output assert "restore failed" in result.output assert jhelper.scale_application.call_args_list == [ - ((deployment.openstack_machines_model, "keystone-mysql", 1),), - ((deployment.openstack_machines_model, "keystone-mysql", 2),), + ((OPENSTACK_MODEL, "keystone-mysql", 1),), + ((OPENSTACK_MODEL, "keystone-mysql", 2),), ] diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py index 49ed50bd9..4cfca93bb 100644 --- a/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py @@ -17,6 +17,7 @@ S3_INTERFACE, BackupComponent, BackupInventory, + BackupItem, BackupResult, BackupTarget, CheckS3RelationsStep, @@ -26,8 +27,8 @@ RunBackupsStep, WriteBackupInventoryManifestStep, WriteBackupManifestStep, - _parse_mysql_backup_ids, - _parse_vault_backup_ids, + _parse_mysql_backups, + _parse_vault_backups, _resolve_mysql_target, _resolve_vault_target, ) @@ -53,7 +54,7 @@ def _cluster_status(secondary_ordinal): "mysql-0.mysql-endpoints": {"memberrole": "PRIMARY"}, f"mysql-{secondary_ordinal}.mysql-endpoints": {"memberrole": "SECONDARY"}, } - return {"status": json.dumps({"defaultreplicaset": {"topology": topology}})} + return {"status": {"defaultreplicaset": {"topology": topology}}} class TestBackupResult: @@ -62,11 +63,10 @@ def test_construction_with_error_does_not_raise(self): app="keystone-mysql", unit="keystone-mysql/0", component="mysql", - success=False, error="boom", ) - assert result.success is False - assert result.backup_id is None + assert result.error == "boom" + assert result.backup is None assert result.error == "boom" @@ -99,13 +99,11 @@ def test_falls_back_to_leader_when_no_secondary(self): jhelper = Mock() jhelper.get_leader_unit.return_value = "cinder-mysql/0" jhelper.run_action.return_value = { - "status": json.dumps( - { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} } - ) + } } jhelper.get_application.return_value = _app_status( "mysql-k8s", units={"cinder-mysql/0": Mock()} @@ -321,11 +319,13 @@ def _run_action(unit, model, action, params=None, timeout=None): assert result.result_type == ResultType.COMPLETED by_app = {r.app: r for r in result.message} - assert by_app["keystone-mysql"].success is True - assert by_app["keystone-mysql"].backup_id == "backup-keystone-mysql-1" - assert by_app["glance-mysql"].success is False + assert by_app["keystone-mysql"].backup is not None + assert by_app["keystone-mysql"].backup.success is True + assert by_app["keystone-mysql"].backup.backup_id == "backup-keystone-mysql-1" + assert by_app["glance-mysql"].backup is None assert by_app["glance-mysql"].error is not None - assert by_app["vault"].success is True + assert by_app["vault"].backup is not None + assert by_app["vault"].backup.success is True def test_force_passes_force_param_only_to_mysql(self, step_context): jhelper = Mock() @@ -357,9 +357,13 @@ def test_parse_mysql_backup_ids_filters_finished_entries(self): ) } - backup_ids = _parse_mysql_backup_ids(action_result) + backups = _parse_mysql_backups(action_result) - assert backup_ids == ["2026-07-15T00:00:00Z"] + assert [b.backup_id for b in backups] == [ + "2026-07-15T00:00:00Z", + "2026-07-14T00:00:00Z", + ] + assert [b.success for b in backups] == [True, False] def test_parse_vault_backup_ids_json_array(self): action_result = { @@ -371,12 +375,13 @@ def test_parse_vault_backup_ids_json_array(self): ) } - backup_ids = _parse_vault_backup_ids(action_result) + backups = _parse_vault_backups(action_result) - assert backup_ids == [ + assert [b.backup_id for b in backups] == [ "vault-backup-openstack-2026-07-15-00-03-28", "vault-backup-openstack-2026-07-14-00-03-28", ] + assert all(b.success for b in backups) class TestListBackupsStep: @@ -409,10 +414,12 @@ def _run_action(unit, model, action, params=None, timeout=None): assert result.result_type == ResultType.COMPLETED by_app = {r.app: r for r in result.message} - assert by_app["keystone-mysql"].success is True - assert by_app["keystone-mysql"].backup_ids == ["2026-07-15T00:00:00Z"] - assert by_app["vault"].success is True - assert by_app["vault"].backup_ids == [ + assert by_app["keystone-mysql"].error is None + assert [b.backup_id for b in by_app["keystone-mysql"].backups] == [ + "2026-07-15T00:00:00Z" + ] + assert by_app["vault"].error is None + assert [b.backup_id for b in by_app["vault"].backups] == [ "vault-backup-openstack-2026-07-15-00-03-28" ] @@ -428,15 +435,20 @@ def test_collects_errors_without_raising(self, step_context): result = ListBackupsStep(jhelper, targets).run(step_context) assert result.result_type == ResultType.COMPLETED - assert result.message[0].success is False + assert result.message[0].backups is None assert result.message[0].error == "boom" class TestWriteBackupManifestStep: def test_writes_manifest(self, step_context, tmp_path): results = [ - BackupResult("keystone-mysql", "keystone-mysql/1", "mysql", True, "id-1"), - BackupResult("glance-mysql", "glance-mysql/1", "mysql", False, None, "err"), + BackupResult( + "keystone-mysql", + "keystone-mysql/1", + "mysql", + BackupItem("id-1", success=True), + ), + BackupResult("glance-mysql", "glance-mysql/1", "mysql", None, "err"), ] step = WriteBackupManifestStep( results, "2026-04-09T14:22:01+00:00", manifest_dir=tmp_path @@ -465,14 +477,12 @@ def test_writes_inventory_manifest(self, step_context, tmp_path): app="keystone-mysql", unit="keystone-mysql/1", component="mysql", - success=True, - backup_ids=["2026-07-15T00:00:00Z"], + backups=[BackupItem("2026-07-15T00:00:00Z", success=True)], ), BackupInventory( app="vault", unit="vault/0", component="vault", - success=False, error="failed", ), ] @@ -524,5 +534,6 @@ def _resolve_fake(jhelper, app, model, force): run = RunBackupsStep(jhelper, fake_targets).run(step_context) assert run.message[0].component == "fake" - assert run.message[0].success is True - assert run.message[0].backup_id == "fake-backup" + assert run.message[0].backup is not None + assert run.message[0].backup.success is True + assert run.message[0].backup.backup_id == "fake-backup" From 552f9123f0b87ce6419cb093d70c410b60395ed6 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Fri, 17 Jul 2026 00:34:07 -0400 Subject: [PATCH 7/8] refactor: backup/restore component agnostic wrappers Signed-off-by: Deezzir --- .../sunbeam/commands/backup_restore.py | 385 ++--- .../sunbeam/steps/backup_restore.py | 1387 ++++++++++------- .../unit/sunbeam/commands/test_backup.py | 138 +- .../sunbeam/commands/test_list_backups.py | 96 +- .../unit/sunbeam/commands/test_restore.py | 105 +- .../tests/unit/sunbeam/steps/test_backup.py | 318 ++-- .../tests/unit/sunbeam/steps/test_restore.py | 150 +- 7 files changed, 1447 insertions(+), 1132 deletions(-) diff --git a/sunbeam-python/sunbeam/commands/backup_restore.py b/sunbeam-python/sunbeam/commands/backup_restore.py index 5a2977f91..fa56ea2fc 100644 --- a/sunbeam-python/sunbeam/commands/backup_restore.py +++ b/sunbeam-python/sunbeam/commands/backup_restore.py @@ -6,37 +6,30 @@ import logging import sys from datetime import datetime, timezone -from typing import Tuple import click from rich.console import Console from rich.table import Table -from sunbeam.core.common import BaseStep, get_step_message, run_plan +from sunbeam.core.common import get_step_message, run_plan from sunbeam.core.deployment import Deployment from sunbeam.core.openstack import OPENSTACK_MODEL +from sunbeam.core.questions import ConfirmQuestion, Question from sunbeam.steps.backup_restore import ( BACKUP_COMPONENTS, DEFAULT_BACKUP_TIMEOUT, DEFAULT_RESTORE_TIMEOUT, - DEFAULT_SCALE_TIMEOUT, - MYSQL_S3_RELATION, RESTORE_TIME_FORMAT, - VAULT_S3_RELATION, + ActionTarget, BackupInventory, BackupResult, - BackupTarget, - CheckAppPauseResumeSupportStep, - CheckS3RelationsStep, DiscoverBackupApplicationsStep, ListBackupsStep, - PauseAppStep, - ResolveBackupTargetsStep, - RestoreMySQLStep, - RestoreVaultStep, - ResumeAppStep, + ResolveActionTargetsStep, + RestoreResult, + RestoreStep, RunBackupsStep, - ScaleAppStep, + ValidateStep, WriteBackupInventoryManifestStep, WriteBackupManifestStep, ) @@ -48,91 +41,94 @@ EXIT_PARTIAL = 1 EXIT_FAILURE = 2 -COMPONENT_LABELS = {"mysql": "MySQL", "vault": "Vault"} -S3_RELATIONS = ( - ("mysql", MYSQL_S3_RELATION), - ("vault", VAULT_S3_RELATION), -) +def backup_questions() -> dict[str, Question]: + """Confirmation questions for the backup/restore commands.""" + return { + "continue_backup": ConfirmQuestion( + "Continue and back up the remaining components?", + default_value=False, + description=( + "Some discovered applications were skipped because they are not" + " ready for backup." + ), + ), + "continue_restore": ConfirmQuestion( + "Continue and restore the remaining components?", + default_value=False, + description=( + "Some discovered applications were skipped because they are not" + " ready for restore. Partial restores may result in " + "dangling OpenStack objects. " + ), + ), + } + + +def _confirm_or_abort(question: Question, no_prompt: bool) -> None: + """Ask a confirmation question, aborting the command if declined.""" + if no_prompt: + return + question.console = console + question.show_hint = True + if not question.ask(): + raise click.Abort() + + +def _components_banner() -> str: + return ",".join(c.name for c in BACKUP_COMPONENTS) -def _discover_applications( - console_: Console, jhelper, model: str, components -) -> dict[str, list[str]]: +def _discover_apps(jhelper, model: str) -> dict[str, list[str]]: """Discover applications for every registered backup component.""" results = run_plan( - [DiscoverBackupApplicationsStep(jhelper, model, components)], console_ + [DiscoverBackupApplicationsStep(jhelper, model, BACKUP_COMPONENTS)], console ) return get_step_message(results, DiscoverBackupApplicationsStep) -def _filter_s3_related_apps( - console_: Console, +def _validate_apps( jhelper, discovered: dict[str, list[str]], model: str, -) -> Tuple[dict[str, list[str]], bool]: - """Filter discovered apps to those related to the required S3 endpoints.""" - was_filtered = False - filtered: dict[str, list[str]] = {} - - for component_name, endpoint_name in S3_RELATIONS: - applications = discovered.get(component_name, []) - if not applications: - continue - - s3_check_results = run_plan( - [ - CheckS3RelationsStep( - jhelper, - applications, - endpoint_name=endpoint_name, - model=model, - ) - ], - console_, +) -> tuple[dict[str, list[str]], bool]: + """Run the validation step and warn on every skipped application.""" + results = run_plan([ValidateStep(jhelper, discovered, model=model)], console) + outcome = get_step_message(results, ValidateStep) + valid = outcome["valid"] + failures = outcome["failures"] + + for app in sorted(failures): + reasons = ", ".join(failures[app]) + console.print( + f"[yellow]Warning:[/yellow] {app} is not ready for backup" + f" ({reasons}) and will be skipped." ) - s3_relations = get_step_message(s3_check_results, CheckS3RelationsStep) - unrelated_apps = s3_relations["unrelated"] - filtered[component_name] = s3_relations["related"] - - if unrelated_apps: - was_filtered = True - label = COMPONENT_LABELS.get(component_name, component_name) - console_.print( - f"[yellow]Warning:[/yellow] the following {label} applications are" - f" not related via '{endpoint_name}' and will be skipped:" - f" {', '.join(unrelated_apps)}" - ) - return filtered, was_filtered + return valid, bool(failures) -def _resolve_targets( - console_: Console, +def _resolve_action_targets( jhelper, discovered: dict[str, list[str]], model: str, - force: bool, -) -> list[BackupTarget]: - """Resolve the unit to act on for every discovered application.""" +) -> list[ActionTarget]: + """Resolve leader targets for generic actions (e.g. list-backups).""" results = run_plan( - [ResolveBackupTargetsStep(jhelper, discovered, model=model, force=force)], - console_, + [ResolveActionTargetsStep(jhelper, discovered, model=model)], console ) - return get_step_message(results, ResolveBackupTargetsStep) + return get_step_message(results, ResolveActionTargetsStep) -def _list_inventory( - console_: Console, +def _list_backup_inventory( jhelper, - targets: list[BackupTarget], + targets: list[ActionTarget], model: str, timeout: int, ) -> list[BackupInventory]: """List available backups for the given targets.""" results = run_plan( - [ListBackupsStep(jhelper, targets, timeout=timeout, model=model)], console_ + [ListBackupsStep(jhelper, targets, timeout=timeout, model=model)], console ) return get_step_message(results, ListBackupsStep) @@ -164,7 +160,7 @@ def _print_inventory(results: list[BackupInventory]) -> None: console.print(table) -def _print_summary(results: list[BackupResult]) -> None: +def _print_backup_summary(results: list[BackupResult]) -> None: table = Table() table.add_column("Application") table.add_column("Component") @@ -188,9 +184,9 @@ def _print_summary(results: list[BackupResult]) -> None: def _filter_restore_targets( - targets: list[BackupTarget], + targets: list[ActionTarget], inventory: list[BackupInventory], -) -> list[BackupTarget]: +) -> list[ActionTarget]: """Warn on missing inventory and keep only restorable targets.""" inventory_by_app = {entry.app: entry for entry in inventory} failed_inventory = sorted( @@ -203,10 +199,9 @@ def _filter_restore_targets( ) for entry in failed_inventory: if entry.error: - details = f": {entry.error}" if entry.error else "" console.print( "[yellow]Warning:[/yellow] Failed to list backups for " - f"{entry.app}{details}" + f"{entry.app}: {entry.error}" ) elif not entry.backups: console.print( @@ -231,72 +226,6 @@ def _filter_restore_targets( return restorable -def _get_api_app_from_mysql_app(app_name: str) -> str: - """Return the API application name for a given MySQL application name.""" - if app_name.endswith("-mysql"): - return app_name.replace("-mysql", "", 1) - return app_name - - -def _run_mysql_restore( - jhelper, - target: BackupTarget, - model: str, - restore_to_time: str | None, - timeout: int, -) -> None: - """Run the restore plan for a MySQL target.""" - api_app = _get_api_app_from_mysql_app(target.app) - - app_plan = [ - PauseAppStep(jhelper, api_app, model=model), - ScaleAppStep( - jhelper, - target.app, - 1, - timeout=DEFAULT_SCALE_TIMEOUT, - model=model, - ), - RestoreMySQLStep( - jhelper, - target, - restore_to_time=restore_to_time, - timeout=timeout, - model=model, - ), - ScaleAppStep( - jhelper, - target.app, - target.scale, - timeout=DEFAULT_SCALE_TIMEOUT, - model=model, - ), - ResumeAppStep(jhelper, api_app, model=model), - ] - - try: - run_plan(app_plan, console) - except click.ClickException as e: - revert_plan = [ - ScaleAppStep( - jhelper, - target.app, - target.scale, - timeout=DEFAULT_SCALE_TIMEOUT, - model=model, - ), - ResumeAppStep(jhelper, api_app, model=model), - ] - try: - run_plan(revert_plan, console) - except click.ClickException: - pass - - raise click.ClickException( - f"{target.app} restore failed: {e.message or str(e)}. Attempted to revert." - ) - - def _validate_restore_to_time( ctx: click.Context, param: click.Parameter, value: str | None ) -> str | None: @@ -339,48 +268,46 @@ def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> No model = OPENSTACK_MODEL console.print( - f"[bold]Backing up \\[{','.join(c.name for c in BACKUP_COMPONENTS)}]" - f" in model '{model}'...[/bold]" + f"[bold]Backing up \\[{_components_banner()}] in model '{model}'...[/bold]" ) - discovered = _discover_applications(console, jhelper, model, BACKUP_COMPONENTS) + discovered = _discover_apps(jhelper, model) if not any(discovered.values()): console.print("No applications found to back up. Exiting.") sys.exit(EXIT_FAILURE) - discovered, was_filtered = _filter_s3_related_apps( - console, jhelper, discovered, model - ) - if was_filtered and not no_prompt: - click.confirm( - "Continue and back up the remaining components?", - default=False, - abort=True, - ) + discovered, was_filtered = _validate_apps(jhelper, discovered, model) + if was_filtered: + _confirm_or_abort(backup_questions()["continue_backup"], no_prompt) + + if not any(discovered.values()): + console.print("No applications remain to back up after validation. Exiting.") + sys.exit(EXIT_FAILURE) - targets: list[BackupTarget] = _resolve_targets( - console, jhelper, discovered, model, force=force + dispatched_at = datetime.now(timezone.utc).strftime(RESTORE_TIME_FORMAT) + console.print(f"Dispatching backups at {dispatched_at} UTC...") + backup_results = run_plan( + [ + RunBackupsStep( + jhelper, discovered, force=force, timeout=timeout, model=model + ) + ], + console, ) + results: list[BackupResult] = get_step_message(backup_results, RunBackupsStep) - if not targets: + if not results: console.print( "Could not resolve a backup target for any application. Re-run with" " --force to back up on leader units regardless of cluster health." ) sys.exit(EXIT_FAILURE) - dispatched_at = datetime.now(timezone.utc).strftime(RESTORE_TIME_FORMAT) - console.print(f"Dispatching backups at {dispatched_at} UTC...") - run_step = RunBackupsStep( - jhelper, targets, force=force, timeout=timeout, model=model - ) - backup_results = run_plan([run_step], console) - results: list[BackupResult] = get_step_message(backup_results, RunBackupsStep) + _print_backup_summary(results) - _print_summary(results) - - manifest_step = WriteBackupManifestStep(results, dispatched_at) - manifest_results = run_plan([manifest_step], console) + manifest_results = run_plan( + [WriteBackupManifestStep(results, dispatched_at)], console + ) manifest_path = get_step_message(manifest_results, WriteBackupManifestStep) if manifest_path: console.print(f"Backup manifest written to: {manifest_path}") @@ -392,14 +319,10 @@ def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> No if failed == 0: sys.exit(EXIT_SUCCESS) - if any( - r.component == "mysql" and (r.backup is None or not r.backup.success) - for r in results - ): - console.print( - "[yellow]Warning:[/yellow] one or more MySQL backups failed. A partial" - " restore from this set may result in dangling OpenStack objects." - ) + console.print( + "[yellow]Warning:[/yellow] one or more backups failed or timed out. A partial" + " restore from a set may result in dangling OpenStack objects." + ) if succeeded == 0: sys.exit(EXIT_FAILURE) @@ -421,44 +344,38 @@ def list_backups(ctx: click.Context, timeout: int) -> None: model = OPENSTACK_MODEL console.print( - f"[bold]Listing backups for \\[{','.join(c.name for c in BACKUP_COMPONENTS)}]" + f"[bold]Listing backups for \\[{_components_banner()}]" f" in model '{model}'...[/bold]" ) - discovered = _discover_applications(console, jhelper, model, BACKUP_COMPONENTS) - discovered, _ = _filter_s3_related_apps(console, jhelper, discovered, model) + discovered = _discover_apps(jhelper, model) + discovered, _ = _validate_apps(jhelper, discovered, model) if not any(discovered.values()): console.print("No applications found to list backups from. Exiting.") sys.exit(EXIT_FAILURE) - targets: list[BackupTarget] = _resolve_targets( - console, jhelper, discovered, model, force=False - ) + targets = _resolve_action_targets(jhelper, discovered, model) listed_at = datetime.now(timezone.utc).strftime(RESTORE_TIME_FORMAT) - inventory: list[BackupInventory] = _list_inventory( - console, jhelper, targets, model, timeout - ) + inventory = _list_backup_inventory(jhelper, targets, model, timeout) failed_inventory = sorted( (entry for entry in inventory if entry.error), key=lambda entry: entry.app, ) for entry in failed_inventory: - details = f": {entry.error}" if entry.error else "" console.print( - f"[yellow]Warning:[/yellow] Failed to list backups for {entry.app}{details}" + f"[yellow]Warning:[/yellow] Failed to list backups for" + f" {entry.app}: {entry.error}" ) _print_inventory(inventory) - manifest_step = WriteBackupInventoryManifestStep(inventory, listed_at) - manifest_results = run_plan([manifest_step], console) - manifest_path = get_step_message( - manifest_results, - WriteBackupInventoryManifestStep, + manifest_results = run_plan( + [WriteBackupInventoryManifestStep(inventory, listed_at)], console ) + manifest_path = get_step_message(manifest_results, WriteBackupInventoryManifestStep) if manifest_path: console.print(f"Backup inventory manifest written to: {manifest_path}") @@ -505,81 +422,59 @@ def restore( model = OPENSTACK_MODEL console.print( - f"[bold]Restoring \\[{','.join(c.name for c in BACKUP_COMPONENTS)}]" + f"[bold]Restoring \\[{_components_banner()}]" f" in model '{model}' from backup...[/bold]" ) - discovered = _discover_applications(console, jhelper, model, BACKUP_COMPONENTS) + discovered = _discover_apps(jhelper, model) if not any(discovered.values()): console.print("No applications found to restore. Exiting.") sys.exit(EXIT_FAILURE) - discovered, was_filtered = _filter_s3_related_apps( - console, jhelper, discovered, model - ) - if was_filtered and not no_prompt: - click.confirm( - "Continue and restore the remaining components?", - default=False, - abort=True, - ) + discovered, was_filtered = _validate_apps(jhelper, discovered, model) + if was_filtered: + _confirm_or_abort(backup_questions()["continue_restore"], no_prompt) - targets: list[BackupTarget] = _resolve_targets( - console, jhelper, discovered, model, force=force - ) + targets = _resolve_action_targets(jhelper, discovered, model) if not targets: console.print("No restore targets could be resolved. Exiting.") sys.exit(EXIT_FAILURE) - inventory: list[BackupInventory] = _list_inventory( - console, jhelper, targets, model, timeout - ) + inventory = _list_backup_inventory(jhelper, targets, model, timeout) targets = _filter_restore_targets(targets, inventory) if not targets: console.print("No backups were found to restore from. Exiting.") sys.exit(EXIT_FAILURE) - if restore_to_time is not None and any(t.component == "vault" for t in targets): - console.print( - "[yellow]Warning:[/yellow] Vault does not support point-in-time restore;" - " the latest Vault backup will be used." - ) - - mysql_targets = [t for t in targets if t.component == "mysql"] - vault_targets = [t for t in targets if t.component == "vault"] - - precheck_plan: list[BaseStep] = [] - - apps_to_pause_resume = sorted( - {_get_api_app_from_mysql_app(target.app) for target in mysql_targets} - ) - if apps_to_pause_resume: - precheck_plan.append( - CheckAppPauseResumeSupportStep( + restore_results = run_plan( + [ + RestoreStep( jhelper, - apps_to_pause_resume, + targets, + restore_to_time=restore_to_time, + timeout=timeout, model=model, ) - ) - - if precheck_plan: - run_plan(precheck_plan, console) + ], + console, + ) + results: list[RestoreResult] = get_step_message(restore_results, RestoreStep) - for target in mysql_targets: - _run_mysql_restore( - jhelper, - target, - model, - restore_to_time, - timeout, - ) + for result in results: + if not result.success: + reverted = " Reverted." if result.reverted else "" + console.print( + f"[red]Error:[/red] {result.app} restore failed:" + f" {result.error}.{reverted}" + ) - vault_plan: list[BaseStep] = [] - for target in vault_targets: - vault_plan.append( - RestoreVaultStep(jhelper, target, timeout=timeout, model=model) - ) + succeeded = sum(1 for r in results if r.success) + failed = sum(1 for r in results if not r.success) + console.print(f"Restore summary: {succeeded} succeeded, {failed} failed.") - if vault_targets: - run_plan(vault_plan, console) + if failed == 0: + sys.exit(EXIT_SUCCESS) + if succeeded == 0: + sys.exit(EXIT_FAILURE) + sys.exit(EXIT_PARTIAL) diff --git a/sunbeam-python/sunbeam/steps/backup_restore.py b/sunbeam-python/sunbeam/steps/backup_restore.py index f14acdffb..1a08a520c 100644 --- a/sunbeam-python/sunbeam/steps/backup_restore.py +++ b/sunbeam-python/sunbeam/steps/backup_restore.py @@ -1,14 +1,21 @@ # SPDX-FileCopyrightText: 2026 - Canonical Ltd # SPDX-License-Identifier: Apache-2.0 -"""Steps and component registry for ``sunbeam backup/restore``.""" +"""Steps and component registry for ``sunbeam backup/restore``. + +The module composes and +runs the top-level steps defined here. All per-component logic (validation +checks, target resolution, and backup/list/restore/revert plans) lives on the +:class:`BackupComponent` registry and is driven by the wrapper steps below. +""" import json import logging from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field from datetime import datetime, timezone +from enum import Enum from pathlib import Path import yaml @@ -31,36 +38,35 @@ MYSQL_CHARM = "mysql-k8s" VAULT_CHARM = "vault-k8s" BACKUP_ACTION = "create-backup" +RESTORE_ACTION = "restore" +BACKUP_RESULT_ID_KEY = "backup-id" LIST_BACKUPS_ACTION = "list-backups" -CLUSTER_STATUS_ACTION = "get-cluster-status" -MYSQL_S3_RELATION = "s3-parameters" -VAULT_S3_RELATION = "s3-parameters" +VAULT_RESTORE_ACTION = "restore-backup" +MYSQL_CLUSTER_STATUS_ACTION = "get-cluster-status" S3_INTERFACE = "s3" -DEFAULT_BACKUP_TIMEOUT = 300 -BACKUP_MANIFEST_DIR = SHARE_PATH / "backups" +S3_ENDPOINT = "s3-parameters" +DEFAULT_BACKUP_TIMEOUT = 1800 DEFAULT_SCALE_TIMEOUT = 1800 DEFAULT_RESTORE_TIMEOUT = 1800 -MYSQL_RESTORE_ACTION = "restore" -VAULT_RESTORE_ACTION = "restore-backup" +BACKUP_MANIFEST_DIR = SHARE_PATH / "backups" PAUSE_ACTION = "pause" RESUME_ACTION = "resume" RESTORE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S" @dataclass -class BackupTarget: - """An application and the unit chosen to run its backup action against.""" +class ActionTarget: + """An application and the unit chosen to run an action against.""" app: str unit: str component: str action: str - scale: int - is_replica: bool = False + scale: int = 1 @dataclass -class BackupItem: +class BackupOutcome: """A single backup entry in a backup inventory.""" backup_id: str @@ -74,7 +80,7 @@ class BackupResult: app: str unit: str component: str - backup: BackupItem | None = None + backup: BackupOutcome | None = None error: str | None = None @@ -85,8 +91,45 @@ class BackupInventory: app: str unit: str component: str - backups: list[BackupItem] | None = None + backups: list[BackupOutcome] | None = None + error: str | None = None + + +@dataclass +class RestoreResult: + """The outcome of attempting a restore for a single application.""" + + app: str + component: str + success: bool error: str | None = None + reverted: bool = False + + +class ValidationCheck(Enum): + """An application-readiness check (predicate resolved by identity).""" + + ACTIVE = "active" + S3_RELATION = "s3-relation" + + +ResolveTargetFn = Callable[[JujuHelper, str, str, bool], "ActionTarget | None"] +BackupPlanFn = Callable[ + [JujuHelper, "BackupComponent", "ActionTarget", bool, int, str], + list[BaseStep], +] +RestorePlanFn = Callable[ + [JujuHelper, "BackupComponent", "ActionTarget", str | None, int, str], + list[BaseStep], +] +RevertPlanFn = Callable[ + [JujuHelper, "BackupComponent", "ActionTarget", str], + list[BaseStep], +] +PrecheckPlanFn = Callable[ + [JujuHelper, "BackupComponent", "ActionTarget", str], + list[BaseStep], +] @dataclass @@ -94,19 +137,32 @@ class BackupComponent: """Descriptor for a kind of stateful application that can be backed up.""" name: str - charm_names: list[str] - action: str - resolve_target: Callable[[JujuHelper, str, str, bool], BackupTarget | None] + + resolve_backup_target: ResolveTargetFn + parse_backup_list: Callable[[dict], list[BackupOutcome]] + build_backup_plan: BackupPlanFn + build_restore_plan: RestorePlanFn + build_restore_revert_plan: RevertPlanFn | None = None + build_restore_precheck_plan: PrecheckPlanFn | None = None + + backup_action: str = BACKUP_ACTION + restore_action: str = RESTORE_ACTION list_action: str = LIST_BACKUPS_ACTION - parse_backups: Callable[[dict], list[BackupItem]] | None = None - backup_id_key: str = "backup-id" + backup_id_key: str = BACKUP_RESULT_ID_KEY + supports_force: bool = False + supports_restore_to_time: bool = False + + validate_checks: list[ValidationCheck] = field(default_factory=list) def _component_for(name: str) -> BackupComponent | None: return next((c for c in BACKUP_COMPONENTS if c.name == name), None) +# --------------------------------------------------------------------------- +# Validation predicates +# --------------------------------------------------------------------------- def _is_app_active(app_status: object) -> bool: """Return whether application workload status is active.""" status = getattr(app_status, "app_status", None) @@ -116,16 +172,23 @@ def _is_app_active(app_status: object) -> bool: return current == "active" -def _app_status_value(app_status: object) -> str: - """Return application workload status string for display/logging.""" - status = getattr(app_status, "app_status", None) - current = getattr(status, "current", None) - if isinstance(current, str): - return current - return "unknown" +def _is_related_to_s3(app_status: object) -> bool: + """Return whether the application is related to S3 via the endpoint.""" + relations = getattr(app_status, "relations", None) or {} + endpoint_relations = relations.get(S3_ENDPOINT, []) + return any(rel.interface == S3_INTERFACE for rel in endpoint_relations) -def _parse_mysql_backups(action_result: dict) -> list[BackupItem]: +_VALIDATION_PREDICATES: dict[ValidationCheck, Callable[[object], bool]] = { + ValidationCheck.ACTIVE: _is_app_active, + ValidationCheck.S3_RELATION: _is_related_to_s3, +} + + +# --------------------------------------------------------------------------- +# Backup-list parsing +# --------------------------------------------------------------------------- +def _parse_mysql_backups(action_result: dict) -> list[BackupOutcome]: """Parse MySQL list-backups output table and return finished backup IDs.""" backups_text = action_result.get("backups") if backups_text is None: @@ -133,7 +196,7 @@ def _parse_mysql_backups(action_result: dict) -> list[BackupItem]: if not isinstance(backups_text, str): return [] - backups: list[BackupItem] = [] + backups: list[BackupOutcome] = [] for raw_line in backups_text.splitlines(): line = raw_line.strip() if not line or line.startswith("backup-id") or set(line) == {"-"}: @@ -142,13 +205,13 @@ def _parse_mysql_backups(action_result: dict) -> list[BackupItem]: if len(columns) < 3: continue if columns[2].lower() != "finished": - backups.append(BackupItem(backup_id=columns[0], success=False)) + backups.append(BackupOutcome(backup_id=columns[0], success=False)) continue - backups.append(BackupItem(columns[0], success=True)) + backups.append(BackupOutcome(columns[0], success=True)) return backups -def _parse_vault_backups(action_result: dict) -> list[BackupItem]: +def _parse_vault_backups(action_result: dict) -> list[BackupOutcome]: """Parse Vault list-backups output and return backup IDs.""" raw_backup_ids = action_result.get("backup-ids") if raw_backup_ids is None or not isinstance(raw_backup_ids, str): @@ -160,9 +223,14 @@ def _parse_vault_backups(action_result: dict) -> list[BackupItem]: return [] if not isinstance(parsed, list): return [] - return [BackupItem(backup_id=str(backup_id), success=True) for backup_id in parsed] + return [ + BackupOutcome(backup_id=str(backup_id), success=True) for backup_id in parsed + ] +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- def _secondary_unit_from_status(units: list[str], action_result: dict) -> str | None: """Map a SECONDARY cluster member to a Juju unit name. @@ -188,7 +256,7 @@ def _secondary_unit_from_status(units: list[str], action_result: dict) -> str | return None -def _latest_backup(backups: list[BackupItem]) -> str | None: +def _latest_backup(backups: list[BackupOutcome]) -> str | None: """Return the lexicographically latest successful backup ID, if present.""" successful = [b for b in backups if b.success] if not successful: @@ -196,9 +264,30 @@ def _latest_backup(backups: list[BackupItem]) -> str | None: return sorted(successful, key=lambda b: b.backup_id)[-1].backup_id -def _resolve_mysql_target( - jhelper: JujuHelper, app: str, model: str, force: bool -) -> BackupTarget | None: +def _api_app_for_mysql(app_name: str) -> str: + """Return the API application name for a given MySQL application name.""" + if app_name.endswith("-mysql"): + return app_name.replace("-mysql", "", 1) + return app_name + + +def _current_scale(jhelper: JujuHelper, app: str, model: str, fallback: int) -> int: + """Read the current unit count for an application, live.""" + try: + return len(list(jhelper.get_application(app, model).units)) + except (ApplicationNotFoundException, JujuException): + return fallback + + +# --------------------------------------------------------------------------- +# Target resolution +# --------------------------------------------------------------------------- +def _resolve_mysql_backup_target( + jhelper: JujuHelper, + app: str, + model: str, + force: bool, +) -> ActionTarget | None: """Resolve a MySQL backup target, preferring a secondary (replica) unit. Falls back to the leader when no secondary is available. When cluster status @@ -214,16 +303,15 @@ def _resolve_mysql_target( scale = len(units) try: - result = jhelper.run_action(leader, model, CLUSTER_STATUS_ACTION) + result = jhelper.run_action(leader, model, MYSQL_CLUSTER_STATUS_ACTION) secondary = _secondary_unit_from_status(units, result) if secondary is not None: - return BackupTarget( + return ActionTarget( app=app, unit=secondary, - component="mysql", + component=MYSQL_CHARM, action=BACKUP_ACTION, scale=scale, - is_replica=True, ) except ActionFailedException as e: if not force: @@ -235,19 +323,21 @@ def _resolve_mysql_target( e, ) - return BackupTarget( + return ActionTarget( app=app, unit=leader, - component="mysql", + component=MYSQL_CHARM, action=BACKUP_ACTION, scale=scale, - is_replica=False, ) -def _resolve_vault_target( - jhelper: JujuHelper, app: str, model: str, force: bool -) -> BackupTarget | None: +def _resolve_vault_backup_target( + jhelper: JujuHelper, + app: str, + model: str, + force: bool, +) -> ActionTarget | None: """Resolve the Vault backup target to the leader unit.""" try: leader = jhelper.get_leader_unit(app, model) @@ -256,293 +346,671 @@ def _resolve_vault_target( LOG.warning("Could not resolve %s, skipping", app) return None - return BackupTarget( + return ActionTarget( app=app, unit=leader, - component="vault", + component=VAULT_CHARM, action=BACKUP_ACTION, scale=len(units), - is_replica=False, ) -def _run_single_backup( - jhelper: JujuHelper, - target: BackupTarget, - backup_id_key: str, - model: str, - timeout: int, - action_params: dict | None = None, -) -> BackupResult: - """Dispatch a single backup action and capture the result. - - Any failure is encoded in the returned :class:`BackupResult`. - """ - try: - result = jhelper.run_action( - target.unit, model, target.action, action_params, timeout=timeout - ) - backup_id = result.get(backup_id_key) - return BackupResult( - app=target.app, - unit=target.unit, - component=target.component, - backup=BackupItem(backup_id=backup_id, success=True) - if backup_id is not None - else None, - ) - except Exception as e: - return BackupResult( - app=target.app, - unit=target.unit, - component=target.component, - error=str(e), - ) - - -def _run_single_list_backups( - jhelper: JujuHelper, - target: BackupTarget, - action_name: str, - parse_backups: Callable[[dict], list[BackupItem]] | None, - model: str, - timeout: int, -) -> BackupInventory: - """Dispatch a single list-backups action and parse backup IDs.""" - try: - result = jhelper.run_action(target.unit, model, action_name, timeout=timeout) - backups = parse_backups(result) if parse_backups is not None else [] - return BackupInventory( - app=target.app, - unit=target.unit, - component=target.component, - backups=backups, - ) - except Exception as e: - return BackupInventory( - app=target.app, - unit=target.unit, - component=target.component, - error=str(e), - ) - - -BACKUP_COMPONENTS: list[BackupComponent] = [ - BackupComponent( - name="mysql", - charm_names=[MYSQL_CHARM], - action=BACKUP_ACTION, - resolve_target=_resolve_mysql_target, - parse_backups=_parse_mysql_backups, - supports_force=True, - ), - BackupComponent( - name="vault", - charm_names=[VAULT_CHARM], - action=BACKUP_ACTION, - resolve_target=_resolve_vault_target, - parse_backups=_parse_vault_backups, - ), -] - - -class DiscoverBackupApplicationsStep(BaseStep): - """Discover applications of every registered backup component in the model.""" +# --------------------------------------------------------------------------- +# Atomic action steps +# --------------------------------------------------------------------------- +class _ApiAppActionStep(BaseStep): + """Dispatch an action on an application's leader.""" def __init__( self, jhelper: JujuHelper, + name: str, + description: str, + app: str, + action_name: str, model: str = OPENSTACK_MODEL, - components: list[BackupComponent] = BACKUP_COMPONENTS, ): - super().__init__( - "Discover backup applications", - "Discovering stateful applications to back up", - ) + super().__init__(name, description) self.jhelper = jhelper - self.components = components self.model = model + self.app = app + self.action_name = action_name def run(self, context: StepContext) -> Result: - """Return a mapping of component name to discovered application names.""" + """Run action on the application's leader.""" try: - status = self.jhelper.get_model_status(self.model) - except (ModelNotFoundException, JujuException) as e: + leader = self.jhelper.get_leader_unit(self.app, self.model) + self.jhelper.run_action( + leader, + self.model, + self.action_name, + timeout=DEFAULT_RESTORE_TIMEOUT, + ) + except ( + ActionFailedException, + LeaderNotFoundException, + ModelNotFoundException, + JujuException, + ) as e: return Result(ResultType.FAILED, str(e)) - discovered: dict[str, list[str]] = {c.name: [] for c in self.components} - non_active_targets: list[tuple[str, str]] = [] - for app_name, app_status in status.apps.items(): - charm_name = app_status.charm_name or "" - for component in self.components: - if charm_name in component.charm_names: - if not _is_app_active(app_status): - non_active_targets.append( - (app_name, _app_status_value(app_status)) - ) - break - discovered[component.name].append(app_name) + return Result(ResultType.COMPLETED) - if non_active_targets: - details = ", ".join( - f"{app}({state})" for app, state in sorted(non_active_targets) - ) - return Result( - ResultType.FAILED, - "Target applications must be active before backup/restore operations: " - + details, - ) - return Result(ResultType.COMPLETED, discovered) +class _PauseAppStep(_ApiAppActionStep): + """Pause application API services.""" + def __init__(self, jhelper: JujuHelper, app: str, model: str = OPENSTACK_MODEL): + super().__init__( + jhelper, + "Pause App", + "Pause API container services", + app, + PAUSE_ACTION, + model, + ) + + +class _ResumeAppStep(_ApiAppActionStep): + """Resume application API services.""" + + def __init__(self, jhelper: JujuHelper, app: str, model: str = OPENSTACK_MODEL): + super().__init__( + jhelper, + "Resume App", + "Resume API container services", + app, + RESUME_ACTION, + model, + ) -class ResolveBackupTargetsStep(BaseStep): - """Resolve the unit to back up for every discovered application.""" + +class _ScaleAppStep(BaseStep): + """Scale an application to a target number of units.""" def __init__( self, jhelper: JujuHelper, - discovered: dict[str, list[str]], + application: str, + scale: int, + timeout: int = DEFAULT_SCALE_TIMEOUT, model: str = OPENSTACK_MODEL, - force: bool = False, - ): - super().__init__( - "Resolve backup targets", - "Resolving units for backup targets", - ) + ) -> None: + super().__init__("Scale App", f"Scaling {application} to {scale} unit(s)") self.jhelper = jhelper - self.discovered = discovered - self.force = force + self.application = application + self.scale = scale + self.timeout = timeout self.model = model def run(self, context: StepContext) -> Result: - """Return a flat list of resolved backup targets across components.""" - targets: list[BackupTarget] = [] - for component_name, apps in self.discovered.items(): - component = _component_for(component_name) - if component is None: - continue - for app in apps: - target = component.resolve_target( - self.jhelper, app, self.model, self.force - ) - if target is None: - self.update_status(context, f"skipped {app}") - continue - targets.append(target) - - return Result(ResultType.COMPLETED, targets) + """Scale the application and wait for it to settle.""" + try: + self.jhelper.scale_application(self.model, self.application, self.scale) + self.jhelper.wait_until_active( + self.model, apps=[self.application], timeout=self.timeout + ) + except (JujuException, JujuWaitException, TimeoutError) as e: + return Result(ResultType.FAILED, str(e)) + return Result(ResultType.COMPLETED) -class CheckS3RelationsStep(BaseStep): - """Partition applications by required relation name and S3 interface.""" +class _RestoreAppStep(BaseStep): + """Restore a single application from a backup (atomic restore action).""" def __init__( self, jhelper: JujuHelper, - applications: list[str], - endpoint_name: str, + component: "BackupComponent", + target: ActionTarget, + restore_to_time: str | None = None, + timeout: int = DEFAULT_RESTORE_TIMEOUT, model: str = OPENSTACK_MODEL, ): - super().__init__( - "Check S3 relations", - "Checking applications related to S3", - ) + super().__init__("Restore app", f"Restoring {target.app}") self.jhelper = jhelper - self.applications = applications - self.endpoint_name = endpoint_name + self.target = target + self.restore_to_time = restore_to_time + self.component = component self.model = model + self.timeout = timeout def run(self, context: StepContext) -> Result: - """Return related/unrelated applications by relation name + interface.""" + """Restore an app using latest backup or restore-to-time.""" try: - status = self.jhelper.get_model_status(self.model) - except (ModelNotFoundException, JujuException) as e: + leader = self.jhelper.get_leader_unit(self.target.app, self.model) + except (LeaderNotFoundException, JujuException) as e: return Result(ResultType.FAILED, str(e)) - related: list[str] = [] - unrelated: list[str] = [] - for app_name in self.applications: - app_status = status.apps.get(app_name) - if app_status is None: - unrelated.append(app_name) - continue - relations = app_status.relations or {} - endpoint_relations = relations.get(self.endpoint_name, []) - if any(rel.interface == S3_INTERFACE for rel in endpoint_relations): - related.append(app_name) - else: - unrelated.append(app_name) - - return Result( - ResultType.COMPLETED, - { - "related": sorted(related), - "unrelated": sorted(unrelated), - }, - ) + params: dict[str, str] = {} + if self.restore_to_time is not None and self.component.supports_restore_to_time: + params["restore-to-time"] = self.restore_to_time + else: + try: + list_result = self.jhelper.run_action( + leader, + self.model, + self.component.list_action, + timeout=self.timeout, + ) + except (ActionFailedException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + latest = _latest_backup(self.component.parse_backup_list(list_result)) + if latest is None: + return Result( + ResultType.FAILED, + f"No finished backups found for {self.target.app}.", + ) + params[self.component.backup_id_key] = latest -class RunBackupsStep(BaseStep): - """Dispatch every target's backup action concurrently and collect results.""" + try: + self.jhelper.run_action( + leader, + self.model, + self.component.restore_action, + params, + timeout=self.timeout, + ) + except (ActionFailedException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED) + + +class _BackupAppStep(BaseStep): + """Dispatch a single application's backup action and capture the outcome.""" def __init__( self, jhelper: JujuHelper, - targets: list[BackupTarget], + component: "BackupComponent", + target: ActionTarget, force: bool = False, timeout: int = DEFAULT_BACKUP_TIMEOUT, model: str = OPENSTACK_MODEL, ): - super().__init__("Run backups", "Dispatching backups across applications") + super().__init__("Backup app", f"Backing up {target.app}") self.jhelper = jhelper - self.targets = targets + self.component = component + self.target = target self.force = force self.timeout = timeout self.model = model + self.result: BackupResult | None = None - def _action_params(self, component: BackupComponent | None) -> dict | None: - if self.force and component is not None and component.supports_force: + def _action_params(self) -> dict | None: + if self.force and self.component.supports_force: return {"force": True} return None def run(self, context: StepContext) -> Result: - """Dispatch backups concurrently, returning a list of results.""" - components = { - target.component: _component_for(target.component) - for target in self.targets - } - results: list[BackupResult] = [] - with ThreadPoolExecutor(max_workers=len(self.targets)) as executor: - futures = [] - for target in self.targets: - component = components[target.component] - backup_id_key = component.backup_id_key if component else "backup-id" - futures.append( - executor.submit( - _run_single_backup, - self.jhelper, - target, - backup_id_key, - self.model, - self.timeout, - self._action_params(component), - ) - ) - for future in as_completed(futures): - results.append(future.result()) + """Dispatch the backup action, recording the outcome on ``self.result``.""" + target = self.target + try: + action_result = self.jhelper.run_action( + target.unit, + self.model, + target.action, + self._action_params(), + timeout=self.timeout, + ) + backup_id = action_result.get(self.component.backup_id_key) + self.result = BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + backup=BackupOutcome(backup_id=backup_id, success=True) + if backup_id is not None + else None, + ) + except Exception as e: + message = str(e) + in_progress = "timed out waiting for results" in message.lower() + self.result = BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + backup=BackupOutcome(backup_id="", success=None) + if in_progress + else None, + error=message, + ) - return Result(ResultType.COMPLETED, results) + return Result(ResultType.COMPLETED, self.result) -class ListBackupsStep(BaseStep): - """Dispatch list-backups actions concurrently and collect inventories.""" +class _CheckPauseResumeSupportStep(BaseStep): + """Validate pause/resume action support for a single application.""" def __init__( self, jhelper: JujuHelper, - targets: list[BackupTarget], - timeout: int = DEFAULT_BACKUP_TIMEOUT, + app: str, + model: str = OPENSTACK_MODEL, + ): + super().__init__( + "Check pause/resume support", + "Validating pause/resume support before restore", + ) + self.jhelper = jhelper + self.app = app + self.model = model + + def run(self, context: StepContext) -> Result: + """Fail fast if the application does not support pause/resume actions.""" + try: + actions = self.jhelper.get_application_actions(self.app, self.model) + except (JujuException, ModelNotFoundException): + return Result( + ResultType.FAILED, + f"Unable to query actions for {self.app}. No changes have been made.", + ) + + if PAUSE_ACTION not in actions or RESUME_ACTION not in actions: + return Result( + ResultType.FAILED, + f"Application {self.app} does not support the 'pause/resume' action " + "required for restore. No changes have been made.", + ) + + return Result(ResultType.COMPLETED) + + +# --------------------------------------------------------------------------- +# Per-component plan builders +# --------------------------------------------------------------------------- +def _build_mysql_backup_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + force: bool, + timeout: int, + model: str, +) -> list[BaseStep]: + """Build the MySQL backup plan: a single create-backup action.""" + return [ + _BackupAppStep( + jhelper, component, target, force=force, timeout=timeout, model=model + ), + ] + + +def _build_vault_backup_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + force: bool, + timeout: int, + model: str, +) -> list[BaseStep]: + """Build the Vault backup plan: a single create-backup action.""" + return [ + _BackupAppStep( + jhelper, component, target, force=force, timeout=timeout, model=model + ), + ] + + +def _build_mysql_restore_precheck_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + model: str, +) -> list[BaseStep]: + """Precheck that the MySQL app's API app supports pause/resume.""" + api_app = _api_app_for_mysql(target.app) + return [_CheckPauseResumeSupportStep(jhelper, api_app, model=model)] + + +def _build_mysql_restore_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + restore_to_time: str | None, + timeout: int, + model: str, +) -> list[BaseStep]: + """Build the MySQL restore plan: pause, scale down, restore, scale up, resume. + + The scale to restore back to is read live at plan-build time so the value + is accurate before the application is scaled down. + """ + api_app = _api_app_for_mysql(target.app) + original_scale = _current_scale(jhelper, target.app, model, target.scale) + + return [ + _PauseAppStep(jhelper, api_app, model=model), + _ScaleAppStep( + jhelper, target.app, 1, timeout=DEFAULT_SCALE_TIMEOUT, model=model + ), + _RestoreAppStep( + jhelper, + component, + target, + restore_to_time=restore_to_time, + timeout=timeout, + model=model, + ), + _ScaleAppStep( + jhelper, + target.app, + original_scale, + timeout=DEFAULT_SCALE_TIMEOUT, + model=model, + ), + _ResumeAppStep(jhelper, api_app, model=model), + ] + + +def _build_mysql_restore_revert_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + model: str, +) -> list[BaseStep]: + """Build the revert plan for a failed MySQL restore: scale back up, resume.""" + api_app = _api_app_for_mysql(target.app) + original_scale = _current_scale(jhelper, target.app, model, target.scale) + return [ + _ScaleAppStep( + jhelper, + target.app, + original_scale, + timeout=DEFAULT_SCALE_TIMEOUT, + model=model, + ), + _ResumeAppStep(jhelper, api_app, model=model), + ] + + +def _build_vault_restore_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + restore_to_time: str | None, + timeout: int, + model: str, +) -> list[BaseStep]: + """Build the Vault restore plan: a single restore step (no pause/scale).""" + return [ + _RestoreAppStep( + jhelper, + component, + target, + restore_to_time=restore_to_time, + timeout=timeout, + model=model, + ), + ] + + +# --------------------------------------------------------------------------- +# Public component registry +# --------------------------------------------------------------------------- +BACKUP_COMPONENTS: list[BackupComponent] = [ + BackupComponent( + name=MYSQL_CHARM, + resolve_backup_target=_resolve_mysql_backup_target, + parse_backup_list=_parse_mysql_backups, + build_backup_plan=_build_mysql_backup_plan, + build_restore_plan=_build_mysql_restore_plan, + build_restore_revert_plan=_build_mysql_restore_revert_plan, + build_restore_precheck_plan=_build_mysql_restore_precheck_plan, + restore_action=RESTORE_ACTION, + supports_force=True, + validate_checks=[ + ValidationCheck.ACTIVE, + ValidationCheck.S3_RELATION, + ], + ), + BackupComponent( + name=VAULT_CHARM, + resolve_backup_target=_resolve_vault_backup_target, + parse_backup_list=_parse_vault_backups, + build_backup_plan=_build_vault_backup_plan, + build_restore_plan=_build_vault_restore_plan, + restore_action=VAULT_RESTORE_ACTION, + validate_checks=[ + ValidationCheck.ACTIVE, + ValidationCheck.S3_RELATION, + ], + ), +] + + +# --------------------------------------------------------------------------- +# Public top-level wrapper steps +# --------------------------------------------------------------------------- +class DiscoverBackupApplicationsStep(BaseStep): + """Discover applications of every registered backup component in the model.""" + + def __init__( + self, + jhelper: JujuHelper, + model: str = OPENSTACK_MODEL, + components: list[BackupComponent] = BACKUP_COMPONENTS, + ): + super().__init__( + "Discover backup applications", + "Discovering stateful applications to back up", + ) + self.jhelper = jhelper + self.components = components + self.model = model + + def run(self, context: StepContext) -> Result: + """Return a mapping of component name to discovered application names.""" + try: + status = self.jhelper.get_model_status(self.model) + except (ModelNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + discovered: dict[str, list[str]] = {c.name: [] for c in self.components} + for app_name, app_status in status.apps.items(): + charm_name = app_status.charm_name or "" + for component in self.components: + if charm_name == component.name: + discovered[component.name].append(app_name) + + return Result(ResultType.COMPLETED, discovered) + + +class ValidateStep(BaseStep): + """Apply each component's own validation checks to its discovered apps. + + Returns a mapping keyed by component with, per application, the list of + failed check names. Applications with an empty failure list are valid. + """ + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + model: str = OPENSTACK_MODEL, + components: list[BackupComponent] = BACKUP_COMPONENTS, + ): + super().__init__("Validate applications", "Validating backup readiness") + self.jhelper = jhelper + self.discovered = discovered + self.model = model + self.components = components + + def run(self, context: StepContext) -> Result: + """Return {'valid': {...}, 'failures': {app: [check names]}}.""" + try: + status = self.jhelper.get_model_status(self.model) + except (ModelNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + valid: dict[str, list[str]] = {} + failures: dict[str, list[str]] = {} + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + valid[component_name] = [] + if component is None: + continue + for app_name in apps: + app_status = status.apps.get(app_name) + failed = self._failed_checks(component, app_status) + if failed: + failures[app_name] = failed + else: + valid[component_name].append(app_name) + + return Result(ResultType.COMPLETED, {"valid": valid, "failures": failures}) + + @staticmethod + def _failed_checks( + component: BackupComponent, app_status: object | None + ) -> list[str]: + if app_status is None: + return [check.value for check in component.validate_checks] or ["present"] + return [ + check.value + for check in component.validate_checks + if not _VALIDATION_PREDICATES[check](app_status) + ] + + +class ResolveActionTargetsStep(BaseStep): + """Resolve every discovered application's leader unit for generic actions.""" + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + model: str = OPENSTACK_MODEL, + ): + super().__init__( + "Resolve action targets", + "Resolving units for action targets", + ) + self.jhelper = jhelper + self.discovered = discovered + self.model = model + + def run(self, context: StepContext) -> Result: + """Return a flat list of resolved leader targets across components.""" + targets: list[ActionTarget] = [] + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + if component is None: + continue + for app in apps: + try: + leader = self.jhelper.get_leader_unit(app, self.model) + except (LeaderNotFoundException, ApplicationNotFoundException): + self.update_status(context, f"skipped {app}") + continue + targets.append( + ActionTarget( + app=app, + unit=leader, + component=component.name, + action=component.list_action, + ) + ) + + return Result(ResultType.COMPLETED, targets) + + +class RunBackupsStep(BaseStep): + """Resolve targets per component, run each backup plan, and collect results.""" + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + force: bool = False, + timeout: int = DEFAULT_BACKUP_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Run backups", "Dispatching backups across applications") + self.jhelper = jhelper + self.discovered = discovered + self.force = force + self.timeout = timeout + self.model = model + + def _resolve_targets( + self, context: StepContext + ) -> list[tuple[BackupComponent, ActionTarget]]: + """Resolve one backup target per discovered application.""" + resolved: list[tuple[BackupComponent, ActionTarget]] = [] + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + if component is None: + continue + for app in apps: + target = component.resolve_backup_target( + self.jhelper, app, self.model, self.force + ) + if target is None: + self.update_status(context, f"skipped {app}") + continue + resolved.append((component, target)) + return resolved + + def _run_backup_plan( + self, + component: BackupComponent, + target: ActionTarget, + context: StepContext, + ) -> BackupResult: + """Run a component's backup plan and return its BackupResult. + + Any step failure short of the result-bearing BackupAppStep is encoded + as a failed BackupResult for the target. + """ + plan = component.build_backup_plan( + self.jhelper, component, target, self.force, self.timeout, self.model + ) + result: BackupResult | None = None + for step in plan: + step_result = step.run(context) + if isinstance(step, _BackupAppStep) and step.result is not None: + result = step.result + if step_result.result_type == ResultType.FAILED: + return result or BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + error=str(step_result.message), + ) + return result or BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + error="Backup plan produced no result.", + ) + + def run(self, context: StepContext) -> Result: + """Resolve targets, run backup plans concurrently, return results.""" + resolved = self._resolve_targets(context) + if not resolved: + return Result(ResultType.COMPLETED, []) + + results: list[BackupResult] = [] + with ThreadPoolExecutor(max_workers=len(resolved)) as executor: + futures = [ + executor.submit(self._run_backup_plan, component, target, context) + for component, target in resolved + ] + for future in as_completed(futures): + results.append(future.result()) + + return Result(ResultType.COMPLETED, results) + + +class ListBackupsStep(BaseStep): + """Dispatch list-backups actions concurrently and collect inventories.""" + + def __init__( + self, + jhelper: JujuHelper, + targets: list[ActionTarget], + timeout: int = DEFAULT_BACKUP_TIMEOUT, model: str = OPENSTACK_MODEL, ): super().__init__("List backups", "Listing backups across applications") @@ -551,35 +1019,50 @@ def __init__( self.timeout = timeout self.model = model + def _list_one( + self, + target: ActionTarget, + action_name: str, + parse_backups: Callable[[dict], list[BackupOutcome]] | None, + ) -> BackupInventory: + """Dispatch a single list-backups action and parse backup IDs.""" + try: + result = self.jhelper.run_action( + target.unit, self.model, action_name, timeout=self.timeout + ) + backups = parse_backups(result) if parse_backups is not None else [] + return BackupInventory( + app=target.app, + unit=target.unit, + component=target.component, + backups=backups, + ) + except Exception as e: + return BackupInventory( + app=target.app, + unit=target.unit, + component=target.component, + error=str(e), + ) + def run(self, context: StepContext) -> Result: """Dispatch list-backups concurrently, returning inventory entries.""" if not self.targets: return Result(ResultType.COMPLETED, []) - components = { - target.component: _component_for(target.component) - for target in self.targets - } inventories: list[BackupInventory] = [] with ThreadPoolExecutor(max_workers=len(self.targets)) as executor: futures = [] for target in self.targets: - component = components[target.component] - action_name = ( - component.list_action - if component is not None - else LIST_BACKUPS_ACTION - ) - parser = component.parse_backups if component is not None else None + component = _component_for(target.component) + if component is None: + continue futures.append( executor.submit( - _run_single_list_backups, - self.jhelper, + self._list_one, target, - action_name, - parser, - self.model, - self.timeout, + component.list_action, + component.parse_backup_list, ) ) for future in as_completed(futures): @@ -588,6 +1071,99 @@ def run(self, context: StepContext) -> Result: return Result(ResultType.COMPLETED, inventories) +class RestoreStep(BaseStep): + """Drive every component's restore plan, reverting on failure.""" + + def __init__( + self, + jhelper: JujuHelper, + targets: list[ActionTarget], + restore_to_time: str | None = None, + timeout: int = DEFAULT_RESTORE_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Restore applications", "Restoring applications from backup") + self.jhelper = jhelper + self.targets = targets + self.restore_to_time = restore_to_time + self.timeout = timeout + self.model = model + + def _run_plan(self, plan: list[BaseStep], context: StepContext) -> None: + """Run a plan of steps, raising RuntimeError on the first failure.""" + for step in plan: + result = step.run(context) + if result.result_type == ResultType.FAILED: + raise RuntimeError(result.message) + + def _restore_one( + self, component: BackupComponent, target: ActionTarget, context: StepContext + ) -> RestoreResult: + plan = component.build_restore_plan( + self.jhelper, + component, + target, + self.restore_to_time, + self.timeout, + self.model, + ) + try: + self._run_plan(plan, context) + return RestoreResult(app=target.app, component=component.name, success=True) + except Exception as e: + revert_plan: list[BaseStep] = [] + if component.build_restore_revert_plan is not None: + revert_plan = component.build_restore_revert_plan( + self.jhelper, component, target, self.model + ) + reverted = False + if revert_plan: + try: + self._run_plan(revert_plan, context) + reverted = True + except Exception as revert_error: + LOG.warning("Revert failed for %s: %s", target.app, revert_error) + return RestoreResult( + app=target.app, + component=component.name, + success=False, + error=str(e), + reverted=reverted, + ) + + def run(self, context: StepContext) -> Result: + """Precheck all targets, then restore each sequentially, aggregating.""" + planned: list[tuple[BackupComponent, ActionTarget]] = [] + for target in self.targets: + component = _component_for(target.component) + if component is None: + return Result( + ResultType.FAILED, + f"No restore plan defined for {target.app}.", + ) + planned.append((component, target)) + + # Global fail-fast precheck: run every component's precheck before any + # restore work so an unsupported app never leaves others half-restored. + for component, target in planned: + if component.build_restore_precheck_plan is None: + continue + precheck = component.build_restore_precheck_plan( + self.jhelper, component, target, self.model + ) + try: + self._run_plan(precheck, context) + except Exception as e: + return Result(ResultType.FAILED, str(e)) + + results: list[RestoreResult] = [] + for component, target in planned: + self.update_status(context, f"restoring {target.app}") + results.append(self._restore_one(component, target, context)) + + return Result(ResultType.COMPLETED, results) + + class WriteBackupManifestStep(BaseStep): """Write a timestamped manifest of the backup run to the snap share path.""" @@ -672,258 +1248,3 @@ def run(self, context: StepContext) -> Result: return Result(ResultType.FAILED, str(e)) return Result(ResultType.COMPLETED, str(path)) - - -class _ApiAppActionStep(BaseStep): - """Dispatch an action on an application's leader.""" - - def __init__( - self, - jhelper: JujuHelper, - name: str, - description: str, - app: str, - action_name: str, - model: str = OPENSTACK_MODEL, - ): - super().__init__(name, description) - self.jhelper = jhelper - self.model = model - self.app = app - self.action_name = action_name - - def run(self, context: StepContext) -> Result: - """Run action on the application's leader.""" - try: - leader = self.jhelper.get_leader_unit(self.app, self.model) - self.jhelper.run_action( - leader, - self.model, - self.action_name, - timeout=DEFAULT_RESTORE_TIMEOUT, - ) - except ( - ActionFailedException, - LeaderNotFoundException, - ModelNotFoundException, - JujuException, - ) as e: - return Result(ResultType.FAILED, str(e)) - - return Result(ResultType.COMPLETED) - - -class PauseAppStep(_ApiAppActionStep): - """Pause application API services.""" - - def __init__(self, jhelper: JujuHelper, app: str, model: str = OPENSTACK_MODEL): - super().__init__( - jhelper, - "Pause App", - "Pause API container services", - app, - PAUSE_ACTION, - model, - ) - - -class ResumeAppStep(_ApiAppActionStep): - """Resume application API services.""" - - def __init__(self, jhelper: JujuHelper, app: str, model: str = OPENSTACK_MODEL): - super().__init__( - jhelper, - "Resume App", - "Resume API container services", - app, - RESUME_ACTION, - model, - ) - - -class ScaleAppStep(BaseStep): - """Scale an application to a target number of units.""" - - def __init__( - self, - jhelper: JujuHelper, - application: str, - scale: int, - timeout: int = DEFAULT_SCALE_TIMEOUT, - model: str = OPENSTACK_MODEL, - ) -> None: - super().__init__("Scale App", f"Scaling {application} to {scale} unit(s)") - self.jhelper = jhelper - self.application = application - self.scale = scale - self.timeout = timeout - self.model = model - - def run(self, context: StepContext) -> Result: - """Scale the application and wait for it to settle.""" - try: - self.jhelper.scale_application(self.model, self.application, self.scale) - self.jhelper.wait_until_active( - self.model, apps=[self.application], timeout=self.timeout - ) - except (JujuException, JujuWaitException, TimeoutError) as e: - return Result(ResultType.FAILED, str(e)) - return Result(ResultType.COMPLETED) - - -class CheckAppPauseResumeSupportStep(BaseStep): - """Validate pause/resume action support for all target applications.""" - - def __init__( - self, - jhelper: JujuHelper, - apps: list[str], - model: str = OPENSTACK_MODEL, - ): - super().__init__( - "Check pause/resume support", - "Validating pause/resume support before restore", - ) - self.jhelper = jhelper - self.apps = apps - self.model = model - - def run(self, context: StepContext) -> Result: - """Fail fast if any app does not support pause/resume actions.""" - unsupported_apps: list[str] = [] - for app in self.apps: - try: - actions = self.jhelper.get_application_actions(app, self.model) - except (JujuException, ModelNotFoundException): - return Result( - ResultType.FAILED, - f"Unable to query actions for {app}. No changes have been made.", - ) - - if PAUSE_ACTION not in actions or RESUME_ACTION not in actions: - unsupported_apps.append(app) - - if unsupported_apps: - details = ", ".join(sorted(unsupported_apps)) - return Result( - ResultType.FAILED, - "The following applications do not support the 'pause/resume' " - f"action required for restore: {details}. No changes have been made.", - ) - - return Result(ResultType.COMPLETED) - - -class RestoreMySQLStep(BaseStep): - """Restore a MySQL application from a backup.""" - - def __init__( - self, - jhelper: JujuHelper, - target: BackupTarget, - restore_to_time: str | None = None, - timeout: int = DEFAULT_RESTORE_TIMEOUT, - model: str = OPENSTACK_MODEL, - ): - super().__init__("Restore MySQL", f"Restoring {target.app}") - self.jhelper = jhelper - self.target = target - self.restore_to_time = restore_to_time - self.model = model - self.timeout = timeout - - def run(self, context: StepContext) -> Result: - """Restore MySQL using latest backup or restore-to-time.""" - try: - leader = self.jhelper.get_leader_unit(self.target.app, self.model) - except (LeaderNotFoundException, JujuException) as e: - return Result(ResultType.FAILED, str(e)) - - params: dict[str, str] = {} - if self.restore_to_time is not None: - params["restore-to-time"] = self.restore_to_time - else: - try: - list_result = self.jhelper.run_action( - leader, - self.model, - LIST_BACKUPS_ACTION, - timeout=self.timeout, - ) - except (ActionFailedException, JujuException) as e: - return Result(ResultType.FAILED, str(e)) - - latest = _latest_backup(_parse_mysql_backups(list_result)) - if latest is None: - return Result( - ResultType.FAILED, - f"No finished MySQL backups found for {self.target.app}.", - ) - params["backup-id"] = latest - - try: - self.jhelper.run_action( - leader, - self.model, - MYSQL_RESTORE_ACTION, - params, - timeout=self.timeout, - ) - except (ActionFailedException, JujuException) as e: - return Result(ResultType.FAILED, str(e)) - - return Result(ResultType.COMPLETED) - - -class RestoreVaultStep(BaseStep): - """Restore a Vault application from a backup.""" - - def __init__( - self, - jhelper: JujuHelper, - target: BackupTarget, - timeout: int = DEFAULT_RESTORE_TIMEOUT, - model: str = OPENSTACK_MODEL, - ): - super().__init__("Restore Vault", f"Restoring {target.app}") - self.jhelper = jhelper - self.target = target - self.model = model - self.timeout = timeout - - def run(self, context: StepContext) -> Result: - """Restore Vault from the latest available backup.""" - try: - leader = self.jhelper.get_leader_unit(self.target.app, self.model) - except (LeaderNotFoundException, JujuException) as e: - return Result(ResultType.FAILED, str(e)) - - try: - list_result = self.jhelper.run_action( - leader, - self.model, - LIST_BACKUPS_ACTION, - timeout=self.timeout, - ) - except (ActionFailedException, JujuException) as e: - return Result(ResultType.FAILED, str(e)) - - latest = _latest_backup(_parse_vault_backups(list_result)) - if latest is None: - return Result( - ResultType.FAILED, - f"No Vault backups found for {self.target.app}.", - ) - - try: - self.jhelper.run_action( - leader, - self.model, - VAULT_RESTORE_ACTION, - {"backup-id": latest}, - timeout=self.timeout, - ) - except (ActionFailedException, JujuException) as e: - return Result(ResultType.FAILED, str(e)) - - return Result(ResultType.COMPLETED) diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py index d08fcd30f..56e596ed6 100644 --- a/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py @@ -49,6 +49,13 @@ def _leader_only_cluster_status(unit, model, action, params=None, timeout=None): return {"backup-id": f"backup-{unit.replace('/', '-')}"} +def _s3_related(app): + relation = Mock() + relation.interface = "s3" + app.relations = {"s3-parameters": [relation]} + return app + + class TestBackupCommand: def test_no_applications(self, deployment, jhelper): jhelper.get_model_status.return_value = _model_status({}) @@ -59,19 +66,10 @@ def test_no_applications(self, deployment, jhelper): assert "No applications found to back up. Exiting." in result.output def test_all_success(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") - mysql_s3_relation = Mock() - mysql_s3_relation.interface = "s3" - mysql.relations = {"s3-parameters": [mysql_s3_relation]} - vault = _app_status("vault-k8s") - vault_s3_relation = Mock() - vault_s3_relation.interface = "s3" - vault.relations = {"s3-parameters": [vault_s3_relation]} + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) jhelper.get_model_status.return_value = _model_status( - { - "keystone-mysql": mysql, - "vault": vault, - } + {"keystone-mysql": mysql, "vault": vault} ) jhelper.get_application.return_value = _app_status("mysql-k8s") jhelper.run_action.side_effect = _leader_only_cluster_status @@ -82,19 +80,10 @@ def test_all_success(self, deployment, jhelper): assert "2 succeeded, 0 failed" in result.output def test_partial_failure(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") - mysql_s3_relation = Mock() - mysql_s3_relation.interface = "s3" - mysql.relations = {"s3-parameters": [mysql_s3_relation]} - vault = _app_status("vault-k8s") - vault_s3_relation = Mock() - vault_s3_relation.interface = "s3" - vault.relations = {"s3-parameters": [vault_s3_relation]} + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) jhelper.get_model_status.return_value = _model_status( - { - "keystone-mysql": mysql, - "vault": vault, - } + {"keystone-mysql": mysql, "vault": vault} ) jhelper.get_application.return_value = _app_status("mysql-k8s") @@ -120,10 +109,7 @@ def _run_action(unit, model, action, params=None, timeout=None): assert "Warning" in result.output def test_all_failed(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") - mysql_s3_relation = Mock() - mysql_s3_relation.interface = "s3" - mysql.relations = {"s3-parameters": [mysql_s3_relation]} + mysql = _s3_related(_app_status("mysql-k8s")) jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) jhelper.get_application.return_value = _app_status("mysql-k8s") @@ -146,17 +132,10 @@ def _run_action(unit, model, action, params=None, timeout=None): assert "0 succeeded, 1 failed" in result.output def test_unrelated_mysql_is_skipped_and_backup_continues(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") - mysql.relations = {} - vault = _app_status("vault-k8s") - vault_s3_relation = Mock() - vault_s3_relation.interface = "s3" - vault.relations = {"s3-parameters": [vault_s3_relation]} + mysql = _app_status("mysql-k8s") # no s3 + vault = _s3_related(_app_status("vault-k8s")) jhelper.get_model_status.return_value = _model_status( - { - "keystone-mysql": mysql, - "vault": vault, - } + {"keystone-mysql": mysql, "vault": vault} ) jhelper.get_application.return_value = vault jhelper.run_action.return_value = {"backup-id": "backup-vault-0"} @@ -164,73 +143,54 @@ def test_unrelated_mysql_is_skipped_and_backup_continues(self, deployment, jhelp result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) assert result.exit_code == 0, result.output - assert "will be skipped" in result.output - called_units = [call.args[0] for call in jhelper.run_action.call_args_list] - assert called_units == ["vault/0"] - - def test_no_supported_apps_left(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") - mysql.relations = {} - jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) - - result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) - - assert result.exit_code == 2, result.output - assert "the following MySQL applications are not related" in result.output - assert "Could not resolve a backup target for any application" in result.output + assert "is not ready for backup" in result.output + assert "keystone-mysql" in result.output - def test_unrelated_vault_is_skipped_and_mysql_backup_continues( - self, deployment, jhelper + def test_declining_confirmation_aborts_backup( + self, deployment, jhelper, monkeypatch ): - mysql = _app_status("mysql-k8s") - mysql_s3_relation = Mock() - mysql_s3_relation.interface = "s3" - mysql.relations = {"s3-parameters": [mysql_s3_relation]} - vault = _app_status("vault-k8s") - vault.relations = {} + mysql = _app_status("mysql-k8s") # no s3, will be skipped -> prompt + vault = _s3_related(_app_status("vault-k8s")) jhelper.get_model_status.return_value = _model_status( - { - "keystone-mysql": mysql, - "vault": vault, - } + {"keystone-mysql": mysql, "vault": vault} ) - jhelper.get_application.return_value = mysql - - def _run_action(unit, model, action, params=None, timeout=None): - if action == "get-cluster-status": - return { - "status": { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } - } - } - return {"backup-id": "backup-keystone-mysql-0"} + jhelper.get_application.return_value = vault - jhelper.run_action.side_effect = _run_action + monkeypatch.setattr( + "sunbeam.commands.backup_restore.ConfirmQuestion.ask", + lambda self, *a, **k: False, + ) - result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + result = CliRunner().invoke(backup, obj=deployment) - assert result.exit_code == 0, result.output - assert "the following Vault applications are not related" in result.output + assert result.exit_code != 0 + assert "Aborted" in result.output + # No backup action dispatched. + actions = [ + call.args[2] + for call in jhelper.run_action.call_args_list + if len(call.args) > 2 + ] + assert "create-backup" not in actions - def test_no_supported_apps_left_when_only_vault(self, deployment, jhelper): - vault = _app_status("vault-k8s") - vault.relations = {} - jhelper.get_model_status.return_value = _model_status({"vault": vault}) + def test_no_supported_apps_left(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") # no s3 + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) assert result.exit_code == 2, result.output - assert "Could not resolve a backup target for any application" in result.output + assert "keystone-mysql" in result.output + assert "No applications remain to back up after validation." in result.output - def test_non_active_target_app_fails(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") + def test_non_active_target_app_is_skipped(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) mysql.app_status.current = "blocked" jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) - assert result.exit_code == 1, result.output + assert result.exit_code == 2, result.output assert "keystone-mysql" in result.output - assert "blocked" in result.output + assert "active" in result.output + assert "No applications remain to back up after validation." in result.output diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py index 90486cf9a..dd8f49d14 100644 --- a/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py @@ -25,6 +25,13 @@ def _model_status(apps): return status +def _s3_related(app): + relation = Mock() + relation.interface = "s3" + app.relations = {"s3-parameters": [relation]} + return app + + @pytest.fixture def deployment(): return Mock() @@ -48,44 +55,23 @@ def test_no_applications(self, deployment, jhelper): assert "No applications found" in result.output def test_no_supported_apps_left(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") - mysql.relations = {} + mysql = _app_status("mysql-k8s") # no s3 jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) result = CliRunner().invoke(list_backups, obj=deployment) assert result.exit_code == 2, result.output - assert "the following MySQL applications are not related" in result.output + assert "keystone-mysql" in result.output assert "No applications found to list backups from. Exiting." in result.output def test_lists_backups_and_writes_manifest(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") - mysql_s3_relation = Mock() - mysql_s3_relation.interface = "s3" - mysql.relations = {"s3-parameters": [mysql_s3_relation]} - vault = _app_status("vault-k8s") - vault_s3_relation = Mock() - vault_s3_relation.interface = "s3" - vault.relations = {"s3-parameters": [vault_s3_relation]} + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) jhelper.get_model_status.return_value = _model_status( - { - "keystone-mysql": mysql, - "vault": vault, - } - ) - jhelper.get_application.side_effect = lambda app, model: ( - mysql if app == "keystone-mysql" else vault + {"keystone-mysql": mysql, "vault": vault} ) def _run_action(unit, model, action, params=None, timeout=None): - if action == "get-cluster-status": - return { - "status": { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } - } - } if unit == "keystone-mysql/0": return { "backups": ( @@ -107,45 +93,49 @@ def _run_action(unit, model, action, params=None, timeout=None): assert "vault" in result.output assert "Backup inventory manifest written to" in result.output - def test_non_active_target_app_fails(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") + def test_lists_from_leader_only(self, deployment, jhelper): + """list-backups resolves leaders; no cluster-status is queried.""" + mysql = _s3_related(_app_status("mysql-k8s")) + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + def _run_action(unit, model, action, params=None, timeout=None): + assert action != "get-cluster-status" + assert unit == "keystone-mysql/0" + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 0, result.output + actions = [call.args[2] for call in jhelper.run_action.call_args_list] + assert "get-cluster-status" not in actions + + def test_non_active_target_app_is_skipped(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) mysql.app_status.current = "waiting" jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) result = CliRunner().invoke(list_backups, obj=deployment) - assert result.exit_code == 1, result.output + assert result.exit_code == 2, result.output assert "keystone-mysql" in result.output - assert "waiting" in result.output + assert "waiting" in result.output or "active" in result.output def test_list_action_failures_exit_2_with_details(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") - mysql_s3_relation = Mock() - mysql_s3_relation.interface = "s3" - mysql.relations = {"s3-parameters": [mysql_s3_relation]} - vault = _app_status("vault-k8s") - vault_s3_relation = Mock() - vault_s3_relation.interface = "s3" - vault.relations = {"s3-parameters": [vault_s3_relation]} + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) jhelper.get_model_status.return_value = _model_status( - { - "keystone-mysql": mysql, - "vault": vault, - } - ) - jhelper.get_application.side_effect = lambda app, model: ( - mysql if app == "keystone-mysql" else vault + {"keystone-mysql": mysql, "vault": vault} ) def _run_action(unit, model, action, params=None, timeout=None): - if action == "get-cluster-status": - return { - "status": { - "defaultreplicaset": { - "topology": {"mysql-0": {"memberrole": "PRIMARY"}} - } - } - } if action == "list-backups": raise Exception("list failed") return {} diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py index 51e43de83..40b26fa3a 100644 --- a/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py @@ -27,7 +27,14 @@ def _model_status(apps): return status -def _default_list_backups(unit, model, action, params=None, timeout=None): +def _s3_related(app): + relation = Mock() + relation.interface = "s3" + app.relations = {"s3-parameters": [relation]} + return app + + +def _default_run_action(unit, model, action, params=None, timeout=None): if action == "get-cluster-status": return { "status": { @@ -61,14 +68,8 @@ def jhelper(deployment): jhelper = Mock() deployment.get_juju_helper.return_value = jhelper jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" - mysql = _app_status("mysql-k8s") - mysql_s3_relation = Mock() - mysql_s3_relation.interface = "s3" - mysql.relations = {"s3-parameters": [mysql_s3_relation]} - vault = _app_status("vault-k8s") - vault_s3_relation = Mock() - vault_s3_relation.interface = "s3" - vault.relations = {"s3-parameters": [vault_s3_relation]} + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) jhelper.get_model_status.return_value = _model_status( { "keystone-mysql": mysql, @@ -78,7 +79,7 @@ def jhelper(deployment): ) jhelper.get_application.return_value = _app_status("mysql-k8s") jhelper.get_application_actions.return_value = ["pause", "resume"] - jhelper.run_action.side_effect = _default_list_backups + jhelper.run_action.side_effect = _default_run_action return jhelper @@ -94,21 +95,10 @@ def test_stops_at_pause_guard_and_is_non_destructive(self, deployment, jhelper): def test_prechecks_pause_resume_for_all_apps_before_any_restore_work( self, deployment, jhelper ): - mysql_a = _app_status("mysql-k8s") - mysql_a_s3 = Mock() - mysql_a_s3.interface = "s3" - mysql_a.relations = {"s3-parameters": [mysql_a_s3]} - - mysql_b = _app_status("mysql-k8s") - mysql_b_s3 = Mock() - mysql_b_s3.interface = "s3" - mysql_b.relations = {"s3-parameters": [mysql_b_s3]} - + mysql_a = _s3_related(_app_status("mysql-k8s")) + mysql_b = _s3_related(_app_status("mysql-k8s")) jhelper.get_model_status.return_value = _model_status( - { - "keystone-mysql": mysql_a, - "nova-mysql": mysql_b, - } + {"keystone-mysql": mysql_a, "nova-mysql": mysql_b} ) def _get_actions(app, model): @@ -155,18 +145,6 @@ def _run_action(unit, model, action, params=None, timeout=None): assert "resume" not in restore_actions assert "restore" not in restore_actions - def test_no_restore_or_resume_action_dispatched(self, deployment, jhelper): - jhelper.get_application_actions.return_value = [] - CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) - - restore_actions = { - call.args[2] - for call in jhelper.run_action.call_args_list - if len(call.args) > 2 - } - assert "restore-backup" not in restore_actions - assert "restore" not in restore_actions - def test_invalid_restore_to_time_fails_fast(self, deployment, jhelper): result = CliRunner().invoke( restore, ["--restore-to-time", "not-a-date"], obj=deployment @@ -180,12 +158,8 @@ def test_invalid_restore_to_time_fails_fast(self, deployment, jhelper): def test_unrelated_mysql_is_skipped_and_restore_continues( self, deployment, jhelper ): - mysql = _app_status("mysql-k8s") - mysql.relations = {} - vault = _app_status("vault-k8s") - vault_s3_relation = Mock() - vault_s3_relation.interface = "s3" - vault.relations = {"s3-parameters": [vault_s3_relation]} + mysql = _app_status("mysql-k8s") # no s3 + vault = _s3_related(_app_status("vault-k8s")) jhelper.get_model_status.return_value = _model_status( { "keystone-mysql": mysql, @@ -197,12 +171,11 @@ def test_unrelated_mysql_is_skipped_and_restore_continues( result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) assert result.exit_code == 0, result.output - assert "will be skipped" in result.output + assert "is not ready for backup" in result.output jhelper.scale_application.assert_not_called() def test_no_supported_apps_left(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") - mysql.relations = {} + mysql = _app_status("mysql-k8s") # no s3 jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) @@ -213,12 +186,8 @@ def test_no_supported_apps_left(self, deployment, jhelper): def test_unrelated_vault_is_skipped_and_restore_continues( self, deployment, jhelper ): - mysql = _app_status("mysql-k8s") - mysql_s3_relation = Mock() - mysql_s3_relation.interface = "s3" - mysql.relations = {"s3-parameters": [mysql_s3_relation]} - vault = _app_status("vault-k8s") - vault.relations = {} + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _app_status("vault-k8s") # no s3 jhelper.get_model_status.return_value = _model_status( { "keystone-mysql": mysql, @@ -230,7 +199,7 @@ def test_unrelated_vault_is_skipped_and_restore_continues( result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) assert result.exit_code == 0, result.output - assert "the following Vault applications are not related" in result.output + assert "vault is not ready for backup" in result.output def test_warns_on_pitr_for_vault(self, deployment, jhelper): result = CliRunner().invoke( @@ -240,17 +209,6 @@ def test_warns_on_pitr_for_vault(self, deployment, jhelper): ) assert result.exit_code == 0, result.output - assert "Vault does not support point-in-time restore" in result.output - - def test_no_supported_apps_left_when_only_vault(self, deployment, jhelper): - vault = _app_status("vault-k8s") - vault.relations = {} - jhelper.get_model_status.return_value = _model_status({"vault": vault}) - - result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) - - assert result.exit_code == 2, result.output - assert "No restore targets could be resolved. Exiting." in result.output def test_no_backups_found(self, deployment, jhelper): def _run_action(unit, model, action, params=None, timeout=None): @@ -303,27 +261,19 @@ def _run_action(unit, model, action, params=None, timeout=None): assert result.exit_code == 2, result.output assert "Failed to list backups for" in result.output - assert "keystone-mysql" in result.output - assert "vault" in result.output - def test_non_active_target_app_fails(self, deployment, jhelper): - mysql = _app_status("mysql-k8s") + def test_non_active_target_app_is_skipped(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) mysql.app_status.current = "error" jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) - assert result.exit_code == 1, result.output + assert result.exit_code == 2, result.output assert "keystone-mysql" in result.output - assert "error" in result.output - def test_mysql_restore_failure_attempts_cleanup_and_reports_app( - self, deployment, jhelper - ): - mysql = _app_status("mysql-k8s") - mysql_s3_relation = Mock() - mysql_s3_relation.interface = "s3" - mysql.relations = {"s3-parameters": [mysql_s3_relation]} + def test_mysql_restore_failure_reverts_and_reports(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) mysql.units = {"keystone-mysql/0": Mock(), "keystone-mysql/1": Mock()} jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) jhelper.get_application.return_value = mysql @@ -353,9 +303,10 @@ def _run_action(unit, model, action, params=None, timeout=None): result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) - assert result.exit_code == 1, result.output + assert result.exit_code == 2, result.output assert "keystone-mysql" in result.output assert "restore failed" in result.output + # scale down to 1 then revert back up to 2 assert jhelper.scale_application.call_args_list == [ ((OPENSTACK_MODEL, "keystone-mysql", 1),), ((OPENSTACK_MODEL, "keystone-mysql", 2),), diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py index 4cfca93bb..1ccef38aa 100644 --- a/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py @@ -12,25 +12,30 @@ ) from sunbeam.steps.backup_restore import ( BACKUP_COMPONENTS, - LIST_BACKUPS_ACTION, - MYSQL_S3_RELATION, + MYSQL_CHARM, + S3_ENDPOINT, S3_INTERFACE, + VAULT_CHARM, + ActionTarget, BackupComponent, BackupInventory, - BackupItem, + BackupOutcome, BackupResult, - BackupTarget, - CheckS3RelationsStep, DiscoverBackupApplicationsStep, ListBackupsStep, - ResolveBackupTargetsStep, + ResolveActionTargetsStep, RunBackupsStep, + ValidateStep, WriteBackupInventoryManifestStep, WriteBackupManifestStep, + _BackupAppStep, + _build_vault_backup_plan, + _build_vault_restore_plan, + _component_for, _parse_mysql_backups, _parse_vault_backups, - _resolve_mysql_target, - _resolve_vault_target, + _resolve_mysql_backup_target, + _resolve_vault_backup_target, ) @@ -62,18 +67,21 @@ def test_construction_with_error_does_not_raise(self): result = BackupResult( app="keystone-mysql", unit="keystone-mysql/0", - component="mysql", + component=MYSQL_CHARM, error="boom", ) assert result.error == "boom" assert result.backup is None - assert result.error == "boom" class TestRegistry: def test_registry_contains_mysql_and_vault(self): names = {c.name for c in BACKUP_COMPONENTS} - assert names == {"mysql", "vault"} + assert names == {MYSQL_CHARM, VAULT_CHARM} + + def test_components_have_restore_plans(self): + for component in BACKUP_COMPONENTS: + assert component.build_restore_plan is not None class TestResolveMySQLTarget: @@ -86,13 +94,13 @@ def test_picks_secondary_unit(self): ) jhelper.get_application.return_value = app - target = _resolve_mysql_target( + target = _resolve_mysql_backup_target( jhelper, "keystone-mysql", "openstack", force=False ) assert target is not None assert target.unit == "keystone-mysql/1" - assert target.is_replica is True + assert target.component == MYSQL_CHARM assert target.scale == 2 def test_falls_back_to_leader_when_no_secondary(self): @@ -109,13 +117,12 @@ def test_falls_back_to_leader_when_no_secondary(self): "mysql-k8s", units={"cinder-mysql/0": Mock()} ) - target = _resolve_mysql_target( + target = _resolve_mysql_backup_target( jhelper, "cinder-mysql", "openstack", force=False ) assert target is not None assert target.unit == "cinder-mysql/0" - assert target.is_replica is False assert target.scale == 1 def test_skips_on_action_failure_without_force(self): @@ -126,7 +133,7 @@ def test_skips_on_action_failure_without_force(self): ) jhelper.run_action.side_effect = ActionFailedException("failed") - target = _resolve_mysql_target( + target = _resolve_mysql_backup_target( jhelper, "keystone-mysql", "openstack", force=False ) @@ -140,20 +147,19 @@ def test_uses_leader_on_action_failure_with_force(self): ) jhelper.run_action.side_effect = ActionFailedException("failed") - target = _resolve_mysql_target( + target = _resolve_mysql_backup_target( jhelper, "keystone-mysql", "openstack", force=True ) assert target is not None assert target.unit == "keystone-mysql/0" - assert target.is_replica is False assert target.scale == 1 def test_skips_when_no_leader(self): jhelper = Mock() jhelper.get_leader_unit.side_effect = LeaderNotFoundException("no leader") - target = _resolve_mysql_target( + target = _resolve_mysql_backup_target( jhelper, "keystone-mysql", "openstack", force=True ) @@ -168,19 +174,22 @@ def test_resolves_leader(self): "vault-k8s", units={"vault/0": Mock()} ) - target = _resolve_vault_target(jhelper, "vault", "openstack", force=False) + target = _resolve_vault_backup_target( + jhelper, "vault", "openstack", force=False + ) assert target is not None assert target.unit == "vault/0" - assert target.component == "vault" - assert target.is_replica is False + assert target.component == VAULT_CHARM assert target.scale == 1 def test_skips_when_no_leader(self): jhelper = Mock() jhelper.get_leader_unit.side_effect = LeaderNotFoundException("no leader") - target = _resolve_vault_target(jhelper, "vault", "openstack", force=False) + target = _resolve_vault_backup_target( + jhelper, "vault", "openstack", force=False + ) assert target is None @@ -200,8 +209,8 @@ def test_discovers_by_charm_name(self, step_context): result = DiscoverBackupApplicationsStep(jhelper).run(step_context) assert result.result_type == ResultType.COMPLETED - assert sorted(result.message["mysql"]) == ["keystone-mysql", "nova-mysql"] - assert result.message["vault"] == ["vault"] + assert sorted(result.message[MYSQL_CHARM]) == ["keystone-mysql", "nova-mysql"] + assert result.message[VAULT_CHARM] == ["vault"] def test_fails_on_model_error(self, step_context): jhelper = Mock() @@ -211,26 +220,21 @@ def test_fails_on_model_error(self, step_context): assert result.result_type == ResultType.FAILED - def test_fails_when_target_app_is_not_active(self, step_context): + def test_non_active_app_is_still_discovered(self, step_context): + """Discovery is state-agnostic; validation filters non-active apps.""" jhelper = Mock() mysql = _app_status("mysql-k8s") mysql.app_status.current = "blocked" - jhelper.get_model_status.return_value = _model_status( - { - "keystone-mysql": mysql, - "vault": _app_status("vault-k8s"), - } - ) + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) result = DiscoverBackupApplicationsStep(jhelper).run(step_context) - assert result.result_type == ResultType.FAILED - assert "keystone-mysql" in result.message - assert "blocked" in result.message + assert result.result_type == ResultType.COMPLETED + assert result.message[MYSQL_CHARM] == ["keystone-mysql"] -class TestResolveBackupTargetsStep: - def test_flattens_targets_and_skips_unresolvable(self, step_context): +class TestResolveActionTargetsStep: + def test_resolves_leaders_and_skips_unresolvable(self, step_context): jhelper = Mock() def _leader(app, model): @@ -239,89 +243,130 @@ def _leader(app, model): return f"{app}/0" jhelper.get_leader_unit.side_effect = _leader - jhelper.run_action.return_value = _cluster_status(1) - jhelper.get_application.side_effect = lambda app, model: _app_status( - "mysql-k8s", units={f"{app}/0": Mock(), f"{app}/1": Mock()} - ) - discovered = {"mysql": ["keystone-mysql", "broken-mysql"], "vault": ["vault"]} - result = ResolveBackupTargetsStep(jhelper, discovered).run(step_context) + discovered = { + MYSQL_CHARM: ["keystone-mysql", "broken-mysql"], + VAULT_CHARM: ["vault"], + } + result = ResolveActionTargetsStep(jhelper, discovered).run(step_context) assert result.result_type == ResultType.COMPLETED apps = {t.app for t in result.message} assert apps == {"keystone-mysql", "vault"} + assert all(t.unit.endswith("/0") for t in result.message) -class TestCheckS3RelationsStep: - def test_partitions_apps_by_endpoint_and_s3_interface(self, step_context): +class TestValidateStep: + def test_partitions_by_active_and_s3(self, step_context): jhelper = Mock() - s3_relation = Mock() - s3_relation.interface = S3_INTERFACE - non_s3_relation = Mock() - non_s3_relation.interface = "other" + s3 = Mock() + s3.interface = S3_INTERFACE + ready = _app_status("mysql-k8s", relations={S3_ENDPOINT: [s3]}) + no_s3 = _app_status("mysql-k8s") + inactive = _app_status("mysql-k8s", relations={S3_ENDPOINT: [s3]}) + inactive.app_status.current = "blocked" jhelper.get_model_status.return_value = _model_status( { - "keystone-mysql": _app_status( - "mysql-k8s", relations={MYSQL_S3_RELATION: [s3_relation]} - ), - "nova-mysql": _app_status( - "mysql-k8s", relations={MYSQL_S3_RELATION: [non_s3_relation]} - ), + "keystone-mysql": ready, + "nova-mysql": no_s3, + "glance-mysql": inactive, } ) - result = CheckS3RelationsStep( - jhelper, - ["keystone-mysql", "nova-mysql"], - endpoint_name=MYSQL_S3_RELATION, - ).run(step_context) + discovered = {MYSQL_CHARM: ["keystone-mysql", "nova-mysql", "glance-mysql"]} + result = ValidateStep(jhelper, discovered).run(step_context) assert result.result_type == ResultType.COMPLETED - assert result.message == { - "related": ["keystone-mysql"], - "unrelated": ["nova-mysql"], - } + assert result.message["valid"][MYSQL_CHARM] == ["keystone-mysql"] + assert result.message["failures"]["nova-mysql"] == ["s3-relation"] + assert result.message["failures"]["glance-mysql"] == ["active"] - def test_missing_app_is_treated_as_unrelated(self, step_context): + def test_missing_app_fails_all_checks(self, step_context): jhelper = Mock() jhelper.get_model_status.return_value = _model_status({}) - result = CheckS3RelationsStep( - jhelper, ["keystone-mysql"], endpoint_name=MYSQL_S3_RELATION - ).run(step_context) + discovered = {MYSQL_CHARM: ["keystone-mysql"]} + result = ValidateStep(jhelper, discovered).run(step_context) assert result.result_type == ResultType.COMPLETED - assert result.message == { - "related": [], - "unrelated": ["keystone-mysql"], - } + assert result.message["valid"][MYSQL_CHARM] == [] + assert "keystone-mysql" in result.message["failures"] + + +class TestBackupAppStep: + def test_success_records_backup_result(self, step_context): + jhelper = Mock() + jhelper.run_action.return_value = {"backup-id": "id-1"} + component = _component_for(MYSQL_CHARM) + target = ActionTarget( + "keystone-mysql", "keystone-mysql/1", MYSQL_CHARM, "create-backup", 2 + ) + + step = _BackupAppStep(jhelper, component, target) + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert step.result is not None + assert step.result.backup is not None + assert step.result.backup.success is True + assert step.result.backup.backup_id == "id-1" + + def test_timeout_is_marked_in_progress(self, step_context): + jhelper = Mock() + jhelper.run_action.side_effect = Exception( + "timed out waiting for results from: unit nova-mysql/0" + ) + component = _component_for(MYSQL_CHARM) + target = ActionTarget( + "nova-mysql", "nova-mysql/0", MYSQL_CHARM, "create-backup", 2 + ) + + step = _BackupAppStep(jhelper, component, target) + step.run(step_context) + + assert step.result is not None + assert step.result.error is not None + assert step.result.backup is not None + assert step.result.backup.success is None + + def test_force_only_applies_when_supported(self, step_context): + jhelper = Mock() + jhelper.run_action.return_value = {"backup-id": "id"} + component = _component_for(VAULT_CHARM) # supports_force is False + target = ActionTarget("vault", "vault/0", VAULT_CHARM, "create-backup", 1) + + _BackupAppStep(jhelper, component, target, force=True).run(step_context) + + assert jhelper.run_action.call_args.args[3] is None class TestRunBackupsStep: - def test_aggregates_mixed_results(self, step_context): + def test_resolves_and_aggregates_mixed_results(self, step_context): jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.side_effect = lambda app, model: _app_status( + "mysql-k8s", units={f"{app}/0": Mock(), f"{app}/1": Mock()} + ) def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return _cluster_status(1) if unit == "glance-mysql/1": raise ActionFailedException("backup failed") return {"backup-id": f"backup-{unit.replace('/', '-')}"} jhelper.run_action.side_effect = _run_action - targets = [ - BackupTarget( - "keystone-mysql", "keystone-mysql/1", "mysql", "create-backup", 2 - ), - BackupTarget("glance-mysql", "glance-mysql/1", "mysql", "create-backup", 2), - BackupTarget("vault", "vault/0", "vault", "create-backup", 1), - ] + discovered = { + MYSQL_CHARM: ["keystone-mysql", "glance-mysql"], + VAULT_CHARM: ["vault"], + } - result = RunBackupsStep(jhelper, targets).run(step_context) + result = RunBackupsStep(jhelper, discovered).run(step_context) assert result.result_type == ResultType.COMPLETED by_app = {r.app: r for r in result.message} assert by_app["keystone-mysql"].backup is not None assert by_app["keystone-mysql"].backup.success is True - assert by_app["keystone-mysql"].backup.backup_id == "backup-keystone-mysql-1" assert by_app["glance-mysql"].backup is None assert by_app["glance-mysql"].error is not None assert by_app["vault"].backup is not None @@ -329,22 +374,64 @@ def _run_action(unit, model, action, params=None, timeout=None): def test_force_passes_force_param_only_to_mysql(self, step_context): jhelper = Mock() - jhelper.run_action.return_value = {"backup-id": "id"} - targets = [ - BackupTarget( - "keystone-mysql", "keystone-mysql/1", "mysql", "create-backup", 2 - ), - BackupTarget("vault", "vault/0", "vault", "create-backup", 1), - ] + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.side_effect = lambda app, model: _app_status( + "mysql-k8s", units={f"{app}/0": Mock()} + ) - RunBackupsStep(jhelper, targets, force=True).run(step_context) + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + return {"backup-id": "id"} + + jhelper.run_action.side_effect = _run_action + discovered = {MYSQL_CHARM: ["keystone-mysql"], VAULT_CHARM: ["vault"]} + + RunBackupsStep(jhelper, discovered, force=True).run(step_context) params_by_unit = { - call.args[0]: call.args[3] for call in jhelper.run_action.call_args_list + call.args[0]: call.args[3] + for call in jhelper.run_action.call_args_list + if call.args[2] == "create-backup" } - assert params_by_unit["keystone-mysql/1"] == {"force": True} + assert params_by_unit["keystone-mysql/0"] == {"force": True} assert params_by_unit["vault/0"] is None + def test_timeout_is_marked_as_in_progress(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.side_effect = lambda app, model: _app_status( + "mysql-k8s", units={f"{app}/0": Mock()} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + raise Exception("timed out waiting for results from: unit nova-mysql/0") + + jhelper.run_action.side_effect = _run_action + discovered = {MYSQL_CHARM: ["nova-mysql"]} + + result = RunBackupsStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + backup_result = result.message[0] + assert backup_result.error is not None + assert backup_result.backup is not None + assert backup_result.backup.success is None + class TestListBackupsParsing: def test_parse_mysql_backup_ids_filters_finished_entries(self): @@ -389,8 +476,8 @@ def test_collects_backup_ids_by_target(self, step_context): jhelper = Mock() def _run_action(unit, model, action, params=None, timeout=None): - assert action == LIST_BACKUPS_ACTION - if unit == "keystone-mysql/1": + assert action == "list-backups" + if unit == "keystone-mysql/0": return { "backups": ( "backup-id | backup-type | backup-status\n" @@ -404,10 +491,10 @@ def _run_action(unit, model, action, params=None, timeout=None): jhelper.run_action.side_effect = _run_action targets = [ - BackupTarget( - "keystone-mysql", "keystone-mysql/1", "mysql", "create-backup", 2 + ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "list-backups", 2 ), - BackupTarget("vault", "vault/0", "vault", "create-backup", 1), + ActionTarget("vault", "vault/0", VAULT_CHARM, "list-backups", 1), ] result = ListBackupsStep(jhelper, targets).run(step_context) @@ -427,8 +514,8 @@ def test_collects_errors_without_raising(self, step_context): jhelper = Mock() jhelper.run_action.side_effect = Exception("boom") targets = [ - BackupTarget( - "keystone-mysql", "keystone-mysql/1", "mysql", "create-backup", 2 + ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "list-backups", 2 ) ] @@ -445,10 +532,10 @@ def test_writes_manifest(self, step_context, tmp_path): BackupResult( "keystone-mysql", "keystone-mysql/1", - "mysql", - BackupItem("id-1", success=True), + MYSQL_CHARM, + BackupOutcome("id-1", success=True), ), - BackupResult("glance-mysql", "glance-mysql/1", "mysql", None, "err"), + BackupResult("glance-mysql", "glance-mysql/1", MYSQL_CHARM, None, "err"), ] step = WriteBackupManifestStep( results, "2026-04-09T14:22:01+00:00", manifest_dir=tmp_path @@ -476,13 +563,13 @@ def test_writes_inventory_manifest(self, step_context, tmp_path): BackupInventory( app="keystone-mysql", unit="keystone-mysql/1", - component="mysql", - backups=[BackupItem("2026-07-15T00:00:00Z", success=True)], + component=MYSQL_CHARM, + backups=[BackupOutcome("2026-07-15T00:00:00Z", success=True)], ), BackupInventory( app="vault", unit="vault/0", - component="vault", + component=VAULT_CHARM, error="failed", ), ] @@ -504,13 +591,14 @@ def test_new_component_flows_through_generic_pipeline( self, step_context, monkeypatch ): def _resolve_fake(jhelper, app, model, force): - return BackupTarget(app, f"{app}/0", "fake", "create-backup", 1) + return ActionTarget(app, f"{app}/0", "fake-charm", "create-backup", 1) fake = BackupComponent( - name="fake", - charm_names=["fake-charm"], - action="create-backup", - resolve_target=_resolve_fake, + name="fake-charm", + resolve_backup_target=_resolve_fake, + parse_backup_list=_parse_vault_backups, + build_backup_plan=_build_vault_backup_plan, + build_restore_plan=_build_vault_restore_plan, ) components = BACKUP_COMPONENTS + [fake] monkeypatch.setattr( @@ -526,14 +614,10 @@ def _resolve_fake(jhelper, app, model, force): discover = DiscoverBackupApplicationsStep(jhelper, components=components).run( step_context ) - assert discover.message["fake"] == ["my-fake"] - - resolve = ResolveBackupTargetsStep(jhelper, discover.message).run(step_context) - fake_targets = [t for t in resolve.message if t.component == "fake"] - assert len(fake_targets) == 1 + assert discover.message["fake-charm"] == ["my-fake"] - run = RunBackupsStep(jhelper, fake_targets).run(step_context) - assert run.message[0].component == "fake" + run = RunBackupsStep(jhelper, {"fake-charm": ["my-fake"]}).run(step_context) + assert run.message[0].component == "fake-charm" assert run.message[0].backup is not None assert run.message[0].backup.success is True assert run.message[0].backup.backup_id == "fake-backup" diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py index 18a5bceb0..136aa1837 100644 --- a/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py @@ -1,26 +1,38 @@ # SPDX-FileCopyrightText: 2026 - Canonical Ltd # SPDX-License-Identifier: Apache-2.0 +import dataclasses from unittest.mock import Mock from sunbeam.core.common import ResultType from sunbeam.core.juju import JujuException from sunbeam.steps.backup_restore import ( - BackupTarget, - PauseAppStep, - RestoreMySQLStep, - RestoreVaultStep, - ResumeAppStep, - ScaleAppStep, + MYSQL_CHARM, + VAULT_CHARM, + ActionTarget, + RestoreStep, + _component_for, + _PauseAppStep, + _RestoreAppStep, + _ResumeAppStep, + _ScaleAppStep, ) +def _mysql_component(): + return _component_for(MYSQL_CHARM) + + +def _vault_component(): + return _component_for(VAULT_CHARM) + + class TestGuardedSteps: def test_pause_dispatches_action_without_is_skip_check(self, step_context): jhelper = Mock() jhelper.get_leader_unit.return_value = "keystone-k8s/0" - step = PauseAppStep(jhelper, app="keystone-k8s") + step = _PauseAppStep(jhelper, app="keystone-k8s") result = step.run(step_context) assert result.result_type == ResultType.COMPLETED @@ -31,7 +43,7 @@ def test_resume_dispatches_action(self, step_context): jhelper = Mock() jhelper.get_leader_unit.return_value = "keystone-k8s/0" - step = ResumeAppStep(jhelper, app="keystone-k8s") + step = _ResumeAppStep(jhelper, app="keystone-k8s") result = step.run(step_context) assert result.result_type == ResultType.COMPLETED @@ -50,10 +62,10 @@ def test_restore_mysql_uses_latest_backup_id(self, step_context): }, {}, ] - target = BackupTarget( - "keystone-mysql", "keystone-mysql/0", "mysql", "restore", 3 + target = ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore", 3 ) - result = RestoreMySQLStep(jhelper, target).run(step_context) + result = _RestoreAppStep(jhelper, _mysql_component(), target).run(step_context) assert result.result_type == ResultType.COMPLETED assert jhelper.run_action.call_args_list[1].args[2] == "restore" @@ -64,12 +76,16 @@ def test_restore_mysql_uses_latest_backup_id(self, step_context): def test_restore_mysql_uses_restore_to_time(self, step_context): jhelper = Mock() jhelper.get_leader_unit.return_value = "keystone-mysql/0" - target = BackupTarget( - "keystone-mysql", "keystone-mysql/0", "mysql", "restore", 3 + component = dataclasses.replace( + _mysql_component(), supports_restore_to_time=True + ) + target = ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore", 3 ) - result = RestoreMySQLStep( + result = _RestoreAppStep( jhelper, + component, target, restore_to_time="2026-07-15 00:00:00", ).run(step_context) @@ -90,9 +106,9 @@ def test_restore_vault_uses_latest_backup(self, step_context): {"backup-ids": '["vault-backup-openstack-2026-07-15-00-03-28"]'}, {}, ] - target = BackupTarget("vault", "vault/0", "vault", "restore-backup", 1) + target = ActionTarget("vault", "vault/0", VAULT_CHARM, "restore-backup", 1) - result = RestoreVaultStep(jhelper, target).run(step_context) + result = _RestoreAppStep(jhelper, _vault_component(), target).run(step_context) assert result.result_type == ResultType.COMPLETED assert jhelper.run_action.call_args_list[1].args[2] == "restore-backup" @@ -104,7 +120,7 @@ def test_restore_vault_uses_latest_backup(self, step_context): class TestScaleMySQLStep: def test_scales_and_waits(self, step_context): jhelper = Mock() - result = ScaleAppStep(jhelper, "keystone-mysql", 1).run(step_context) + result = _ScaleAppStep(jhelper, "keystone-mysql", 1).run(step_context) assert result.result_type == ResultType.COMPLETED jhelper.scale_application.assert_called_once_with( "openstack", "keystone-mysql", 1 @@ -114,5 +130,103 @@ def test_scales_and_waits(self, step_context): def test_returns_failed_on_juju_error(self, step_context): jhelper = Mock() jhelper.scale_application.side_effect = JujuException("boom") - result = ScaleAppStep(jhelper, "keystone-mysql", 1).run(step_context) + result = _ScaleAppStep(jhelper, "keystone-mysql", 1).run(step_context) + assert result.result_type == ResultType.FAILED + + +def _finished_backup_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + +class TestRestoreStepWrapper: + def test_prechecks_all_before_any_restore_work(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.return_value = Mock( + units={"a/0": Mock(), "a/1": Mock()} + ) + jhelper.get_application_actions.return_value = [] # no pause/resume + jhelper.run_action.side_effect = _finished_backup_action + + targets = [ + ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore", 2 + ) + ] + + result = RestoreStep(jhelper, targets).run(step_context) + assert result.result_type == ResultType.FAILED + assert "pause/resume" in result.message + jhelper.scale_application.assert_not_called() + + def test_restores_each_target(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.return_value = Mock( + units={"a/0": Mock(), "a/1": Mock()} + ) + jhelper.get_application_actions.return_value = ["pause", "resume"] + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + return {"backup-ids": '["vault-backup-openstack-2026-07-15-00-03-28"]'} + return {} + + jhelper.run_action.side_effect = _run_action + + targets = [ + ActionTarget("vault", "vault/0", VAULT_CHARM, "restore-backup", 1), + ] + + result = RestoreStep(jhelper, targets).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message[0].success is True + + def test_reverts_mysql_on_restore_failure(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.return_value = Mock( + units={"a/0": Mock(), "a/1": Mock()} + ) + jhelper.get_application_actions.return_value = ["pause", "resume"] + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + if action == "restore": + raise JujuException("restore failed") + return {} + + jhelper.run_action.side_effect = _run_action + + targets = [ + ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore", 2 + ) + ] + + result = RestoreStep(jhelper, targets).run(step_context) + + assert result.result_type == ResultType.COMPLETED + outcome = result.message[0] + assert outcome.success is False + assert outcome.reverted is True + # scale down to 1, then revert scale back up + scale_calls = [call.args for call in jhelper.scale_application.call_args_list] + assert ("openstack", "keystone-mysql", 1) in scale_calls From e33a28b05e14e65f88887331bb7a4def44b9934f Mon Sep 17 00:00:00 2001 From: Deezzir Date: Fri, 17 Jul 2026 04:23:55 -0400 Subject: [PATCH 8/8] refactor: centralize unresolved target handling in list inventory flow Signed-off-by: Deezzir --- .../sunbeam/commands/backup_restore.py | 200 +++++++----- .../sunbeam/steps/backup_restore.py | 298 ++++++++++++------ .../sunbeam/commands/test_list_backups.py | 35 ++ .../unit/sunbeam/commands/test_restore.py | 180 ++++++++++- .../tests/unit/sunbeam/steps/test_backup.py | 38 +-- .../tests/unit/sunbeam/steps/test_restore.py | 30 +- 6 files changed, 548 insertions(+), 233 deletions(-) diff --git a/sunbeam-python/sunbeam/commands/backup_restore.py b/sunbeam-python/sunbeam/commands/backup_restore.py index fa56ea2fc..63241f603 100644 --- a/sunbeam-python/sunbeam/commands/backup_restore.py +++ b/sunbeam-python/sunbeam/commands/backup_restore.py @@ -20,7 +20,6 @@ DEFAULT_BACKUP_TIMEOUT, DEFAULT_RESTORE_TIMEOUT, RESTORE_TIME_FORMAT, - ActionTarget, BackupInventory, BackupResult, DiscoverBackupApplicationsStep, @@ -41,28 +40,36 @@ EXIT_PARTIAL = 1 EXIT_FAILURE = 2 +CONTINUE_BACKUP_QUESTION = ConfirmQuestion( + "Continue and back up the remaining components?", + default_value=False, + description=( + "Some discovered applications were skipped because they are not" + " ready for backup." + ), +) -def backup_questions() -> dict[str, Question]: - """Confirmation questions for the backup/restore commands.""" - return { - "continue_backup": ConfirmQuestion( - "Continue and back up the remaining components?", - default_value=False, - description=( - "Some discovered applications were skipped because they are not" - " ready for backup." - ), - ), - "continue_restore": ConfirmQuestion( - "Continue and restore the remaining components?", - default_value=False, - description=( - "Some discovered applications were skipped because they are not" - " ready for restore. Partial restores may result in " - "dangling OpenStack objects. " - ), - ), - } +CONTINUE_RESTORE_READY_QUESTION = ConfirmQuestion( + "Continue and restore the remaining components?", + default_value=False, + description=( + "Some discovered applications were skipped because they are not" + " ready for restore. Partial restores may result in " + "dangling OpenStack objects. " + ), +) + +CONTINUE_RESTORE_FILTER_QUESTION = ConfirmQuestion( + "Continue and restore the remaining components?", + default_value=False, + description=( + "Some discovered applications have missing or failed backups.\n" + "- Applications without successful backups will be skipped.\n" + "- Applications with mixed backup results will restore from successful " + "backups only.\n" + "Partial restores may result in dangling OpenStack objects." + ), +) def _confirm_or_abort(question: Question, no_prompt: bool) -> None: @@ -91,9 +98,12 @@ def _validate_apps( jhelper, discovered: dict[str, list[str]], model: str, + force: bool, ) -> tuple[dict[str, list[str]], bool]: """Run the validation step and warn on every skipped application.""" - results = run_plan([ValidateStep(jhelper, discovered, model=model)], console) + results = run_plan( + [ValidateStep(jhelper, discovered, model=model, force=force)], console + ) outcome = get_step_message(results, ValidateStep) valid = outcome["valid"] failures = outcome["failures"] @@ -108,29 +118,42 @@ def _validate_apps( return valid, bool(failures) -def _resolve_action_targets( +def _list_backup_inventory( jhelper, discovered: dict[str, list[str]], model: str, -) -> list[ActionTarget]: - """Resolve leader targets for generic actions (e.g. list-backups).""" + timeout: int, +) -> list[BackupInventory]: + """List available backups for the given targets.""" results = run_plan( [ResolveActionTargetsStep(jhelper, discovered, model=model)], console ) - return get_step_message(results, ResolveActionTargetsStep) - + resolved = get_step_message(results, ResolveActionTargetsStep) + targets = resolved["targets"] + unresolved_targets = resolved["unresolved"] -def _list_backup_inventory( - jhelper, - targets: list[ActionTarget], - model: str, - timeout: int, -) -> list[BackupInventory]: - """List available backups for the given targets.""" results = run_plan( - [ListBackupsStep(jhelper, targets, timeout=timeout, model=model)], console + [ + ListBackupsStep( + jhelper, + targets, + timeout=timeout, + model=model, + ) + ], + console, ) - return get_step_message(results, ListBackupsStep) + inventories = get_step_message(results, ListBackupsStep) + for unresolved in unresolved_targets: + inventories.append( + BackupInventory( + app=unresolved["app"], + unit="-", + component=unresolved["component"], + error="Could not resolve target.", + ) + ) + return inventories def _print_inventory(results: list[BackupInventory]) -> None: @@ -184,46 +207,61 @@ def _print_backup_summary(results: list[BackupResult]) -> None: def _filter_restore_targets( - targets: list[ActionTarget], + discovered: dict[str, list[str]], inventory: list[BackupInventory], -) -> list[ActionTarget]: +) -> tuple[dict[str, list[str]], bool]: """Warn on missing inventory and keep only restorable targets.""" - inventory_by_app = {entry.app: entry for entry in inventory} - failed_inventory = sorted( - ( - entry - for entry in inventory - if not entry.backups or not any(b.success is True for b in entry.backups) - ), - key=lambda entry: entry.app, - ) - for entry in failed_inventory: - if entry.error: + partially_failed_apps = { + entry.app + for entry in inventory + if entry.backups + and any(b.success is True for b in entry.backups) + and any(b.success is False for b in entry.backups) + } + + failed_inventory_by_app: dict[str, BackupInventory] = { + entry.app: entry + for entry in inventory + if entry.error is not None + or not entry.backups + or not any(b.success is True for b in entry.backups) + } + + for entry, inv in failed_inventory_by_app.items(): + if inv.error: console.print( "[yellow]Warning:[/yellow] Failed to list backups for " - f"{entry.app}: {entry.error}" + f"{entry}: {inv.error}" + ) + elif not inv.backups: + console.print( + f"[yellow]Warning:[/yellow] No backups available for {entry}." ) - elif not entry.backups: + elif any(b.success is False for b in inv.backups): console.print( - f"[yellow]Warning:[/yellow] No backups available for {entry.app}." + f"[yellow]Warning:[/yellow] Some backups failed for {entry}." + f" No successful backups are available for restore." ) else: console.print( f"[yellow]Warning:[/yellow] No successful backups available for" - f" {entry.app}." + f" {entry}." ) - restorable = [] - for target in targets: - target_inventory = inventory_by_app.get(target.app) - if ( - target_inventory is not None - and target_inventory.error is None - and target_inventory.backups - and any(b.success is True for b in target_inventory.backups) - ): - restorable.append(target) - return restorable + for app in sorted(partially_failed_apps): + console.print( + f"[yellow]Warning:[/yellow] Some backups for {app} failed." + " Only successful backups will be considered for restore" + " (possible out-of-band state)." + ) + + restorable: dict[str, list[str]] = {} + for component, apps in discovered.items(): + for app in apps: + if app in failed_inventory_by_app: + continue + restorable.setdefault(component, []).append(app) + return restorable, bool(failed_inventory_by_app) or bool(partially_failed_apps) def _validate_restore_to_time( @@ -276,9 +314,9 @@ def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> No console.print("No applications found to back up. Exiting.") sys.exit(EXIT_FAILURE) - discovered, was_filtered = _validate_apps(jhelper, discovered, model) + discovered, was_filtered = _validate_apps(jhelper, discovered, model, force=force) if was_filtered: - _confirm_or_abort(backup_questions()["continue_backup"], no_prompt) + _confirm_or_abort(CONTINUE_BACKUP_QUESTION, no_prompt) if not any(discovered.values()): console.print("No applications remain to back up after validation. Exiting.") @@ -349,16 +387,14 @@ def list_backups(ctx: click.Context, timeout: int) -> None: ) discovered = _discover_apps(jhelper, model) - discovered, _ = _validate_apps(jhelper, discovered, model) + discovered, _ = _validate_apps(jhelper, discovered, model, force=False) if not any(discovered.values()): console.print("No applications found to list backups from. Exiting.") sys.exit(EXIT_FAILURE) - targets = _resolve_action_targets(jhelper, discovered, model) - listed_at = datetime.now(timezone.utc).strftime(RESTORE_TIME_FORMAT) - inventory = _list_backup_inventory(jhelper, targets, model, timeout) + inventory = _list_backup_inventory(jhelper, discovered, model, timeout) failed_inventory = sorted( (entry for entry in inventory if entry.error), @@ -431,27 +467,27 @@ def restore( console.print("No applications found to restore. Exiting.") sys.exit(EXIT_FAILURE) - discovered, was_filtered = _validate_apps(jhelper, discovered, model) + discovered, was_filtered = _validate_apps(jhelper, discovered, model, force=force) if was_filtered: - _confirm_or_abort(backup_questions()["continue_restore"], no_prompt) + _confirm_or_abort(CONTINUE_RESTORE_READY_QUESTION, no_prompt) - targets = _resolve_action_targets(jhelper, discovered, model) - if not targets: - console.print("No restore targets could be resolved. Exiting.") - sys.exit(EXIT_FAILURE) + inventory = _list_backup_inventory(jhelper, discovered, model, timeout) + _print_inventory(inventory) + + discovered, was_filtered = _filter_restore_targets(discovered, inventory) - inventory = _list_backup_inventory(jhelper, targets, model, timeout) - targets = _filter_restore_targets(targets, inventory) + if was_filtered: + _confirm_or_abort(CONTINUE_RESTORE_FILTER_QUESTION, no_prompt) - if not targets: - console.print("No backups were found to restore from. Exiting.") + if not any(discovered.values()): + console.print("No applications remain to restore after validation. Exiting.") sys.exit(EXIT_FAILURE) restore_results = run_plan( [ RestoreStep( jhelper, - targets, + discovered, restore_to_time=restore_to_time, timeout=timeout, model=model, diff --git a/sunbeam-python/sunbeam/steps/backup_restore.py b/sunbeam-python/sunbeam/steps/backup_restore.py index 1a08a520c..7c091cd2c 100644 --- a/sunbeam-python/sunbeam/steps/backup_restore.py +++ b/sunbeam-python/sunbeam/steps/backup_restore.py @@ -15,7 +15,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import asdict, dataclass, field from datetime import datetime, timezone -from enum import Enum from pathlib import Path import yaml @@ -46,8 +45,8 @@ S3_INTERFACE = "s3" S3_ENDPOINT = "s3-parameters" DEFAULT_BACKUP_TIMEOUT = 1800 -DEFAULT_SCALE_TIMEOUT = 1800 DEFAULT_RESTORE_TIMEOUT = 1800 +DEFAULT_ACTION_TIMEOUT = 120 BACKUP_MANIFEST_DIR = SHARE_PATH / "backups" PAUSE_ACTION = "pause" RESUME_ACTION = "resume" @@ -62,7 +61,6 @@ class ActionTarget: unit: str component: str action: str - scale: int = 1 @dataclass @@ -106,11 +104,13 @@ class RestoreResult: reverted: bool = False -class ValidationCheck(Enum): - """An application-readiness check (predicate resolved by identity).""" +@dataclass +class ValidationCheck: + """An application-readiness check.""" - ACTIVE = "active" - S3_RELATION = "s3-relation" + name: str + predicate: Callable[[object], bool] + forceable: bool = False ResolveTargetFn = Callable[[JujuHelper, str, str, bool], "ActionTarget | None"] @@ -119,15 +119,15 @@ class ValidationCheck(Enum): list[BaseStep], ] RestorePlanFn = Callable[ - [JujuHelper, "BackupComponent", "ActionTarget", str | None, int, str], + [JujuHelper, "BackupComponent", "ActionTarget", str | None, bool, int, str], list[BaseStep], ] RevertPlanFn = Callable[ - [JujuHelper, "BackupComponent", "ActionTarget", str], + [JujuHelper, "BackupComponent", "ActionTarget", bool, int, str], list[BaseStep], ] PrecheckPlanFn = Callable[ - [JujuHelper, "BackupComponent", "ActionTarget", str], + [JujuHelper, "BackupComponent", "ActionTarget", bool, int, str], list[BaseStep], ] @@ -140,6 +140,7 @@ class BackupComponent: resolve_backup_target: ResolveTargetFn parse_backup_list: Callable[[dict], list[BackupOutcome]] + parse_backup: Callable[[dict], BackupOutcome | None] build_backup_plan: BackupPlanFn build_restore_plan: RestorePlanFn build_restore_revert_plan: RevertPlanFn | None = None @@ -148,10 +149,9 @@ class BackupComponent: backup_action: str = BACKUP_ACTION restore_action: str = RESTORE_ACTION list_action: str = LIST_BACKUPS_ACTION - backup_id_key: str = BACKUP_RESULT_ID_KEY - supports_force: bool = False - supports_restore_to_time: bool = False + restore_to_time_param: str | None = None + backup_id_param: str = BACKUP_RESULT_ID_KEY validate_checks: list[ValidationCheck] = field(default_factory=list) @@ -179,14 +179,20 @@ def _is_related_to_s3(app_status: object) -> bool: return any(rel.interface == S3_INTERFACE for rel in endpoint_relations) -_VALIDATION_PREDICATES: dict[ValidationCheck, Callable[[object], bool]] = { - ValidationCheck.ACTIVE: _is_app_active, - ValidationCheck.S3_RELATION: _is_related_to_s3, -} +APP_READY_VALIDATION_CHECK: ValidationCheck = ValidationCheck( + name="active", + predicate=_is_app_active, + forceable=True, +) + +S3_RELATION_VALIDATION_CHECK: ValidationCheck = ValidationCheck( + name="s3-relation", + predicate=_is_related_to_s3, +) # --------------------------------------------------------------------------- -# Backup-list parsing +# Backup result parsing # --------------------------------------------------------------------------- def _parse_mysql_backups(action_result: dict) -> list[BackupOutcome]: """Parse MySQL list-backups output table and return finished backup IDs.""" @@ -211,6 +217,14 @@ def _parse_mysql_backups(action_result: dict) -> list[BackupOutcome]: return backups +def _parse_backup(action_result: dict) -> BackupOutcome | None: + """Parse create-backup output and return the backup ID.""" + backup_id = action_result.get(BACKUP_RESULT_ID_KEY) + if not isinstance(backup_id, str): + return None + return BackupOutcome(backup_id=backup_id, success=True) + + def _parse_vault_backups(action_result: dict) -> list[BackupOutcome]: """Parse Vault list-backups output and return backup IDs.""" raw_backup_ids = action_result.get("backup-ids") @@ -271,12 +285,13 @@ def _api_app_for_mysql(app_name: str) -> str: return app_name -def _current_scale(jhelper: JujuHelper, app: str, model: str, fallback: int) -> int: +def _current_scale(jhelper: JujuHelper, app: str, model: str) -> int: """Read the current unit count for an application, live.""" try: return len(list(jhelper.get_application(app, model).units)) except (ApplicationNotFoundException, JujuException): - return fallback + LOG.warning("Could not read current scale for %s, assuming 1", app) + return 1 # --------------------------------------------------------------------------- @@ -301,7 +316,6 @@ def _resolve_mysql_backup_target( LOG.warning("Could not resolve %s, skipping", app) return None - scale = len(units) try: result = jhelper.run_action(leader, model, MYSQL_CLUSTER_STATUS_ACTION) secondary = _secondary_unit_from_status(units, result) @@ -311,7 +325,6 @@ def _resolve_mysql_backup_target( unit=secondary, component=MYSQL_CHARM, action=BACKUP_ACTION, - scale=scale, ) except ActionFailedException as e: if not force: @@ -328,7 +341,6 @@ def _resolve_mysql_backup_target( unit=leader, component=MYSQL_CHARM, action=BACKUP_ACTION, - scale=scale, ) @@ -341,7 +353,6 @@ def _resolve_vault_backup_target( """Resolve the Vault backup target to the leader unit.""" try: leader = jhelper.get_leader_unit(app, model) - units = list(jhelper.get_application(app, model).units) except (LeaderNotFoundException, ApplicationNotFoundException): LOG.warning("Could not resolve %s, skipping", app) return None @@ -351,14 +362,13 @@ def _resolve_vault_backup_target( unit=leader, component=VAULT_CHARM, action=BACKUP_ACTION, - scale=len(units), ) # --------------------------------------------------------------------------- # Atomic action steps # --------------------------------------------------------------------------- -class _ApiAppActionStep(BaseStep): +class _ActionStep(BaseStep): """Dispatch an action on an application's leader.""" def __init__( @@ -368,6 +378,8 @@ def __init__( description: str, app: str, action_name: str, + force: bool = False, + timeout: int = DEFAULT_ACTION_TIMEOUT, model: str = OPENSTACK_MODEL, ): super().__init__(name, description) @@ -375,6 +387,8 @@ def __init__( self.model = model self.app = app self.action_name = action_name + self.force = force + self.timeout = timeout def run(self, context: StepContext) -> Result: """Run action on the application's leader.""" @@ -384,8 +398,16 @@ def run(self, context: StepContext) -> Result: leader, self.model, self.action_name, - timeout=DEFAULT_RESTORE_TIMEOUT, + timeout=self.timeout, ) + if not self.force: + self.jhelper.wait_until_desired_status( + self.model, + apps=[self.app], + status=[], + agent_status=["idle"], + timeout=self.timeout, + ) except ( ActionFailedException, LeaderNotFoundException, @@ -397,31 +419,49 @@ def run(self, context: StepContext) -> Result: return Result(ResultType.COMPLETED) -class _PauseAppStep(_ApiAppActionStep): +class _PauseAppStep(_ActionStep): """Pause application API services.""" - def __init__(self, jhelper: JujuHelper, app: str, model: str = OPENSTACK_MODEL): + def __init__( + self, + jhelper: JujuHelper, + app: str, + force: bool = False, + timeout=DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): super().__init__( jhelper, "Pause App", "Pause API container services", app, PAUSE_ACTION, - model, + force=force, + timeout=timeout, + model=model, ) -class _ResumeAppStep(_ApiAppActionStep): +class _ResumeAppStep(_ActionStep): """Resume application API services.""" - def __init__(self, jhelper: JujuHelper, app: str, model: str = OPENSTACK_MODEL): + def __init__( + self, + jhelper: JujuHelper, + app: str, + force: bool = False, + timeout=DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): super().__init__( jhelper, "Resume App", "Resume API container services", app, RESUME_ACTION, - model, + force=force, + timeout=timeout, + model=model, ) @@ -433,7 +473,8 @@ def __init__( jhelper: JujuHelper, application: str, scale: int, - timeout: int = DEFAULT_SCALE_TIMEOUT, + force: bool = False, + timeout: int = DEFAULT_ACTION_TIMEOUT, model: str = OPENSTACK_MODEL, ) -> None: super().__init__("Scale App", f"Scaling {application} to {scale} unit(s)") @@ -442,6 +483,7 @@ def __init__( self.scale = scale self.timeout = timeout self.model = model + self.force = force def run(self, context: StepContext) -> Result: """Scale the application and wait for it to settle.""" @@ -464,6 +506,7 @@ def __init__( component: "BackupComponent", target: ActionTarget, restore_to_time: str | None = None, + force: bool = False, timeout: int = DEFAULT_RESTORE_TIMEOUT, model: str = OPENSTACK_MODEL, ): @@ -474,6 +517,7 @@ def __init__( self.component = component self.model = model self.timeout = timeout + self.force = force def run(self, context: StepContext) -> Result: """Restore an app using latest backup or restore-to-time.""" @@ -482,9 +526,11 @@ def run(self, context: StepContext) -> Result: except (LeaderNotFoundException, JujuException) as e: return Result(ResultType.FAILED, str(e)) - params: dict[str, str] = {} - if self.restore_to_time is not None and self.component.supports_restore_to_time: - params["restore-to-time"] = self.restore_to_time + params: dict[str, str | bool] = {"force": True} if self.force else {} + restore_to_time_param = self.component.restore_to_time_param + + if self.restore_to_time is not None and restore_to_time_param is not None: + params[restore_to_time_param] = self.restore_to_time else: try: list_result = self.jhelper.run_action( @@ -502,7 +548,7 @@ def run(self, context: StepContext) -> Result: ResultType.FAILED, f"No finished backups found for {self.target.app}.", ) - params[self.component.backup_id_key] = latest + params[self.component.backup_id_param] = latest try: self.jhelper.run_action( @@ -539,30 +585,25 @@ def __init__( self.model = model self.result: BackupResult | None = None - def _action_params(self) -> dict | None: - if self.force and self.component.supports_force: - return {"force": True} - return None - def run(self, context: StepContext) -> Result: """Dispatch the backup action, recording the outcome on ``self.result``.""" target = self.target + params: dict[str, str | bool] = {"force": True} if self.force else {} + try: action_result = self.jhelper.run_action( target.unit, self.model, target.action, - self._action_params(), + params, timeout=self.timeout, ) - backup_id = action_result.get(self.component.backup_id_key) + backup = self.component.parse_backup(action_result) self.result = BackupResult( app=target.app, unit=target.unit, component=target.component, - backup=BackupOutcome(backup_id=backup_id, success=True) - if backup_id is not None - else None, + backup=backup, ) except Exception as e: message = str(e) @@ -587,6 +628,8 @@ def __init__( self, jhelper: JujuHelper, app: str, + force: bool = False, + timeout: int = DEFAULT_ACTION_TIMEOUT, model: str = OPENSTACK_MODEL, ): super().__init__( @@ -596,6 +639,8 @@ def __init__( self.jhelper = jhelper self.app = app self.model = model + self.force = force + self.timeout = timeout def run(self, context: StepContext) -> Result: """Fail fast if the application does not support pause/resume actions.""" @@ -608,11 +653,19 @@ def run(self, context: StepContext) -> Result: ) if PAUSE_ACTION not in actions or RESUME_ACTION not in actions: - return Result( - ResultType.FAILED, - f"Application {self.app} does not support the 'pause/resume' action " - "required for restore. No changes have been made.", - ) + if not self.force: + return Result( + ResultType.FAILED, + f"Control-plane application {self.app} does not support the " + "'pause/resume' action required for restore. " + "No changes have been made.", + ) + else: + LOG.warning( + "Control-plane application %s does not support pause/resume" + ", proceeding (--force).", + self.app, + ) return Result(ResultType.COMPLETED) @@ -656,11 +709,17 @@ def _build_mysql_restore_precheck_plan( jhelper: JujuHelper, component: BackupComponent, target: ActionTarget, + force: bool, + timeout: int, model: str, ) -> list[BaseStep]: """Precheck that the MySQL app's API app supports pause/resume.""" api_app = _api_app_for_mysql(target.app) - return [_CheckPauseResumeSupportStep(jhelper, api_app, model=model)] + return [ + _CheckPauseResumeSupportStep( + jhelper, api_app, force=force, timeout=timeout, model=model + ) + ] def _build_mysql_restore_plan( @@ -668,27 +727,25 @@ def _build_mysql_restore_plan( component: BackupComponent, target: ActionTarget, restore_to_time: str | None, + force: bool, timeout: int, model: str, ) -> list[BaseStep]: - """Build the MySQL restore plan: pause, scale down, restore, scale up, resume. - - The scale to restore back to is read live at plan-build time so the value - is accurate before the application is scaled down. - """ + """Build the MySQL restore plan: pause, scale down, restore, scale up, resume.""" api_app = _api_app_for_mysql(target.app) - original_scale = _current_scale(jhelper, target.app, model, target.scale) + original_scale = _current_scale(jhelper, target.app, model) return [ - _PauseAppStep(jhelper, api_app, model=model), + _PauseAppStep(jhelper, api_app, force=force, timeout=timeout, model=model), _ScaleAppStep( - jhelper, target.app, 1, timeout=DEFAULT_SCALE_TIMEOUT, model=model + jhelper, target.app, 1, force=force, timeout=timeout, model=model ), _RestoreAppStep( jhelper, component, target, restore_to_time=restore_to_time, + force=force, timeout=timeout, model=model, ), @@ -696,10 +753,17 @@ def _build_mysql_restore_plan( jhelper, target.app, original_scale, - timeout=DEFAULT_SCALE_TIMEOUT, + force=force, + timeout=timeout, + model=model, + ), + _ResumeAppStep( + jhelper, + api_app, + force=force, + timeout=timeout, model=model, ), - _ResumeAppStep(jhelper, api_app, model=model), ] @@ -707,20 +771,24 @@ def _build_mysql_restore_revert_plan( jhelper: JujuHelper, component: BackupComponent, target: ActionTarget, + force: bool, + timeout: int, model: str, ) -> list[BaseStep]: """Build the revert plan for a failed MySQL restore: scale back up, resume.""" api_app = _api_app_for_mysql(target.app) - original_scale = _current_scale(jhelper, target.app, model, target.scale) + original_scale = _current_scale(jhelper, target.app, model) + return [ _ScaleAppStep( jhelper, target.app, original_scale, - timeout=DEFAULT_SCALE_TIMEOUT, + force=force, + timeout=timeout, model=model, ), - _ResumeAppStep(jhelper, api_app, model=model), + _ResumeAppStep(jhelper, api_app, force=force, timeout=timeout, model=model), ] @@ -729,6 +797,7 @@ def _build_vault_restore_plan( component: BackupComponent, target: ActionTarget, restore_to_time: str | None, + force: bool, timeout: int, model: str, ) -> list[BaseStep]: @@ -739,6 +808,7 @@ def _build_vault_restore_plan( component, target, restore_to_time=restore_to_time, + force=force, timeout=timeout, model=model, ), @@ -753,28 +823,22 @@ def _build_vault_restore_plan( name=MYSQL_CHARM, resolve_backup_target=_resolve_mysql_backup_target, parse_backup_list=_parse_mysql_backups, + parse_backup=_parse_backup, build_backup_plan=_build_mysql_backup_plan, build_restore_plan=_build_mysql_restore_plan, build_restore_revert_plan=_build_mysql_restore_revert_plan, build_restore_precheck_plan=_build_mysql_restore_precheck_plan, - restore_action=RESTORE_ACTION, - supports_force=True, - validate_checks=[ - ValidationCheck.ACTIVE, - ValidationCheck.S3_RELATION, - ], + validate_checks=[APP_READY_VALIDATION_CHECK, S3_RELATION_VALIDATION_CHECK], ), BackupComponent( name=VAULT_CHARM, resolve_backup_target=_resolve_vault_backup_target, parse_backup_list=_parse_vault_backups, + parse_backup=_parse_backup, build_backup_plan=_build_vault_backup_plan, build_restore_plan=_build_vault_restore_plan, + validate_checks=[APP_READY_VALIDATION_CHECK, S3_RELATION_VALIDATION_CHECK], restore_action=VAULT_RESTORE_ACTION, - validate_checks=[ - ValidationCheck.ACTIVE, - ValidationCheck.S3_RELATION, - ], ), ] @@ -829,12 +893,14 @@ def __init__( discovered: dict[str, list[str]], model: str = OPENSTACK_MODEL, components: list[BackupComponent] = BACKUP_COMPONENTS, + force: bool = False, ): super().__init__("Validate applications", "Validating backup readiness") self.jhelper = jhelper self.discovered = discovered self.model = model self.components = components + self.force = force def run(self, context: StepContext) -> Result: """Return {'valid': {...}, 'failures': {app: [check names]}}.""" @@ -852,7 +918,7 @@ def run(self, context: StepContext) -> Result: continue for app_name in apps: app_status = status.apps.get(app_name) - failed = self._failed_checks(component, app_status) + failed = self._failed_checks(component, app_status, self.force) if failed: failures[app_name] = failed else: @@ -862,14 +928,14 @@ def run(self, context: StepContext) -> Result: @staticmethod def _failed_checks( - component: BackupComponent, app_status: object | None + component: BackupComponent, app_status: object | None, force: bool = False ) -> list[str]: if app_status is None: - return [check.value for check in component.validate_checks] or ["present"] + return [check.name for check in component.validate_checks] or ["present"] return [ - check.value + check.name for check in component.validate_checks - if not _VALIDATION_PREDICATES[check](app_status) + if not (check.forceable and force) and not check.predicate(app_status) ] @@ -891,8 +957,9 @@ def __init__( self.model = model def run(self, context: StepContext) -> Result: - """Return a flat list of resolved leader targets across components.""" + """Return resolved targets and unresolved apps across components.""" targets: list[ActionTarget] = [] + unresolved: list[dict[str, str]] = [] for component_name, apps in self.discovered.items(): component = _component_for(component_name) if component is None: @@ -902,6 +969,7 @@ def run(self, context: StepContext) -> Result: leader = self.jhelper.get_leader_unit(app, self.model) except (LeaderNotFoundException, ApplicationNotFoundException): self.update_status(context, f"skipped {app}") + unresolved.append({"app": app, "component": component.name}) continue targets.append( ActionTarget( @@ -912,7 +980,13 @@ def run(self, context: StepContext) -> Result: ) ) - return Result(ResultType.COMPLETED, targets) + return Result( + ResultType.COMPLETED, + { + "targets": targets, + "unresolved": sorted(unresolved, key=lambda item: item["app"]), + }, + ) class RunBackupsStep(BaseStep): @@ -1077,17 +1151,19 @@ class RestoreStep(BaseStep): def __init__( self, jhelper: JujuHelper, - targets: list[ActionTarget], + discovered: dict[str, list[str]], restore_to_time: str | None = None, + force: bool = False, timeout: int = DEFAULT_RESTORE_TIMEOUT, model: str = OPENSTACK_MODEL, ): super().__init__("Restore applications", "Restoring applications from backup") self.jhelper = jhelper - self.targets = targets + self.discovered = discovered self.restore_to_time = restore_to_time self.timeout = timeout self.model = model + self.force = force def _run_plan(self, plan: list[BaseStep], context: StepContext) -> None: """Run a plan of steps, raising RuntimeError on the first failure.""" @@ -1096,14 +1172,39 @@ def _run_plan(self, plan: list[BaseStep], context: StepContext) -> None: if result.result_type == ResultType.FAILED: raise RuntimeError(result.message) + def _resolve_targets( + self, context: StepContext + ) -> list[tuple[BackupComponent, ActionTarget]]: + """Resolve one backup target per discovered application.""" + resolved: list[tuple[BackupComponent, ActionTarget]] = [] + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + if component is None: + continue + for app in apps: + target = ActionTarget( + app=app, + unit=self.jhelper.get_leader_unit(app, self.model), + component=component.name, + action=component.restore_action, + ) + resolved.append((component, target)) + return resolved + def _restore_one( self, component: BackupComponent, target: ActionTarget, context: StepContext ) -> RestoreResult: + revert_plan: list[BaseStep] = [] + if component.build_restore_revert_plan is not None: + revert_plan = component.build_restore_revert_plan( + self.jhelper, component, target, self.force, self.timeout, self.model + ) plan = component.build_restore_plan( self.jhelper, component, target, self.restore_to_time, + self.force, self.timeout, self.model, ) @@ -1111,11 +1212,6 @@ def _restore_one( self._run_plan(plan, context) return RestoreResult(app=target.app, component=component.name, success=True) except Exception as e: - revert_plan: list[BaseStep] = [] - if component.build_restore_revert_plan is not None: - revert_plan = component.build_restore_revert_plan( - self.jhelper, component, target, self.model - ) reverted = False if revert_plan: try: @@ -1133,23 +1229,15 @@ def _restore_one( def run(self, context: StepContext) -> Result: """Precheck all targets, then restore each sequentially, aggregating.""" - planned: list[tuple[BackupComponent, ActionTarget]] = [] - for target in self.targets: - component = _component_for(target.component) - if component is None: - return Result( - ResultType.FAILED, - f"No restore plan defined for {target.app}.", - ) - planned.append((component, target)) + resolved = self._resolve_targets(context) + if not resolved: + return Result(ResultType.COMPLETED, []) - # Global fail-fast precheck: run every component's precheck before any - # restore work so an unsupported app never leaves others half-restored. - for component, target in planned: + for component, target in resolved: if component.build_restore_precheck_plan is None: continue precheck = component.build_restore_precheck_plan( - self.jhelper, component, target, self.model + self.jhelper, component, target, self.force, self.timeout, self.model ) try: self._run_plan(precheck, context) @@ -1157,7 +1245,7 @@ def run(self, context: StepContext) -> Result: return Result(ResultType.FAILED, str(e)) results: list[RestoreResult] = [] - for component, target in planned: + for component, target in resolved: self.update_status(context, f"restoring {target.app}") results.append(self._restore_one(component, target, context)) diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py index dd8f49d14..45395182b 100644 --- a/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py @@ -8,6 +8,7 @@ from click.testing import CliRunner from sunbeam.commands.backup_restore import list_backups +from sunbeam.core.juju import LeaderNotFoundException def _app_status(charm_name): @@ -148,3 +149,37 @@ def _run_action(unit, model, action, params=None, timeout=None): assert "Failed to list backups for" in result.output assert "keystone-mysql" in result.output assert "vault" in result.output + + def test_unresolved_targets_exit_2_with_warning(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + + def _leader(app, model): + if app == "vault": + raise LeaderNotFoundException("no leader") + return f"{app}/0" + + def _run_action(unit, model, action, params=None, timeout=None): + if unit == "keystone-mysql/0": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.get_leader_unit.side_effect = _leader + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert ( + "Failed to list backups for vault: Could not resolve target." + in result.output + ) diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py index 40b26fa3a..603a0727e 100644 --- a/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py @@ -8,7 +8,7 @@ from click.testing import CliRunner from sunbeam.commands.backup_restore import restore -from sunbeam.core.juju import JujuException +from sunbeam.core.juju import JujuException, LeaderNotFoundException from sunbeam.core.openstack import OPENSTACK_MODEL @@ -181,7 +181,10 @@ def test_no_supported_apps_left(self, deployment, jhelper): result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) assert result.exit_code == 2, result.output - assert "No restore targets could be resolved. Exiting." in result.output + assert ( + "No applications remain to restore after validation. Exiting." + in result.output + ) def test_unrelated_vault_is_skipped_and_restore_continues( self, deployment, jhelper @@ -237,7 +240,89 @@ def _run_action(unit, model, action, params=None, timeout=None): result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) assert result.exit_code == 2, result.output - assert "No backups were found to restore from. Exiting." in result.output + assert ( + "No applications remain to restore after validation. Exiting." + in result.output + ) + + def test_warns_when_some_backups_failed_but_restore_continues( + self, deployment, jhelper + ): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | failed\n" + "2026-07-14T00:00:00Z | physical | finished" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return { + "backup-ids": json.dumps( + ["vault-backup-openstack-2026-07-15-00-03-28"] + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "Some backups for keystone-mysql failed." in result.output + assert "Only successful backups will be" in result.output + assert "considered for restore" in result.output + + def test_partial_backup_failures_prompt_and_decline_aborts( + self, deployment, jhelper, monkeypatch + ): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | failed\n" + "2026-07-14T00:00:00Z | physical | finished" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return { + "backup-ids": json.dumps( + ["vault-backup-openstack-2026-07-15-00-03-28"] + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + + monkeypatch.setattr( + "sunbeam.commands.backup_restore.ConfirmQuestion.ask", + lambda self, *a, **k: False, + ) + + result = CliRunner().invoke(restore, obj=deployment) + + assert result.exit_code == 1, result.output + assert "Aborted" in result.output def test_inventory_lookup_failures_are_reported_and_exit_2( self, deployment, jhelper @@ -262,6 +347,95 @@ def _run_action(unit, model, action, params=None, timeout=None): assert result.exit_code == 2, result.output assert "Failed to list backups for" in result.output + def test_force_proceeds_when_target_app_is_inactive(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + mysql.app_status.current = "blocked" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--force", "--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + restore_calls = [ + call + for call in jhelper.run_action.call_args_list + if call.args[2] == "restore" + ] + assert restore_calls + + def test_force_does_not_bypass_inventory_target_resolution_failure( + self, deployment, jhelper + ): + mysql = _s3_related(_app_status("mysql-k8s")) + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + leader_calls = {"keystone-mysql": 0} + + def _leader(app, model): + if app != "keystone-mysql": + return f"{app}/0" + leader_calls[app] += 1 + if leader_calls[app] == 1: + raise LeaderNotFoundException("temporary leader lookup failure") + return "keystone-mysql/0" + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.get_leader_unit.side_effect = _leader + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--force", "--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert ( + "Failed to list backups for keystone-mysql: Could not resolve target." + in result.output + ) + assert ( + "No applications remain to restore after validation. Exiting." + in result.output + ) + assert not any( + call.args[2] == "restore" for call in jhelper.run_action.call_args_list + ) + def test_non_active_target_app_is_skipped(self, deployment, jhelper): mysql = _s3_related(_app_status("mysql-k8s")) mysql.app_status.current = "error" diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py index 1ccef38aa..c894f9760 100644 --- a/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py @@ -32,6 +32,7 @@ _build_vault_backup_plan, _build_vault_restore_plan, _component_for, + _parse_backup, _parse_mysql_backups, _parse_vault_backups, _resolve_mysql_backup_target, @@ -101,7 +102,6 @@ def test_picks_secondary_unit(self): assert target is not None assert target.unit == "keystone-mysql/1" assert target.component == MYSQL_CHARM - assert target.scale == 2 def test_falls_back_to_leader_when_no_secondary(self): jhelper = Mock() @@ -123,7 +123,6 @@ def test_falls_back_to_leader_when_no_secondary(self): assert target is not None assert target.unit == "cinder-mysql/0" - assert target.scale == 1 def test_skips_on_action_failure_without_force(self): jhelper = Mock() @@ -153,7 +152,6 @@ def test_uses_leader_on_action_failure_with_force(self): assert target is not None assert target.unit == "keystone-mysql/0" - assert target.scale == 1 def test_skips_when_no_leader(self): jhelper = Mock() @@ -181,7 +179,6 @@ def test_resolves_leader(self): assert target is not None assert target.unit == "vault/0" assert target.component == VAULT_CHARM - assert target.scale == 1 def test_skips_when_no_leader(self): jhelper = Mock() @@ -251,9 +248,12 @@ def _leader(app, model): result = ResolveActionTargetsStep(jhelper, discovered).run(step_context) assert result.result_type == ResultType.COMPLETED - apps = {t.app for t in result.message} + apps = {t.app for t in result.message["targets"]} assert apps == {"keystone-mysql", "vault"} - assert all(t.unit.endswith("/0") for t in result.message) + assert all(t.unit.endswith("/0") for t in result.message["targets"]) + assert result.message["unresolved"] == [ + {"app": "broken-mysql", "component": MYSQL_CHARM} + ] class TestValidateStep: @@ -299,7 +299,7 @@ def test_success_records_backup_result(self, step_context): jhelper.run_action.return_value = {"backup-id": "id-1"} component = _component_for(MYSQL_CHARM) target = ActionTarget( - "keystone-mysql", "keystone-mysql/1", MYSQL_CHARM, "create-backup", 2 + "keystone-mysql", "keystone-mysql/1", MYSQL_CHARM, "create-backup" ) step = _BackupAppStep(jhelper, component, target) @@ -318,7 +318,7 @@ def test_timeout_is_marked_in_progress(self, step_context): ) component = _component_for(MYSQL_CHARM) target = ActionTarget( - "nova-mysql", "nova-mysql/0", MYSQL_CHARM, "create-backup", 2 + "nova-mysql", "nova-mysql/0", MYSQL_CHARM, "create-backup" ) step = _BackupAppStep(jhelper, component, target) @@ -329,16 +329,6 @@ def test_timeout_is_marked_in_progress(self, step_context): assert step.result.backup is not None assert step.result.backup.success is None - def test_force_only_applies_when_supported(self, step_context): - jhelper = Mock() - jhelper.run_action.return_value = {"backup-id": "id"} - component = _component_for(VAULT_CHARM) # supports_force is False - target = ActionTarget("vault", "vault/0", VAULT_CHARM, "create-backup", 1) - - _BackupAppStep(jhelper, component, target, force=True).run(step_context) - - assert jhelper.run_action.call_args.args[3] is None - class TestRunBackupsStep: def test_resolves_and_aggregates_mixed_results(self, step_context): @@ -401,7 +391,8 @@ def _run_action(unit, model, action, params=None, timeout=None): if call.args[2] == "create-backup" } assert params_by_unit["keystone-mysql/0"] == {"force": True} - assert params_by_unit["vault/0"] is None + # vault does not support force, but we still pass it as a param ignored by Juju + assert params_by_unit["vault/0"] == {"force": True} def test_timeout_is_marked_as_in_progress(self, step_context): jhelper = Mock() @@ -492,9 +483,9 @@ def _run_action(unit, model, action, params=None, timeout=None): jhelper.run_action.side_effect = _run_action targets = [ ActionTarget( - "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "list-backups", 2 + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "list-backups" ), - ActionTarget("vault", "vault/0", VAULT_CHARM, "list-backups", 1), + ActionTarget("vault", "vault/0", VAULT_CHARM, "list-backups"), ] result = ListBackupsStep(jhelper, targets).run(step_context) @@ -515,7 +506,7 @@ def test_collects_errors_without_raising(self, step_context): jhelper.run_action.side_effect = Exception("boom") targets = [ ActionTarget( - "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "list-backups", 2 + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "list-backups" ) ] @@ -591,12 +582,13 @@ def test_new_component_flows_through_generic_pipeline( self, step_context, monkeypatch ): def _resolve_fake(jhelper, app, model, force): - return ActionTarget(app, f"{app}/0", "fake-charm", "create-backup", 1) + return ActionTarget(app, f"{app}/0", "fake-charm", "create-backup") fake = BackupComponent( name="fake-charm", resolve_backup_target=_resolve_fake, parse_backup_list=_parse_vault_backups, + parse_backup=_parse_backup, build_backup_plan=_build_vault_backup_plan, build_restore_plan=_build_vault_restore_plan, ) diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py index 136aa1837..19481dc35 100644 --- a/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py @@ -63,7 +63,7 @@ def test_restore_mysql_uses_latest_backup_id(self, step_context): {}, ] target = ActionTarget( - "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore", 3 + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore" ) result = _RestoreAppStep(jhelper, _mysql_component(), target).run(step_context) @@ -77,10 +77,10 @@ def test_restore_mysql_uses_restore_to_time(self, step_context): jhelper = Mock() jhelper.get_leader_unit.return_value = "keystone-mysql/0" component = dataclasses.replace( - _mysql_component(), supports_restore_to_time=True + _mysql_component(), restore_to_time_param="restore-to-time" ) target = ActionTarget( - "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore", 3 + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore" ) result = _RestoreAppStep( @@ -106,7 +106,7 @@ def test_restore_vault_uses_latest_backup(self, step_context): {"backup-ids": '["vault-backup-openstack-2026-07-15-00-03-28"]'}, {}, ] - target = ActionTarget("vault", "vault/0", VAULT_CHARM, "restore-backup", 1) + target = ActionTarget("vault", "vault/0", VAULT_CHARM, "restore-backup") result = _RestoreAppStep(jhelper, _vault_component(), target).run(step_context) @@ -156,13 +156,9 @@ def test_prechecks_all_before_any_restore_work(self, step_context): jhelper.get_application_actions.return_value = [] # no pause/resume jhelper.run_action.side_effect = _finished_backup_action - targets = [ - ActionTarget( - "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore", 2 - ) - ] + discovered = {MYSQL_CHARM: ["keystone-mysql"]} - result = RestoreStep(jhelper, targets).run(step_context) + result = RestoreStep(jhelper, discovered).run(step_context) assert result.result_type == ResultType.FAILED assert "pause/resume" in result.message @@ -183,11 +179,9 @@ def _run_action(unit, model, action, params=None, timeout=None): jhelper.run_action.side_effect = _run_action - targets = [ - ActionTarget("vault", "vault/0", VAULT_CHARM, "restore-backup", 1), - ] + discovered = {VAULT_CHARM: ["vault"]} - result = RestoreStep(jhelper, targets).run(step_context) + result = RestoreStep(jhelper, discovered).run(step_context) assert result.result_type == ResultType.COMPLETED assert result.message[0].success is True @@ -215,13 +209,9 @@ def _run_action(unit, model, action, params=None, timeout=None): jhelper.run_action.side_effect = _run_action - targets = [ - ActionTarget( - "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore", 2 - ) - ] + discovered = {MYSQL_CHARM: ["keystone-mysql"]} - result = RestoreStep(jhelper, targets).run(step_context) + result = RestoreStep(jhelper, discovered).run(step_context) assert result.result_type == ResultType.COMPLETED outcome = result.message[0]