diff --git a/sunbeam-python/sunbeam/commands/backup_restore.py b/sunbeam-python/sunbeam/commands/backup_restore.py new file mode 100644 index 000000000..63241f603 --- /dev/null +++ b/sunbeam-python/sunbeam/commands/backup_restore.py @@ -0,0 +1,516 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""``sunbeam backup``, ``sunbeam restore`` and ``sunbeam list-backups`` commands.""" + +import logging +import sys +from datetime import datetime, timezone + +import click +from rich.console import Console +from rich.table import Table + +from sunbeam.core.common import get_step_message, run_plan +from sunbeam.core.deployment import Deployment +from sunbeam.core.openstack import OPENSTACK_MODEL +from sunbeam.core.questions import ConfirmQuestion, Question +from sunbeam.steps.backup_restore import ( + BACKUP_COMPONENTS, + DEFAULT_BACKUP_TIMEOUT, + DEFAULT_RESTORE_TIMEOUT, + RESTORE_TIME_FORMAT, + BackupInventory, + BackupResult, + DiscoverBackupApplicationsStep, + ListBackupsStep, + ResolveActionTargetsStep, + RestoreResult, + RestoreStep, + RunBackupsStep, + ValidateStep, + WriteBackupInventoryManifestStep, + WriteBackupManifestStep, +) + +LOG = logging.getLogger(__name__) +console = Console() + +EXIT_SUCCESS = 0 +EXIT_PARTIAL = 1 +EXIT_FAILURE = 2 + +CONTINUE_BACKUP_QUESTION = ConfirmQuestion( + "Continue and back up the remaining components?", + default_value=False, + description=( + "Some discovered applications were skipped because they are not" + " ready for backup." + ), +) + +CONTINUE_RESTORE_READY_QUESTION = ConfirmQuestion( + "Continue and restore the remaining components?", + default_value=False, + description=( + "Some discovered applications were skipped because they are not" + " ready for restore. Partial restores may result in " + "dangling OpenStack objects. " + ), +) + +CONTINUE_RESTORE_FILTER_QUESTION = ConfirmQuestion( + "Continue and restore the remaining components?", + default_value=False, + description=( + "Some discovered applications have missing or failed backups.\n" + "- Applications without successful backups will be skipped.\n" + "- Applications with mixed backup results will restore from successful " + "backups only.\n" + "Partial restores may result in dangling OpenStack objects." + ), +) + + +def _confirm_or_abort(question: Question, no_prompt: bool) -> None: + """Ask a confirmation question, aborting the command if declined.""" + if no_prompt: + return + question.console = console + question.show_hint = True + if not question.ask(): + raise click.Abort() + + +def _components_banner() -> str: + return ",".join(c.name for c in BACKUP_COMPONENTS) + + +def _discover_apps(jhelper, model: str) -> dict[str, list[str]]: + """Discover applications for every registered backup component.""" + results = run_plan( + [DiscoverBackupApplicationsStep(jhelper, model, BACKUP_COMPONENTS)], console + ) + return get_step_message(results, DiscoverBackupApplicationsStep) + + +def _validate_apps( + jhelper, + discovered: dict[str, list[str]], + model: str, + force: bool, +) -> tuple[dict[str, list[str]], bool]: + """Run the validation step and warn on every skipped application.""" + results = run_plan( + [ValidateStep(jhelper, discovered, model=model, force=force)], console + ) + outcome = get_step_message(results, ValidateStep) + valid = outcome["valid"] + failures = outcome["failures"] + + for app in sorted(failures): + reasons = ", ".join(failures[app]) + console.print( + f"[yellow]Warning:[/yellow] {app} is not ready for backup" + f" ({reasons}) and will be skipped." + ) + + return valid, bool(failures) + + +def _list_backup_inventory( + jhelper, + discovered: dict[str, list[str]], + model: str, + timeout: int, +) -> list[BackupInventory]: + """List available backups for the given targets.""" + results = run_plan( + [ResolveActionTargetsStep(jhelper, discovered, model=model)], console + ) + resolved = get_step_message(results, ResolveActionTargetsStep) + targets = resolved["targets"] + unresolved_targets = resolved["unresolved"] + + results = run_plan( + [ + ListBackupsStep( + jhelper, + targets, + timeout=timeout, + model=model, + ) + ], + console, + ) + inventories = get_step_message(results, ListBackupsStep) + for unresolved in unresolved_targets: + inventories.append( + BackupInventory( + app=unresolved["app"], + unit="-", + component=unresolved["component"], + error="Could not resolve target.", + ) + ) + return inventories + + +def _print_inventory(results: list[BackupInventory]) -> None: + table = Table() + table.add_column("Application") + table.add_column("Component") + table.add_column("Backup IDs") + table.add_column("Status") + + for result in sorted(results, key=lambda inventory: inventory.app): + if result.backups: + ordered = sorted(result.backups, key=lambda b: b.backup_id, reverse=True) + backup_ids = "\n".join(b.backup_id for b in ordered) + statuses = "\n".join( + "[green]ok[/green]" + if b.success is True + else "[red]failed[/red]" + if b.success is False + else "-" + for b in ordered + ) + else: + backup_ids = "-" + statuses = "[red]failed[/red]" if result.error else "-" + table.add_row(result.app, result.component, backup_ids, statuses) + + console.print(table) + + +def _print_backup_summary(results: list[BackupResult]) -> None: + table = Table() + table.add_column("Application") + table.add_column("Component") + table.add_column("Backup ID") + table.add_column("Status") + for result in results: + if result.backup is None: + table.add_row(result.app, result.component, "-", "-") + continue + status = ( + "[green]ok[/green]" + if result.backup.success is True + else "[red]failed[/red]" + if result.backup.success is False + else "-" + ) + table.add_row( + result.app, result.component, result.backup.backup_id or "-", status + ) + console.print(table) + + +def _filter_restore_targets( + discovered: dict[str, list[str]], + inventory: list[BackupInventory], +) -> tuple[dict[str, list[str]], bool]: + """Warn on missing inventory and keep only restorable targets.""" + partially_failed_apps = { + entry.app + for entry in inventory + if entry.backups + and any(b.success is True for b in entry.backups) + and any(b.success is False for b in entry.backups) + } + + failed_inventory_by_app: dict[str, BackupInventory] = { + entry.app: entry + for entry in inventory + if entry.error is not None + or not entry.backups + or not any(b.success is True for b in entry.backups) + } + + for entry, inv in failed_inventory_by_app.items(): + if inv.error: + console.print( + "[yellow]Warning:[/yellow] Failed to list backups for " + f"{entry}: {inv.error}" + ) + elif not inv.backups: + console.print( + f"[yellow]Warning:[/yellow] No backups available for {entry}." + ) + elif any(b.success is False for b in inv.backups): + console.print( + f"[yellow]Warning:[/yellow] Some backups failed for {entry}." + f" No successful backups are available for restore." + ) + else: + console.print( + f"[yellow]Warning:[/yellow] No successful backups available for" + f" {entry}." + ) + + for app in sorted(partially_failed_apps): + console.print( + f"[yellow]Warning:[/yellow] Some backups for {app} failed." + " Only successful backups will be considered for restore" + " (possible out-of-band state)." + ) + + restorable: dict[str, list[str]] = {} + for component, apps in discovered.items(): + for app in apps: + if app in failed_inventory_by_app: + continue + restorable.setdefault(component, []).append(app) + return restorable, bool(failed_inventory_by_app) or bool(partially_failed_apps) + + +def _validate_restore_to_time( + ctx: click.Context, param: click.Parameter, value: str | None +) -> str | None: + if value is None: + return None + try: + datetime.strptime(value, RESTORE_TIME_FORMAT) + except ValueError: + raise click.BadParameter( + f"expected format 'YYYY-MM-DD HH:MM:SS', got {value!r}" + ) + return value + + +@click.command() +@click.option( + "--force", + is_flag=True, + default=False, + show_default=True, + help=( + "Back up applications whose cluster health cannot be verified, using the" + " leader unit. May capture stale data; use with caution." + ), +) +@click.option( + "--timeout", + default=DEFAULT_BACKUP_TIMEOUT, + show_default=True, + help="Time in seconds to wait for each backup to complete.", +) +@click.option( + "--no-prompt", is_flag=True, default=False, help="Do not prompt for confirmation." +) +@click.pass_context +def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> None: + """Create backups of stateful Sunbeam applications (MySQL and Vault).""" + deployment: Deployment = ctx.obj + jhelper = deployment.get_juju_helper() + model = OPENSTACK_MODEL + + console.print( + f"[bold]Backing up \\[{_components_banner()}] in model '{model}'...[/bold]" + ) + + discovered = _discover_apps(jhelper, model) + if not any(discovered.values()): + console.print("No applications found to back up. Exiting.") + sys.exit(EXIT_FAILURE) + + discovered, was_filtered = _validate_apps(jhelper, discovered, model, force=force) + if was_filtered: + _confirm_or_abort(CONTINUE_BACKUP_QUESTION, no_prompt) + + if not any(discovered.values()): + console.print("No applications remain to back up after validation. Exiting.") + sys.exit(EXIT_FAILURE) + + dispatched_at = datetime.now(timezone.utc).strftime(RESTORE_TIME_FORMAT) + console.print(f"Dispatching backups at {dispatched_at} UTC...") + backup_results = run_plan( + [ + RunBackupsStep( + jhelper, discovered, force=force, timeout=timeout, model=model + ) + ], + console, + ) + results: list[BackupResult] = get_step_message(backup_results, RunBackupsStep) + + if not results: + console.print( + "Could not resolve a backup target for any application. Re-run with" + " --force to back up on leader units regardless of cluster health." + ) + sys.exit(EXIT_FAILURE) + + _print_backup_summary(results) + + manifest_results = run_plan( + [WriteBackupManifestStep(results, dispatched_at)], console + ) + manifest_path = get_step_message(manifest_results, WriteBackupManifestStep) + if manifest_path: + console.print(f"Backup manifest written to: {manifest_path}") + + succeeded = sum(1 for r in results if r.backup is not None and r.backup.success) + failed = sum(1 for r in results if r.error is not None) + console.print(f"Backup summary: {succeeded} succeeded, {failed} failed.") + + if failed == 0: + sys.exit(EXIT_SUCCESS) + + console.print( + "[yellow]Warning:[/yellow] one or more backups failed or timed out. A partial" + " restore from a set may result in dangling OpenStack objects." + ) + + if succeeded == 0: + sys.exit(EXIT_FAILURE) + sys.exit(EXIT_PARTIAL) + + +@click.command("list-backups") +@click.option( + "--timeout", + default=DEFAULT_BACKUP_TIMEOUT, + show_default=True, + help="Time in seconds to wait for each list action to complete.", +) +@click.pass_context +def list_backups(ctx: click.Context, timeout: int) -> None: + """List backup IDs from stateful Sunbeam applications.""" + deployment: Deployment = ctx.obj + jhelper = deployment.get_juju_helper() + model = OPENSTACK_MODEL + + console.print( + f"[bold]Listing backups for \\[{_components_banner()}]" + f" in model '{model}'...[/bold]" + ) + + discovered = _discover_apps(jhelper, model) + discovered, _ = _validate_apps(jhelper, discovered, model, force=False) + + if not any(discovered.values()): + console.print("No applications found to list backups from. Exiting.") + sys.exit(EXIT_FAILURE) + + listed_at = datetime.now(timezone.utc).strftime(RESTORE_TIME_FORMAT) + inventory = _list_backup_inventory(jhelper, discovered, model, timeout) + + failed_inventory = sorted( + (entry for entry in inventory if entry.error), + key=lambda entry: entry.app, + ) + for entry in failed_inventory: + console.print( + f"[yellow]Warning:[/yellow] Failed to list backups for" + f" {entry.app}: {entry.error}" + ) + + _print_inventory(inventory) + + manifest_results = run_plan( + [WriteBackupInventoryManifestStep(inventory, listed_at)], console + ) + manifest_path = get_step_message(manifest_results, WriteBackupInventoryManifestStep) + if manifest_path: + console.print(f"Backup inventory manifest written to: {manifest_path}") + + if failed_inventory: + sys.exit(EXIT_FAILURE) + + sys.exit(EXIT_SUCCESS) + + +@click.command() +@click.option( + "--restore-to-time", + default=None, + callback=_validate_restore_to_time, + help="Point-in-time to restore to, formatted 'YYYY-MM-DD HH:MM:SS'.", +) +@click.option( + "--force", + is_flag=True, + default=False, + show_default=True, + help="Proceed with restore despite cluster health concerns.", +) +@click.option( + "--timeout", + default=DEFAULT_RESTORE_TIMEOUT, + show_default=True, + help="Time in seconds to wait for restore operations to complete.", +) +@click.option( + "--no-prompt", is_flag=True, default=False, help="Do not prompt for confirmation." +) +@click.pass_context +def restore( + ctx: click.Context, + restore_to_time: str | None, + force: bool, + timeout: int, + no_prompt: bool, +) -> None: + """Restore stateful Sunbeam applications from a backup.""" + deployment: Deployment = ctx.obj + jhelper = deployment.get_juju_helper() + model = OPENSTACK_MODEL + + console.print( + f"[bold]Restoring \\[{_components_banner()}]" + f" in model '{model}' from backup...[/bold]" + ) + + discovered = _discover_apps(jhelper, model) + if not any(discovered.values()): + console.print("No applications found to restore. Exiting.") + sys.exit(EXIT_FAILURE) + + discovered, was_filtered = _validate_apps(jhelper, discovered, model, force=force) + if was_filtered: + _confirm_or_abort(CONTINUE_RESTORE_READY_QUESTION, no_prompt) + + inventory = _list_backup_inventory(jhelper, discovered, model, timeout) + _print_inventory(inventory) + + discovered, was_filtered = _filter_restore_targets(discovered, inventory) + + if was_filtered: + _confirm_or_abort(CONTINUE_RESTORE_FILTER_QUESTION, no_prompt) + + if not any(discovered.values()): + console.print("No applications remain to restore after validation. Exiting.") + sys.exit(EXIT_FAILURE) + + restore_results = run_plan( + [ + RestoreStep( + jhelper, + discovered, + restore_to_time=restore_to_time, + timeout=timeout, + model=model, + ) + ], + console, + ) + results: list[RestoreResult] = get_step_message(restore_results, RestoreStep) + + for result in results: + if not result.success: + reverted = " Reverted." if result.reverted else "" + console.print( + f"[red]Error:[/red] {result.app} restore failed:" + f" {result.error}.{reverted}" + ) + + succeeded = sum(1 for r in results if r.success) + failed = sum(1 for r in results if not r.success) + console.print(f"Restore summary: {succeeded} succeeded, {failed} failed.") + + if failed == 0: + sys.exit(EXIT_SUCCESS) + if succeeded == 0: + sys.exit(EXIT_FAILURE) + sys.exit(EXIT_PARTIAL) diff --git a/sunbeam-python/sunbeam/core/juju.py b/sunbeam-python/sunbeam/core/juju.py index e1f095f8d..e875923af 100644 --- a/sunbeam-python/sunbeam/core/juju.py +++ b/sunbeam-python/sunbeam/core/juju.py @@ -857,6 +857,24 @@ def run_action( raise ActionFailedException(str(task)) return task.results + def get_application_actions(self, application: str, model: str) -> list[str]: + """Return action names available for an application in a model.""" + with self._model(model) as juju: + try: + response = self.cli( + "actions", + application, + include_controller=False, + juju=juju, + ) + except jubilant.CLIError as e: + raise JujuException(str(e)) from e + + actions = response.get(application) + if not isinstance(actions, dict): + return [] + return sorted(actions.keys()) + def add_secret(self, model: str, name: str, data: dict, info: str) -> str: """Add secret to the model. diff --git a/sunbeam-python/sunbeam/main.py b/sunbeam-python/sunbeam/main.py index 69efba9d3..96fc35833 100644 --- a/sunbeam-python/sunbeam/main.py +++ b/sunbeam-python/sunbeam/main.py @@ -9,6 +9,7 @@ from snaphelpers import Snap from sunbeam import log +from sunbeam.commands import backup_restore as backup_restore_cmds from sunbeam.commands import configure as configure_cmds from sunbeam.commands import dashboard as dashboard_cmds from sunbeam.commands import generate_cloud_config as generate_cloud_config_cmds @@ -123,6 +124,9 @@ def main(): cli.add_command(launch_cmds.launch) cli.add_command(openrc_cmds.openrc) cli.add_command(dashboard_cmds.dashboard) + cli.add_command(backup_restore_cmds.backup) + cli.add_command(backup_restore_cmds.list_backups) + cli.add_command(backup_restore_cmds.restore) # Add identity group cli.add_command(identity_group) diff --git a/sunbeam-python/sunbeam/steps/backup_restore.py b/sunbeam-python/sunbeam/steps/backup_restore.py new file mode 100644 index 000000000..7c091cd2c --- /dev/null +++ b/sunbeam-python/sunbeam/steps/backup_restore.py @@ -0,0 +1,1338 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Steps and component registry for ``sunbeam backup/restore``. + +The module composes and +runs the top-level steps defined here. All per-component logic (validation +checks, target resolution, and backup/list/restore/revert plans) lives on the +:class:`BackupComponent` registry and is driven by the wrapper steps below. +""" + +import json +import logging +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +import yaml +from snaphelpers import Snap + +from sunbeam.core.common import SHARE_PATH, BaseStep, Result, ResultType, StepContext +from sunbeam.core.juju import ( + ActionFailedException, + ApplicationNotFoundException, + JujuException, + JujuHelper, + JujuWaitException, + LeaderNotFoundException, + ModelNotFoundException, +) +from sunbeam.core.openstack import OPENSTACK_MODEL + +LOG = logging.getLogger(__name__) + +MYSQL_CHARM = "mysql-k8s" +VAULT_CHARM = "vault-k8s" +BACKUP_ACTION = "create-backup" +RESTORE_ACTION = "restore" +BACKUP_RESULT_ID_KEY = "backup-id" +LIST_BACKUPS_ACTION = "list-backups" +VAULT_RESTORE_ACTION = "restore-backup" +MYSQL_CLUSTER_STATUS_ACTION = "get-cluster-status" +S3_INTERFACE = "s3" +S3_ENDPOINT = "s3-parameters" +DEFAULT_BACKUP_TIMEOUT = 1800 +DEFAULT_RESTORE_TIMEOUT = 1800 +DEFAULT_ACTION_TIMEOUT = 120 +BACKUP_MANIFEST_DIR = SHARE_PATH / "backups" +PAUSE_ACTION = "pause" +RESUME_ACTION = "resume" +RESTORE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S" + + +@dataclass +class ActionTarget: + """An application and the unit chosen to run an action against.""" + + app: str + unit: str + component: str + action: str + + +@dataclass +class BackupOutcome: + """A single backup entry in a backup inventory.""" + + backup_id: str + success: bool | None = None + + +@dataclass +class BackupResult: + """The outcome of attempting a backup for a single application.""" + + app: str + unit: str + component: str + backup: BackupOutcome | None = None + error: str | None = None + + +@dataclass +class BackupInventory: + """The result of listing backup IDs for a single application target.""" + + app: str + unit: str + component: str + backups: list[BackupOutcome] | None = None + error: str | None = None + + +@dataclass +class RestoreResult: + """The outcome of attempting a restore for a single application.""" + + app: str + component: str + success: bool + error: str | None = None + reverted: bool = False + + +@dataclass +class ValidationCheck: + """An application-readiness check.""" + + name: str + predicate: Callable[[object], bool] + forceable: bool = False + + +ResolveTargetFn = Callable[[JujuHelper, str, str, bool], "ActionTarget | None"] +BackupPlanFn = Callable[ + [JujuHelper, "BackupComponent", "ActionTarget", bool, int, str], + list[BaseStep], +] +RestorePlanFn = Callable[ + [JujuHelper, "BackupComponent", "ActionTarget", str | None, bool, int, str], + list[BaseStep], +] +RevertPlanFn = Callable[ + [JujuHelper, "BackupComponent", "ActionTarget", bool, int, str], + list[BaseStep], +] +PrecheckPlanFn = Callable[ + [JujuHelper, "BackupComponent", "ActionTarget", bool, int, str], + list[BaseStep], +] + + +@dataclass +class BackupComponent: + """Descriptor for a kind of stateful application that can be backed up.""" + + name: str + + resolve_backup_target: ResolveTargetFn + parse_backup_list: Callable[[dict], list[BackupOutcome]] + parse_backup: Callable[[dict], BackupOutcome | None] + build_backup_plan: BackupPlanFn + build_restore_plan: RestorePlanFn + build_restore_revert_plan: RevertPlanFn | None = None + build_restore_precheck_plan: PrecheckPlanFn | None = None + + backup_action: str = BACKUP_ACTION + restore_action: str = RESTORE_ACTION + list_action: str = LIST_BACKUPS_ACTION + + restore_to_time_param: str | None = None + backup_id_param: str = BACKUP_RESULT_ID_KEY + + validate_checks: list[ValidationCheck] = field(default_factory=list) + + +def _component_for(name: str) -> BackupComponent | None: + return next((c for c in BACKUP_COMPONENTS if c.name == name), None) + + +# --------------------------------------------------------------------------- +# Validation predicates +# --------------------------------------------------------------------------- +def _is_app_active(app_status: object) -> bool: + """Return whether application workload status is active.""" + status = getattr(app_status, "app_status", None) + current = getattr(status, "current", None) + if not isinstance(current, str): + return True + return current == "active" + + +def _is_related_to_s3(app_status: object) -> bool: + """Return whether the application is related to S3 via the endpoint.""" + relations = getattr(app_status, "relations", None) or {} + endpoint_relations = relations.get(S3_ENDPOINT, []) + return any(rel.interface == S3_INTERFACE for rel in endpoint_relations) + + +APP_READY_VALIDATION_CHECK: ValidationCheck = ValidationCheck( + name="active", + predicate=_is_app_active, + forceable=True, +) + +S3_RELATION_VALIDATION_CHECK: ValidationCheck = ValidationCheck( + name="s3-relation", + predicate=_is_related_to_s3, +) + + +# --------------------------------------------------------------------------- +# Backup result parsing +# --------------------------------------------------------------------------- +def _parse_mysql_backups(action_result: dict) -> list[BackupOutcome]: + """Parse MySQL list-backups output table and return finished backup IDs.""" + backups_text = action_result.get("backups") + if backups_text is None: + backups_text = (action_result.get("results") or {}).get("backups") + if not isinstance(backups_text, str): + return [] + + backups: list[BackupOutcome] = [] + for raw_line in backups_text.splitlines(): + line = raw_line.strip() + if not line or line.startswith("backup-id") or set(line) == {"-"}: + continue + columns = [column.strip() for column in line.split("|")] + if len(columns) < 3: + continue + if columns[2].lower() != "finished": + backups.append(BackupOutcome(backup_id=columns[0], success=False)) + continue + backups.append(BackupOutcome(columns[0], success=True)) + return backups + + +def _parse_backup(action_result: dict) -> BackupOutcome | None: + """Parse create-backup output and return the backup ID.""" + backup_id = action_result.get(BACKUP_RESULT_ID_KEY) + if not isinstance(backup_id, str): + return None + return BackupOutcome(backup_id=backup_id, success=True) + + +def _parse_vault_backups(action_result: dict) -> list[BackupOutcome]: + """Parse Vault list-backups output and return backup IDs.""" + raw_backup_ids = action_result.get("backup-ids") + if raw_backup_ids is None or not isinstance(raw_backup_ids, str): + return [] + + try: + parsed = json.loads(raw_backup_ids) + except json.JSONDecodeError: + return [] + if not isinstance(parsed, list): + return [] + return [ + BackupOutcome(backup_id=str(backup_id), success=True) for backup_id in parsed + ] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +def _secondary_unit_from_status(units: list[str], action_result: dict) -> str | None: + """Map a SECONDARY cluster member to a Juju unit name. + + The cluster status reports members by address/label rather than Juju unit + name, so match members flagged SECONDARY back to one of the application's + Juju units by ordinal. + """ + status = action_result.get("status") or {} + topology = status.get("defaultreplicaset", {}).get("topology", {}) + secondary_labels = [ + label + for label, info in topology.items() + if isinstance(info, dict) and info.get("memberrole", "").lower() == "secondary" + ] + if not secondary_labels: + return None + + for label in secondary_labels: + ordinal = label.split(".")[0].rsplit("-", 1)[-1] + for unit in units: + if unit.rsplit("/", 1)[-1] == ordinal: + return unit + return None + + +def _latest_backup(backups: list[BackupOutcome]) -> str | None: + """Return the lexicographically latest successful backup ID, if present.""" + successful = [b for b in backups if b.success] + if not successful: + return None + return sorted(successful, key=lambda b: b.backup_id)[-1].backup_id + + +def _api_app_for_mysql(app_name: str) -> str: + """Return the API application name for a given MySQL application name.""" + if app_name.endswith("-mysql"): + return app_name.replace("-mysql", "", 1) + return app_name + + +def _current_scale(jhelper: JujuHelper, app: str, model: str) -> int: + """Read the current unit count for an application, live.""" + try: + return len(list(jhelper.get_application(app, model).units)) + except (ApplicationNotFoundException, JujuException): + LOG.warning("Could not read current scale for %s, assuming 1", app) + return 1 + + +# --------------------------------------------------------------------------- +# Target resolution +# --------------------------------------------------------------------------- +def _resolve_mysql_backup_target( + jhelper: JujuHelper, + app: str, + model: str, + force: bool, +) -> ActionTarget | None: + """Resolve a MySQL backup target, preferring a secondary (replica) unit. + + Falls back to the leader when no secondary is available. When cluster status + cannot be read, the application is skipped unless ``force`` is set, in which + case the leader is used as a best-effort target. + """ + try: + leader = jhelper.get_leader_unit(app, model) + units = list(jhelper.get_application(app, model).units) + except (LeaderNotFoundException, ApplicationNotFoundException): + LOG.warning("Could not resolve %s, skipping", app) + return None + + try: + result = jhelper.run_action(leader, model, MYSQL_CLUSTER_STATUS_ACTION) + secondary = _secondary_unit_from_status(units, result) + if secondary is not None: + return ActionTarget( + app=app, + unit=secondary, + component=MYSQL_CHARM, + action=BACKUP_ACTION, + ) + except ActionFailedException as e: + if not force: + LOG.warning("Could not resolve backup target for %s, skipping: %s", app, e) + return None + LOG.warning( + "Could not resolve backup target for %s, using leader (--force): %s", + app, + e, + ) + + return ActionTarget( + app=app, + unit=leader, + component=MYSQL_CHARM, + action=BACKUP_ACTION, + ) + + +def _resolve_vault_backup_target( + jhelper: JujuHelper, + app: str, + model: str, + force: bool, +) -> ActionTarget | None: + """Resolve the Vault backup target to the leader unit.""" + try: + leader = jhelper.get_leader_unit(app, model) + except (LeaderNotFoundException, ApplicationNotFoundException): + LOG.warning("Could not resolve %s, skipping", app) + return None + + return ActionTarget( + app=app, + unit=leader, + component=VAULT_CHARM, + action=BACKUP_ACTION, + ) + + +# --------------------------------------------------------------------------- +# Atomic action steps +# --------------------------------------------------------------------------- +class _ActionStep(BaseStep): + """Dispatch an action on an application's leader.""" + + def __init__( + self, + jhelper: JujuHelper, + name: str, + description: str, + app: str, + action_name: str, + force: bool = False, + timeout: int = DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__(name, description) + self.jhelper = jhelper + self.model = model + self.app = app + self.action_name = action_name + self.force = force + self.timeout = timeout + + def run(self, context: StepContext) -> Result: + """Run action on the application's leader.""" + try: + leader = self.jhelper.get_leader_unit(self.app, self.model) + self.jhelper.run_action( + leader, + self.model, + self.action_name, + timeout=self.timeout, + ) + if not self.force: + self.jhelper.wait_until_desired_status( + self.model, + apps=[self.app], + status=[], + agent_status=["idle"], + timeout=self.timeout, + ) + except ( + ActionFailedException, + LeaderNotFoundException, + ModelNotFoundException, + JujuException, + ) as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED) + + +class _PauseAppStep(_ActionStep): + """Pause application API services.""" + + def __init__( + self, + jhelper: JujuHelper, + app: str, + force: bool = False, + timeout=DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__( + jhelper, + "Pause App", + "Pause API container services", + app, + PAUSE_ACTION, + force=force, + timeout=timeout, + model=model, + ) + + +class _ResumeAppStep(_ActionStep): + """Resume application API services.""" + + def __init__( + self, + jhelper: JujuHelper, + app: str, + force: bool = False, + timeout=DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__( + jhelper, + "Resume App", + "Resume API container services", + app, + RESUME_ACTION, + force=force, + timeout=timeout, + model=model, + ) + + +class _ScaleAppStep(BaseStep): + """Scale an application to a target number of units.""" + + def __init__( + self, + jhelper: JujuHelper, + application: str, + scale: int, + force: bool = False, + timeout: int = DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ) -> None: + super().__init__("Scale App", f"Scaling {application} to {scale} unit(s)") + self.jhelper = jhelper + self.application = application + self.scale = scale + self.timeout = timeout + self.model = model + self.force = force + + def run(self, context: StepContext) -> Result: + """Scale the application and wait for it to settle.""" + try: + self.jhelper.scale_application(self.model, self.application, self.scale) + self.jhelper.wait_until_active( + self.model, apps=[self.application], timeout=self.timeout + ) + except (JujuException, JujuWaitException, TimeoutError) as e: + return Result(ResultType.FAILED, str(e)) + return Result(ResultType.COMPLETED) + + +class _RestoreAppStep(BaseStep): + """Restore a single application from a backup (atomic restore action).""" + + def __init__( + self, + jhelper: JujuHelper, + component: "BackupComponent", + target: ActionTarget, + restore_to_time: str | None = None, + force: bool = False, + timeout: int = DEFAULT_RESTORE_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Restore app", f"Restoring {target.app}") + self.jhelper = jhelper + self.target = target + self.restore_to_time = restore_to_time + self.component = component + self.model = model + self.timeout = timeout + self.force = force + + def run(self, context: StepContext) -> Result: + """Restore an app using latest backup or restore-to-time.""" + try: + leader = self.jhelper.get_leader_unit(self.target.app, self.model) + except (LeaderNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + params: dict[str, str | bool] = {"force": True} if self.force else {} + restore_to_time_param = self.component.restore_to_time_param + + if self.restore_to_time is not None and restore_to_time_param is not None: + params[restore_to_time_param] = self.restore_to_time + else: + try: + list_result = self.jhelper.run_action( + leader, + self.model, + self.component.list_action, + timeout=self.timeout, + ) + except (ActionFailedException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + latest = _latest_backup(self.component.parse_backup_list(list_result)) + if latest is None: + return Result( + ResultType.FAILED, + f"No finished backups found for {self.target.app}.", + ) + params[self.component.backup_id_param] = latest + + try: + self.jhelper.run_action( + leader, + self.model, + self.component.restore_action, + params, + timeout=self.timeout, + ) + except (ActionFailedException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED) + + +class _BackupAppStep(BaseStep): + """Dispatch a single application's backup action and capture the outcome.""" + + def __init__( + self, + jhelper: JujuHelper, + component: "BackupComponent", + target: ActionTarget, + force: bool = False, + timeout: int = DEFAULT_BACKUP_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Backup app", f"Backing up {target.app}") + self.jhelper = jhelper + self.component = component + self.target = target + self.force = force + self.timeout = timeout + self.model = model + self.result: BackupResult | None = None + + def run(self, context: StepContext) -> Result: + """Dispatch the backup action, recording the outcome on ``self.result``.""" + target = self.target + params: dict[str, str | bool] = {"force": True} if self.force else {} + + try: + action_result = self.jhelper.run_action( + target.unit, + self.model, + target.action, + params, + timeout=self.timeout, + ) + backup = self.component.parse_backup(action_result) + self.result = BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + backup=backup, + ) + except Exception as e: + message = str(e) + in_progress = "timed out waiting for results" in message.lower() + self.result = BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + backup=BackupOutcome(backup_id="", success=None) + if in_progress + else None, + error=message, + ) + + return Result(ResultType.COMPLETED, self.result) + + +class _CheckPauseResumeSupportStep(BaseStep): + """Validate pause/resume action support for a single application.""" + + def __init__( + self, + jhelper: JujuHelper, + app: str, + force: bool = False, + timeout: int = DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__( + "Check pause/resume support", + "Validating pause/resume support before restore", + ) + self.jhelper = jhelper + self.app = app + self.model = model + self.force = force + self.timeout = timeout + + def run(self, context: StepContext) -> Result: + """Fail fast if the application does not support pause/resume actions.""" + try: + actions = self.jhelper.get_application_actions(self.app, self.model) + except (JujuException, ModelNotFoundException): + return Result( + ResultType.FAILED, + f"Unable to query actions for {self.app}. No changes have been made.", + ) + + if PAUSE_ACTION not in actions or RESUME_ACTION not in actions: + if not self.force: + return Result( + ResultType.FAILED, + f"Control-plane application {self.app} does not support the " + "'pause/resume' action required for restore. " + "No changes have been made.", + ) + else: + LOG.warning( + "Control-plane application %s does not support pause/resume" + ", proceeding (--force).", + self.app, + ) + + return Result(ResultType.COMPLETED) + + +# --------------------------------------------------------------------------- +# Per-component plan builders +# --------------------------------------------------------------------------- +def _build_mysql_backup_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + force: bool, + timeout: int, + model: str, +) -> list[BaseStep]: + """Build the MySQL backup plan: a single create-backup action.""" + return [ + _BackupAppStep( + jhelper, component, target, force=force, timeout=timeout, model=model + ), + ] + + +def _build_vault_backup_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + force: bool, + timeout: int, + model: str, +) -> list[BaseStep]: + """Build the Vault backup plan: a single create-backup action.""" + return [ + _BackupAppStep( + jhelper, component, target, force=force, timeout=timeout, model=model + ), + ] + + +def _build_mysql_restore_precheck_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + force: bool, + timeout: int, + model: str, +) -> list[BaseStep]: + """Precheck that the MySQL app's API app supports pause/resume.""" + api_app = _api_app_for_mysql(target.app) + return [ + _CheckPauseResumeSupportStep( + jhelper, api_app, force=force, timeout=timeout, model=model + ) + ] + + +def _build_mysql_restore_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + restore_to_time: str | None, + force: bool, + timeout: int, + model: str, +) -> list[BaseStep]: + """Build the MySQL restore plan: pause, scale down, restore, scale up, resume.""" + api_app = _api_app_for_mysql(target.app) + original_scale = _current_scale(jhelper, target.app, model) + + return [ + _PauseAppStep(jhelper, api_app, force=force, timeout=timeout, model=model), + _ScaleAppStep( + jhelper, target.app, 1, force=force, timeout=timeout, model=model + ), + _RestoreAppStep( + jhelper, + component, + target, + restore_to_time=restore_to_time, + force=force, + timeout=timeout, + model=model, + ), + _ScaleAppStep( + jhelper, + target.app, + original_scale, + force=force, + timeout=timeout, + model=model, + ), + _ResumeAppStep( + jhelper, + api_app, + force=force, + timeout=timeout, + model=model, + ), + ] + + +def _build_mysql_restore_revert_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + force: bool, + timeout: int, + model: str, +) -> list[BaseStep]: + """Build the revert plan for a failed MySQL restore: scale back up, resume.""" + api_app = _api_app_for_mysql(target.app) + original_scale = _current_scale(jhelper, target.app, model) + + return [ + _ScaleAppStep( + jhelper, + target.app, + original_scale, + force=force, + timeout=timeout, + model=model, + ), + _ResumeAppStep(jhelper, api_app, force=force, timeout=timeout, model=model), + ] + + +def _build_vault_restore_plan( + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + restore_to_time: str | None, + force: bool, + timeout: int, + model: str, +) -> list[BaseStep]: + """Build the Vault restore plan: a single restore step (no pause/scale).""" + return [ + _RestoreAppStep( + jhelper, + component, + target, + restore_to_time=restore_to_time, + force=force, + timeout=timeout, + model=model, + ), + ] + + +# --------------------------------------------------------------------------- +# Public component registry +# --------------------------------------------------------------------------- +BACKUP_COMPONENTS: list[BackupComponent] = [ + BackupComponent( + name=MYSQL_CHARM, + resolve_backup_target=_resolve_mysql_backup_target, + parse_backup_list=_parse_mysql_backups, + parse_backup=_parse_backup, + build_backup_plan=_build_mysql_backup_plan, + build_restore_plan=_build_mysql_restore_plan, + build_restore_revert_plan=_build_mysql_restore_revert_plan, + build_restore_precheck_plan=_build_mysql_restore_precheck_plan, + validate_checks=[APP_READY_VALIDATION_CHECK, S3_RELATION_VALIDATION_CHECK], + ), + BackupComponent( + name=VAULT_CHARM, + resolve_backup_target=_resolve_vault_backup_target, + parse_backup_list=_parse_vault_backups, + parse_backup=_parse_backup, + build_backup_plan=_build_vault_backup_plan, + build_restore_plan=_build_vault_restore_plan, + validate_checks=[APP_READY_VALIDATION_CHECK, S3_RELATION_VALIDATION_CHECK], + restore_action=VAULT_RESTORE_ACTION, + ), +] + + +# --------------------------------------------------------------------------- +# Public top-level wrapper steps +# --------------------------------------------------------------------------- +class DiscoverBackupApplicationsStep(BaseStep): + """Discover applications of every registered backup component in the model.""" + + def __init__( + self, + jhelper: JujuHelper, + model: str = OPENSTACK_MODEL, + components: list[BackupComponent] = BACKUP_COMPONENTS, + ): + super().__init__( + "Discover backup applications", + "Discovering stateful applications to back up", + ) + self.jhelper = jhelper + self.components = components + self.model = model + + def run(self, context: StepContext) -> Result: + """Return a mapping of component name to discovered application names.""" + try: + status = self.jhelper.get_model_status(self.model) + except (ModelNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + discovered: dict[str, list[str]] = {c.name: [] for c in self.components} + for app_name, app_status in status.apps.items(): + charm_name = app_status.charm_name or "" + for component in self.components: + if charm_name == component.name: + discovered[component.name].append(app_name) + + return Result(ResultType.COMPLETED, discovered) + + +class ValidateStep(BaseStep): + """Apply each component's own validation checks to its discovered apps. + + Returns a mapping keyed by component with, per application, the list of + failed check names. Applications with an empty failure list are valid. + """ + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + model: str = OPENSTACK_MODEL, + components: list[BackupComponent] = BACKUP_COMPONENTS, + force: bool = False, + ): + super().__init__("Validate applications", "Validating backup readiness") + self.jhelper = jhelper + self.discovered = discovered + self.model = model + self.components = components + self.force = force + + def run(self, context: StepContext) -> Result: + """Return {'valid': {...}, 'failures': {app: [check names]}}.""" + try: + status = self.jhelper.get_model_status(self.model) + except (ModelNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + valid: dict[str, list[str]] = {} + failures: dict[str, list[str]] = {} + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + valid[component_name] = [] + if component is None: + continue + for app_name in apps: + app_status = status.apps.get(app_name) + failed = self._failed_checks(component, app_status, self.force) + if failed: + failures[app_name] = failed + else: + valid[component_name].append(app_name) + + return Result(ResultType.COMPLETED, {"valid": valid, "failures": failures}) + + @staticmethod + def _failed_checks( + component: BackupComponent, app_status: object | None, force: bool = False + ) -> list[str]: + if app_status is None: + return [check.name for check in component.validate_checks] or ["present"] + return [ + check.name + for check in component.validate_checks + if not (check.forceable and force) and not check.predicate(app_status) + ] + + +class ResolveActionTargetsStep(BaseStep): + """Resolve every discovered application's leader unit for generic actions.""" + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + model: str = OPENSTACK_MODEL, + ): + super().__init__( + "Resolve action targets", + "Resolving units for action targets", + ) + self.jhelper = jhelper + self.discovered = discovered + self.model = model + + def run(self, context: StepContext) -> Result: + """Return resolved targets and unresolved apps across components.""" + targets: list[ActionTarget] = [] + unresolved: list[dict[str, str]] = [] + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + if component is None: + continue + for app in apps: + try: + leader = self.jhelper.get_leader_unit(app, self.model) + except (LeaderNotFoundException, ApplicationNotFoundException): + self.update_status(context, f"skipped {app}") + unresolved.append({"app": app, "component": component.name}) + continue + targets.append( + ActionTarget( + app=app, + unit=leader, + component=component.name, + action=component.list_action, + ) + ) + + return Result( + ResultType.COMPLETED, + { + "targets": targets, + "unresolved": sorted(unresolved, key=lambda item: item["app"]), + }, + ) + + +class RunBackupsStep(BaseStep): + """Resolve targets per component, run each backup plan, and collect results.""" + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + force: bool = False, + timeout: int = DEFAULT_BACKUP_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Run backups", "Dispatching backups across applications") + self.jhelper = jhelper + self.discovered = discovered + self.force = force + self.timeout = timeout + self.model = model + + def _resolve_targets( + self, context: StepContext + ) -> list[tuple[BackupComponent, ActionTarget]]: + """Resolve one backup target per discovered application.""" + resolved: list[tuple[BackupComponent, ActionTarget]] = [] + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + if component is None: + continue + for app in apps: + target = component.resolve_backup_target( + self.jhelper, app, self.model, self.force + ) + if target is None: + self.update_status(context, f"skipped {app}") + continue + resolved.append((component, target)) + return resolved + + def _run_backup_plan( + self, + component: BackupComponent, + target: ActionTarget, + context: StepContext, + ) -> BackupResult: + """Run a component's backup plan and return its BackupResult. + + Any step failure short of the result-bearing BackupAppStep is encoded + as a failed BackupResult for the target. + """ + plan = component.build_backup_plan( + self.jhelper, component, target, self.force, self.timeout, self.model + ) + result: BackupResult | None = None + for step in plan: + step_result = step.run(context) + if isinstance(step, _BackupAppStep) and step.result is not None: + result = step.result + if step_result.result_type == ResultType.FAILED: + return result or BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + error=str(step_result.message), + ) + return result or BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + error="Backup plan produced no result.", + ) + + def run(self, context: StepContext) -> Result: + """Resolve targets, run backup plans concurrently, return results.""" + resolved = self._resolve_targets(context) + if not resolved: + return Result(ResultType.COMPLETED, []) + + results: list[BackupResult] = [] + with ThreadPoolExecutor(max_workers=len(resolved)) as executor: + futures = [ + executor.submit(self._run_backup_plan, component, target, context) + for component, target in resolved + ] + for future in as_completed(futures): + results.append(future.result()) + + return Result(ResultType.COMPLETED, results) + + +class ListBackupsStep(BaseStep): + """Dispatch list-backups actions concurrently and collect inventories.""" + + def __init__( + self, + jhelper: JujuHelper, + targets: list[ActionTarget], + timeout: int = DEFAULT_BACKUP_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("List backups", "Listing backups across applications") + self.jhelper = jhelper + self.targets = targets + self.timeout = timeout + self.model = model + + def _list_one( + self, + target: ActionTarget, + action_name: str, + parse_backups: Callable[[dict], list[BackupOutcome]] | None, + ) -> BackupInventory: + """Dispatch a single list-backups action and parse backup IDs.""" + try: + result = self.jhelper.run_action( + target.unit, self.model, action_name, timeout=self.timeout + ) + backups = parse_backups(result) if parse_backups is not None else [] + return BackupInventory( + app=target.app, + unit=target.unit, + component=target.component, + backups=backups, + ) + except Exception as e: + return BackupInventory( + app=target.app, + unit=target.unit, + component=target.component, + error=str(e), + ) + + def run(self, context: StepContext) -> Result: + """Dispatch list-backups concurrently, returning inventory entries.""" + if not self.targets: + return Result(ResultType.COMPLETED, []) + + inventories: list[BackupInventory] = [] + with ThreadPoolExecutor(max_workers=len(self.targets)) as executor: + futures = [] + for target in self.targets: + component = _component_for(target.component) + if component is None: + continue + futures.append( + executor.submit( + self._list_one, + target, + component.list_action, + component.parse_backup_list, + ) + ) + for future in as_completed(futures): + inventories.append(future.result()) + + return Result(ResultType.COMPLETED, inventories) + + +class RestoreStep(BaseStep): + """Drive every component's restore plan, reverting on failure.""" + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + restore_to_time: str | None = None, + force: bool = False, + timeout: int = DEFAULT_RESTORE_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Restore applications", "Restoring applications from backup") + self.jhelper = jhelper + self.discovered = discovered + self.restore_to_time = restore_to_time + self.timeout = timeout + self.model = model + self.force = force + + def _run_plan(self, plan: list[BaseStep], context: StepContext) -> None: + """Run a plan of steps, raising RuntimeError on the first failure.""" + for step in plan: + result = step.run(context) + if result.result_type == ResultType.FAILED: + raise RuntimeError(result.message) + + def _resolve_targets( + self, context: StepContext + ) -> list[tuple[BackupComponent, ActionTarget]]: + """Resolve one backup target per discovered application.""" + resolved: list[tuple[BackupComponent, ActionTarget]] = [] + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + if component is None: + continue + for app in apps: + target = ActionTarget( + app=app, + unit=self.jhelper.get_leader_unit(app, self.model), + component=component.name, + action=component.restore_action, + ) + resolved.append((component, target)) + return resolved + + def _restore_one( + self, component: BackupComponent, target: ActionTarget, context: StepContext + ) -> RestoreResult: + revert_plan: list[BaseStep] = [] + if component.build_restore_revert_plan is not None: + revert_plan = component.build_restore_revert_plan( + self.jhelper, component, target, self.force, self.timeout, self.model + ) + plan = component.build_restore_plan( + self.jhelper, + component, + target, + self.restore_to_time, + self.force, + self.timeout, + self.model, + ) + try: + self._run_plan(plan, context) + return RestoreResult(app=target.app, component=component.name, success=True) + except Exception as e: + reverted = False + if revert_plan: + try: + self._run_plan(revert_plan, context) + reverted = True + except Exception as revert_error: + LOG.warning("Revert failed for %s: %s", target.app, revert_error) + return RestoreResult( + app=target.app, + component=component.name, + success=False, + error=str(e), + reverted=reverted, + ) + + def run(self, context: StepContext) -> Result: + """Precheck all targets, then restore each sequentially, aggregating.""" + resolved = self._resolve_targets(context) + if not resolved: + return Result(ResultType.COMPLETED, []) + + for component, target in resolved: + if component.build_restore_precheck_plan is None: + continue + precheck = component.build_restore_precheck_plan( + self.jhelper, component, target, self.force, self.timeout, self.model + ) + try: + self._run_plan(precheck, context) + except Exception as e: + return Result(ResultType.FAILED, str(e)) + + results: list[RestoreResult] = [] + for component, target in resolved: + self.update_status(context, f"restoring {target.app}") + results.append(self._restore_one(component, target, context)) + + return Result(ResultType.COMPLETED, results) + + +class WriteBackupManifestStep(BaseStep): + """Write a timestamped manifest of the backup run to the snap share path.""" + + def __init__( + self, + results: list[BackupResult], + dispatched_at: str, + manifest_dir: Path | None = None, + ): + super().__init__("Write backup manifest", "Writing backup manifest") + self.results = results + self.dispatched_at = dispatched_at + self.manifest_dir = manifest_dir + + def run(self, context: StepContext) -> Result: + """Write the manifest and return its path.""" + if self.manifest_dir is not None: + directory = self.manifest_dir + else: + directory = Snap().paths.user_common / BACKUP_MANIFEST_DIR + directory.mkdir(parents=True, exist_ok=True) + + succeeded = sum( + 1 for r in self.results if r.backup is not None and r.backup.success + ) + failed = sum(1 for r in self.results if r.error is not None) + manifest = { + "dispatched_at": self.dispatched_at, + "summary": {"succeeded": succeeded, "failed": failed}, + "results": [asdict(r) for r in self.results], + } + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + path = directory / f"backup-manifest-{timestamp}.yaml" + try: + with path.open("w") as manifest_file: + yaml.safe_dump(manifest, manifest_file, sort_keys=False) + except OSError as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED, str(path)) + + +class WriteBackupInventoryManifestStep(BaseStep): + """Write a timestamped manifest for backup inventory results.""" + + def __init__( + self, + results: list[BackupInventory], + listed_at: str, + manifest_dir: Path | None = None, + ): + super().__init__("Write backup inventory manifest", "Writing backup inventory") + self.results = results + self.listed_at = listed_at + self.manifest_dir = manifest_dir + + def run(self, context: StepContext) -> Result: + """Write the inventory manifest and return its path.""" + if self.manifest_dir is not None: + directory = self.manifest_dir + else: + directory = Snap().paths.user_common / BACKUP_MANIFEST_DIR + directory.mkdir(parents=True, exist_ok=True) + + succeeded = sum( + 1 for r in self.results if r.backups and any(b.success for b in r.backups) + ) + failed = len(self.results) - succeeded + manifest = { + "listed_at": self.listed_at, + "summary": {"succeeded": succeeded, "failed": failed}, + "results": [asdict(r) for r in self.results], + } + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + path = directory / f"backup-inventory-{timestamp}.yaml" + try: + with path.open("w") as manifest_file: + yaml.safe_dump(manifest, manifest_file, sort_keys=False) + except OSError as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED, str(path)) diff --git a/sunbeam-python/tests/functional/local/test_backup.py b/sunbeam-python/tests/functional/local/test_backup.py new file mode 100644 index 000000000..19f92a2fb --- /dev/null +++ b/sunbeam-python/tests/functional/local/test_backup.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Functional smoke test for ``sunbeam backup``. + +Skipped by default: requires a bootstrapped cloud with the ``openstack`` snap +installed and the object-storage backup prerequisite configured. Consistent with +the rest of the hardware/environment-gated functional suite. +""" + +import os + +import pytest + +from .utils import sunbeam_command + +pytestmark = pytest.mark.skipif( + not os.environ.get("SUNBEAM_FUNCTIONAL_BACKUP"), + reason="requires a bootstrapped cloud; set SUNBEAM_FUNCTIONAL_BACKUP to enable", +) + + +def test_backup_smoke(): + """``sunbeam backup --help`` is available and the command runs.""" + output = sunbeam_command("backup --help", capture_output=True) + assert "Create backups" in output diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py new file mode 100644 index 000000000..56e596ed6 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from sunbeam.commands.backup_restore import backup + + +def _app_status(charm_name): + app = Mock() + app.charm_name = charm_name + app.units = {} + app.relations = {} + app.app_status.current = "active" + return app + + +def _model_status(apps): + status = Mock() + status.apps = apps + return status + + +@pytest.fixture +def deployment(): + return Mock() + + +@pytest.fixture +def jhelper(deployment): + jhelper = Mock() + deployment.get_juju_helper.return_value = jhelper + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + return jhelper + + +def _leader_only_cluster_status(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + return {"backup-id": f"backup-{unit.replace('/', '-')}"} + + +def _s3_related(app): + relation = Mock() + relation.interface = "s3" + app.relations = {"s3-parameters": [relation]} + return app + + +class TestBackupCommand: + def test_no_applications(self, deployment, jhelper): + jhelper.get_model_status.return_value = _model_status({}) + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "No applications found to back up. Exiting." in result.output + + def test_all_success(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + jhelper.get_application.return_value = _app_status("mysql-k8s") + jhelper.run_action.side_effect = _leader_only_cluster_status + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "2 succeeded, 0 failed" in result.output + + def test_partial_failure(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + jhelper.get_application.return_value = _app_status("mysql-k8s") + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if unit == "keystone-mysql/0": + raise Exception("backup failed") + return {"backup-id": "backup-vault-0"} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "1 succeeded, 1 failed" in result.output + assert "Warning" in result.output + + def test_all_failed(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + jhelper.get_application.return_value = _app_status("mysql-k8s") + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + raise Exception("backup failed") + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "0 succeeded, 1 failed" in result.output + + def test_unrelated_mysql_is_skipped_and_backup_continues(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") # no s3 + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + jhelper.get_application.return_value = vault + jhelper.run_action.return_value = {"backup-id": "backup-vault-0"} + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "is not ready for backup" in result.output + assert "keystone-mysql" in result.output + + def test_declining_confirmation_aborts_backup( + self, deployment, jhelper, monkeypatch + ): + mysql = _app_status("mysql-k8s") # no s3, will be skipped -> prompt + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + jhelper.get_application.return_value = vault + + monkeypatch.setattr( + "sunbeam.commands.backup_restore.ConfirmQuestion.ask", + lambda self, *a, **k: False, + ) + + result = CliRunner().invoke(backup, obj=deployment) + + assert result.exit_code != 0 + assert "Aborted" in result.output + # No backup action dispatched. + actions = [ + call.args[2] + for call in jhelper.run_action.call_args_list + if len(call.args) > 2 + ] + assert "create-backup" not in actions + + def test_no_supported_apps_left(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") # no s3 + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + assert "No applications remain to back up after validation." in result.output + + def test_non_active_target_app_is_skipped(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + mysql.app_status.current = "blocked" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + assert "active" in result.output + assert "No applications remain to back up after validation." in result.output diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py new file mode 100644 index 000000000..45395182b --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import json +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from sunbeam.commands.backup_restore import list_backups +from sunbeam.core.juju import LeaderNotFoundException + + +def _app_status(charm_name): + app = Mock() + app.charm_name = charm_name + app.units = {} + app.relations = {} + app.app_status.current = "active" + return app + + +def _model_status(apps): + status = Mock() + status.apps = apps + return status + + +def _s3_related(app): + relation = Mock() + relation.interface = "s3" + app.relations = {"s3-parameters": [relation]} + return app + + +@pytest.fixture +def deployment(): + return Mock() + + +@pytest.fixture +def jhelper(deployment): + jhelper = Mock() + deployment.get_juju_helper.return_value = jhelper + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + return jhelper + + +class TestListBackupsCommand: + def test_no_applications(self, deployment, jhelper): + jhelper.get_model_status.return_value = _model_status({}) + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert "No applications found" in result.output + + def test_no_supported_apps_left(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") # no s3 + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + assert "No applications found to list backups from. Exiting." in result.output + + def test_lists_backups_and_writes_manifest(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if unit == "keystone-mysql/0": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return { + "backup-ids": json.dumps(["vault-backup-openstack-2026-07-15-00-03-28"]) + } + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 0, result.output + assert "keystone-mysql" in result.output + assert "vault" in result.output + assert "Backup inventory manifest written to" in result.output + + def test_lists_from_leader_only(self, deployment, jhelper): + """list-backups resolves leaders; no cluster-status is queried.""" + mysql = _s3_related(_app_status("mysql-k8s")) + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + def _run_action(unit, model, action, params=None, timeout=None): + assert action != "get-cluster-status" + assert unit == "keystone-mysql/0" + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 0, result.output + actions = [call.args[2] for call in jhelper.run_action.call_args_list] + assert "get-cluster-status" not in actions + + def test_non_active_target_app_is_skipped(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + mysql.app_status.current = "waiting" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + assert "waiting" in result.output or "active" in result.output + + def test_list_action_failures_exit_2_with_details(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + raise Exception("list failed") + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert "Failed to list backups for" in result.output + assert "keystone-mysql" in result.output + assert "vault" in result.output + + def test_unresolved_targets_exit_2_with_warning(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + + def _leader(app, model): + if app == "vault": + raise LeaderNotFoundException("no leader") + return f"{app}/0" + + def _run_action(unit, model, action, params=None, timeout=None): + if unit == "keystone-mysql/0": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.get_leader_unit.side_effect = _leader + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert ( + "Failed to list backups for vault: Could not resolve target." + in result.output + ) diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py new file mode 100644 index 000000000..603a0727e --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py @@ -0,0 +1,487 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import json +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from sunbeam.commands.backup_restore import restore +from sunbeam.core.juju import JujuException, LeaderNotFoundException +from sunbeam.core.openstack import OPENSTACK_MODEL + + +def _app_status(charm_name): + app = Mock() + app.charm_name = charm_name + app.units = {} + app.relations = {} + app.app_status.current = "active" + return app + + +def _model_status(apps): + status = Mock() + status.apps = apps + return status + + +def _s3_related(app): + relation = Mock() + relation.interface = "s3" + app.relations = {"s3-parameters": [relation]} + return app + + +def _default_run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return { + "backup-ids": json.dumps(["vault-backup-openstack-2026-07-15-00-03-28"]) + } + return {} + + +@pytest.fixture +def deployment(): + return Mock() + + +@pytest.fixture +def jhelper(deployment): + jhelper = Mock() + deployment.get_juju_helper.return_value = jhelper + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + "keystone-k8s": _app_status("keystone-k8s"), + } + ) + jhelper.get_application.return_value = _app_status("mysql-k8s") + jhelper.get_application_actions.return_value = ["pause", "resume"] + jhelper.run_action.side_effect = _default_run_action + return jhelper + + +class TestRestoreCommand: + def test_stops_at_pause_guard_and_is_non_destructive(self, deployment, jhelper): + jhelper.get_application_actions.return_value = [] + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "pause/resume" in result.output + jhelper.scale_application.assert_not_called() + + def test_prechecks_pause_resume_for_all_apps_before_any_restore_work( + self, deployment, jhelper + ): + mysql_a = _s3_related(_app_status("mysql-k8s")) + mysql_b = _s3_related(_app_status("mysql-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql_a, "nova-mysql": mysql_b} + ) + + def _get_actions(app, model): + if app == "nova": + return [] + return ["pause", "resume"] + + jhelper.get_application_actions.side_effect = _get_actions + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "pause/resume" in result.output + assert "nova" in result.output + jhelper.scale_application.assert_not_called() + + restore_actions = { + call.args[2] + for call in jhelper.run_action.call_args_list + if len(call.args) > 2 + } + assert "pause" not in restore_actions + assert "resume" not in restore_actions + assert "restore" not in restore_actions + + def test_invalid_restore_to_time_fails_fast(self, deployment, jhelper): + result = CliRunner().invoke( + restore, ["--restore-to-time", "not-a-date"], obj=deployment + ) + + assert result.exit_code != 0 + assert "YYYY-MM-DD HH:MM:SS" in result.output + jhelper.get_model_status.assert_not_called() + jhelper.scale_application.assert_not_called() + + def test_unrelated_mysql_is_skipped_and_restore_continues( + self, deployment, jhelper + ): + mysql = _app_status("mysql-k8s") # no s3 + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + "keystone-k8s": _app_status("keystone-k8s"), + } + ) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "is not ready for backup" in result.output + jhelper.scale_application.assert_not_called() + + def test_no_supported_apps_left(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") # no s3 + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert ( + "No applications remain to restore after validation. Exiting." + in result.output + ) + + def test_unrelated_vault_is_skipped_and_restore_continues( + self, deployment, jhelper + ): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _app_status("vault-k8s") # no s3 + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + "keystone-k8s": _app_status("keystone-k8s"), + } + ) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "vault is not ready for backup" in result.output + + def test_warns_on_pitr_for_vault(self, deployment, jhelper): + result = CliRunner().invoke( + restore, + ["--restore-to-time", "2026-07-15 00:00:00", "--no-prompt"], + obj=deployment, + ) + + assert result.exit_code == 0, result.output + + def test_no_backups_found(self, deployment, jhelper): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | failed" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return {"backup-ids": json.dumps([])} + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert ( + "No applications remain to restore after validation. Exiting." + in result.output + ) + + def test_warns_when_some_backups_failed_but_restore_continues( + self, deployment, jhelper + ): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | failed\n" + "2026-07-14T00:00:00Z | physical | finished" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return { + "backup-ids": json.dumps( + ["vault-backup-openstack-2026-07-15-00-03-28"] + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "Some backups for keystone-mysql failed." in result.output + assert "Only successful backups will be" in result.output + assert "considered for restore" in result.output + + def test_partial_backup_failures_prompt_and_decline_aborts( + self, deployment, jhelper, monkeypatch + ): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | failed\n" + "2026-07-14T00:00:00Z | physical | finished" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return { + "backup-ids": json.dumps( + ["vault-backup-openstack-2026-07-15-00-03-28"] + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + + monkeypatch.setattr( + "sunbeam.commands.backup_restore.ConfirmQuestion.ask", + lambda self, *a, **k: False, + ) + + result = CliRunner().invoke(restore, obj=deployment) + + assert result.exit_code == 1, result.output + assert "Aborted" in result.output + + def test_inventory_lookup_failures_are_reported_and_exit_2( + self, deployment, jhelper + ): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + raise Exception("list failed") + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "Failed to list backups for" in result.output + + def test_force_proceeds_when_target_app_is_inactive(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + mysql.app_status.current = "blocked" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--force", "--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + restore_calls = [ + call + for call in jhelper.run_action.call_args_list + if call.args[2] == "restore" + ] + assert restore_calls + + def test_force_does_not_bypass_inventory_target_resolution_failure( + self, deployment, jhelper + ): + mysql = _s3_related(_app_status("mysql-k8s")) + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + leader_calls = {"keystone-mysql": 0} + + def _leader(app, model): + if app != "keystone-mysql": + return f"{app}/0" + leader_calls[app] += 1 + if leader_calls[app] == 1: + raise LeaderNotFoundException("temporary leader lookup failure") + return "keystone-mysql/0" + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.get_leader_unit.side_effect = _leader + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--force", "--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert ( + "Failed to list backups for keystone-mysql: Could not resolve target." + in result.output + ) + assert ( + "No applications remain to restore after validation. Exiting." + in result.output + ) + assert not any( + call.args[2] == "restore" for call in jhelper.run_action.call_args_list + ) + + def test_non_active_target_app_is_skipped(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + mysql.app_status.current = "error" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + + def test_mysql_restore_failure_reverts_and_reports(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + mysql.units = {"keystone-mysql/0": Mock(), "keystone-mysql/1": Mock()} + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + jhelper.get_application.return_value = mysql + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + if action == "restore": + raise JujuException("restore failed") + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + assert "restore failed" in result.output + # scale down to 1 then revert back up to 2 + assert jhelper.scale_application.call_args_list == [ + ((OPENSTACK_MODEL, "keystone-mysql", 1),), + ((OPENSTACK_MODEL, "keystone-mysql", 2),), + ] diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py new file mode 100644 index 000000000..c894f9760 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py @@ -0,0 +1,615 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import json +from unittest.mock import Mock + +from sunbeam.core.common import ResultType +from sunbeam.core.juju import ( + ActionFailedException, + LeaderNotFoundException, + ModelNotFoundException, +) +from sunbeam.steps.backup_restore import ( + BACKUP_COMPONENTS, + MYSQL_CHARM, + S3_ENDPOINT, + S3_INTERFACE, + VAULT_CHARM, + ActionTarget, + BackupComponent, + BackupInventory, + BackupOutcome, + BackupResult, + DiscoverBackupApplicationsStep, + ListBackupsStep, + ResolveActionTargetsStep, + RunBackupsStep, + ValidateStep, + WriteBackupInventoryManifestStep, + WriteBackupManifestStep, + _BackupAppStep, + _build_vault_backup_plan, + _build_vault_restore_plan, + _component_for, + _parse_backup, + _parse_mysql_backups, + _parse_vault_backups, + _resolve_mysql_backup_target, + _resolve_vault_backup_target, +) + + +def _app_status(charm_name, units=None, relations=None): + app = Mock() + app.charm_name = charm_name + app.units = units or {} + app.relations = relations or {} + app.app_status.current = "active" + return app + + +def _model_status(apps): + status = Mock() + status.apps = apps + return status + + +def _cluster_status(secondary_ordinal): + topology = { + "mysql-0.mysql-endpoints": {"memberrole": "PRIMARY"}, + f"mysql-{secondary_ordinal}.mysql-endpoints": {"memberrole": "SECONDARY"}, + } + return {"status": {"defaultreplicaset": {"topology": topology}}} + + +class TestBackupResult: + def test_construction_with_error_does_not_raise(self): + result = BackupResult( + app="keystone-mysql", + unit="keystone-mysql/0", + component=MYSQL_CHARM, + error="boom", + ) + assert result.error == "boom" + assert result.backup is None + + +class TestRegistry: + def test_registry_contains_mysql_and_vault(self): + names = {c.name for c in BACKUP_COMPONENTS} + assert names == {MYSQL_CHARM, VAULT_CHARM} + + def test_components_have_restore_plans(self): + for component in BACKUP_COMPONENTS: + assert component.build_restore_plan is not None + + +class TestResolveMySQLTarget: + def test_picks_secondary_unit(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.run_action.return_value = _cluster_status(1) + app = _app_status( + "mysql-k8s", units={"keystone-mysql/0": Mock(), "keystone-mysql/1": Mock()} + ) + jhelper.get_application.return_value = app + + target = _resolve_mysql_backup_target( + jhelper, "keystone-mysql", "openstack", force=False + ) + + assert target is not None + assert target.unit == "keystone-mysql/1" + assert target.component == MYSQL_CHARM + + def test_falls_back_to_leader_when_no_secondary(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "cinder-mysql/0" + jhelper.run_action.return_value = { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + jhelper.get_application.return_value = _app_status( + "mysql-k8s", units={"cinder-mysql/0": Mock()} + ) + + target = _resolve_mysql_backup_target( + jhelper, "cinder-mysql", "openstack", force=False + ) + + assert target is not None + assert target.unit == "cinder-mysql/0" + + def test_skips_on_action_failure_without_force(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.get_application.return_value = _app_status( + "mysql-k8s", units={"keystone-mysql/0": Mock()} + ) + jhelper.run_action.side_effect = ActionFailedException("failed") + + target = _resolve_mysql_backup_target( + jhelper, "keystone-mysql", "openstack", force=False + ) + + assert target is None + + def test_uses_leader_on_action_failure_with_force(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.get_application.return_value = _app_status( + "mysql-k8s", units={"keystone-mysql/0": Mock()} + ) + jhelper.run_action.side_effect = ActionFailedException("failed") + + target = _resolve_mysql_backup_target( + jhelper, "keystone-mysql", "openstack", force=True + ) + + assert target is not None + assert target.unit == "keystone-mysql/0" + + def test_skips_when_no_leader(self): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = LeaderNotFoundException("no leader") + + target = _resolve_mysql_backup_target( + jhelper, "keystone-mysql", "openstack", force=True + ) + + assert target is None + + +class TestResolveVaultTarget: + def test_resolves_leader(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "vault/0" + jhelper.get_application.return_value = _app_status( + "vault-k8s", units={"vault/0": Mock()} + ) + + target = _resolve_vault_backup_target( + jhelper, "vault", "openstack", force=False + ) + + assert target is not None + assert target.unit == "vault/0" + assert target.component == VAULT_CHARM + + def test_skips_when_no_leader(self): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = LeaderNotFoundException("no leader") + + target = _resolve_vault_backup_target( + jhelper, "vault", "openstack", force=False + ) + + assert target is None + + +class TestDiscoverBackupApplicationsStep: + def test_discovers_by_charm_name(self, step_context): + jhelper = Mock() + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": _app_status("mysql-k8s"), + "nova-mysql": _app_status("mysql-k8s"), + "vault": _app_status("vault-k8s"), + "keystone": _app_status("keystone-k8s"), + } + ) + + result = DiscoverBackupApplicationsStep(jhelper).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert sorted(result.message[MYSQL_CHARM]) == ["keystone-mysql", "nova-mysql"] + assert result.message[VAULT_CHARM] == ["vault"] + + def test_fails_on_model_error(self, step_context): + jhelper = Mock() + jhelper.get_model_status.side_effect = ModelNotFoundException("missing") + + result = DiscoverBackupApplicationsStep(jhelper).run(step_context) + + assert result.result_type == ResultType.FAILED + + def test_non_active_app_is_still_discovered(self, step_context): + """Discovery is state-agnostic; validation filters non-active apps.""" + jhelper = Mock() + mysql = _app_status("mysql-k8s") + mysql.app_status.current = "blocked" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = DiscoverBackupApplicationsStep(jhelper).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message[MYSQL_CHARM] == ["keystone-mysql"] + + +class TestResolveActionTargetsStep: + def test_resolves_leaders_and_skips_unresolvable(self, step_context): + jhelper = Mock() + + def _leader(app, model): + if app == "broken-mysql": + raise LeaderNotFoundException("no leader") + return f"{app}/0" + + jhelper.get_leader_unit.side_effect = _leader + + discovered = { + MYSQL_CHARM: ["keystone-mysql", "broken-mysql"], + VAULT_CHARM: ["vault"], + } + result = ResolveActionTargetsStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + apps = {t.app for t in result.message["targets"]} + assert apps == {"keystone-mysql", "vault"} + assert all(t.unit.endswith("/0") for t in result.message["targets"]) + assert result.message["unresolved"] == [ + {"app": "broken-mysql", "component": MYSQL_CHARM} + ] + + +class TestValidateStep: + def test_partitions_by_active_and_s3(self, step_context): + jhelper = Mock() + s3 = Mock() + s3.interface = S3_INTERFACE + ready = _app_status("mysql-k8s", relations={S3_ENDPOINT: [s3]}) + no_s3 = _app_status("mysql-k8s") + inactive = _app_status("mysql-k8s", relations={S3_ENDPOINT: [s3]}) + inactive.app_status.current = "blocked" + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": ready, + "nova-mysql": no_s3, + "glance-mysql": inactive, + } + ) + + discovered = {MYSQL_CHARM: ["keystone-mysql", "nova-mysql", "glance-mysql"]} + result = ValidateStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message["valid"][MYSQL_CHARM] == ["keystone-mysql"] + assert result.message["failures"]["nova-mysql"] == ["s3-relation"] + assert result.message["failures"]["glance-mysql"] == ["active"] + + def test_missing_app_fails_all_checks(self, step_context): + jhelper = Mock() + jhelper.get_model_status.return_value = _model_status({}) + + discovered = {MYSQL_CHARM: ["keystone-mysql"]} + result = ValidateStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message["valid"][MYSQL_CHARM] == [] + assert "keystone-mysql" in result.message["failures"] + + +class TestBackupAppStep: + def test_success_records_backup_result(self, step_context): + jhelper = Mock() + jhelper.run_action.return_value = {"backup-id": "id-1"} + component = _component_for(MYSQL_CHARM) + target = ActionTarget( + "keystone-mysql", "keystone-mysql/1", MYSQL_CHARM, "create-backup" + ) + + step = _BackupAppStep(jhelper, component, target) + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert step.result is not None + assert step.result.backup is not None + assert step.result.backup.success is True + assert step.result.backup.backup_id == "id-1" + + def test_timeout_is_marked_in_progress(self, step_context): + jhelper = Mock() + jhelper.run_action.side_effect = Exception( + "timed out waiting for results from: unit nova-mysql/0" + ) + component = _component_for(MYSQL_CHARM) + target = ActionTarget( + "nova-mysql", "nova-mysql/0", MYSQL_CHARM, "create-backup" + ) + + step = _BackupAppStep(jhelper, component, target) + step.run(step_context) + + assert step.result is not None + assert step.result.error is not None + assert step.result.backup is not None + assert step.result.backup.success is None + + +class TestRunBackupsStep: + def test_resolves_and_aggregates_mixed_results(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.side_effect = lambda app, model: _app_status( + "mysql-k8s", units={f"{app}/0": Mock(), f"{app}/1": Mock()} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return _cluster_status(1) + if unit == "glance-mysql/1": + raise ActionFailedException("backup failed") + return {"backup-id": f"backup-{unit.replace('/', '-')}"} + + jhelper.run_action.side_effect = _run_action + discovered = { + MYSQL_CHARM: ["keystone-mysql", "glance-mysql"], + VAULT_CHARM: ["vault"], + } + + result = RunBackupsStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + by_app = {r.app: r for r in result.message} + assert by_app["keystone-mysql"].backup is not None + assert by_app["keystone-mysql"].backup.success is True + assert by_app["glance-mysql"].backup is None + assert by_app["glance-mysql"].error is not None + assert by_app["vault"].backup is not None + assert by_app["vault"].backup.success is True + + def test_force_passes_force_param_only_to_mysql(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.side_effect = lambda app, model: _app_status( + "mysql-k8s", units={f"{app}/0": Mock()} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + return {"backup-id": "id"} + + jhelper.run_action.side_effect = _run_action + discovered = {MYSQL_CHARM: ["keystone-mysql"], VAULT_CHARM: ["vault"]} + + RunBackupsStep(jhelper, discovered, force=True).run(step_context) + + params_by_unit = { + call.args[0]: call.args[3] + for call in jhelper.run_action.call_args_list + if call.args[2] == "create-backup" + } + assert params_by_unit["keystone-mysql/0"] == {"force": True} + # vault does not support force, but we still pass it as a param ignored by Juju + assert params_by_unit["vault/0"] == {"force": True} + + def test_timeout_is_marked_as_in_progress(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.side_effect = lambda app, model: _app_status( + "mysql-k8s", units={f"{app}/0": Mock()} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + raise Exception("timed out waiting for results from: unit nova-mysql/0") + + jhelper.run_action.side_effect = _run_action + discovered = {MYSQL_CHARM: ["nova-mysql"]} + + result = RunBackupsStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + backup_result = result.message[0] + assert backup_result.error is not None + assert backup_result.backup is not None + assert backup_result.backup.success is None + + +class TestListBackupsParsing: + def test_parse_mysql_backup_ids_filters_finished_entries(self): + action_result = { + "backups": ( + "backup-id | backup-type | backup-status\n" + "--------------------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished\n" + "2026-07-14T00:00:00Z | physical | failed" + ) + } + + backups = _parse_mysql_backups(action_result) + + assert [b.backup_id for b in backups] == [ + "2026-07-15T00:00:00Z", + "2026-07-14T00:00:00Z", + ] + assert [b.success for b in backups] == [True, False] + + def test_parse_vault_backup_ids_json_array(self): + action_result = { + "backup-ids": json.dumps( + [ + "vault-backup-openstack-2026-07-15-00-03-28", + "vault-backup-openstack-2026-07-14-00-03-28", + ] + ) + } + + backups = _parse_vault_backups(action_result) + + assert [b.backup_id for b in backups] == [ + "vault-backup-openstack-2026-07-15-00-03-28", + "vault-backup-openstack-2026-07-14-00-03-28", + ] + assert all(b.success for b in backups) + + +class TestListBackupsStep: + def test_collects_backup_ids_by_target(self, step_context): + jhelper = Mock() + + def _run_action(unit, model, action, params=None, timeout=None): + assert action == "list-backups" + if unit == "keystone-mysql/0": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return { + "backup-ids": json.dumps(["vault-backup-openstack-2026-07-15-00-03-28"]) + } + + jhelper.run_action.side_effect = _run_action + targets = [ + ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "list-backups" + ), + ActionTarget("vault", "vault/0", VAULT_CHARM, "list-backups"), + ] + + result = ListBackupsStep(jhelper, targets).run(step_context) + + assert result.result_type == ResultType.COMPLETED + by_app = {r.app: r for r in result.message} + assert by_app["keystone-mysql"].error is None + assert [b.backup_id for b in by_app["keystone-mysql"].backups] == [ + "2026-07-15T00:00:00Z" + ] + assert by_app["vault"].error is None + assert [b.backup_id for b in by_app["vault"].backups] == [ + "vault-backup-openstack-2026-07-15-00-03-28" + ] + + def test_collects_errors_without_raising(self, step_context): + jhelper = Mock() + jhelper.run_action.side_effect = Exception("boom") + targets = [ + ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "list-backups" + ) + ] + + result = ListBackupsStep(jhelper, targets).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message[0].backups is None + assert result.message[0].error == "boom" + + +class TestWriteBackupManifestStep: + def test_writes_manifest(self, step_context, tmp_path): + results = [ + BackupResult( + "keystone-mysql", + "keystone-mysql/1", + MYSQL_CHARM, + BackupOutcome("id-1", success=True), + ), + BackupResult("glance-mysql", "glance-mysql/1", MYSQL_CHARM, None, "err"), + ] + step = WriteBackupManifestStep( + results, "2026-04-09T14:22:01+00:00", manifest_dir=tmp_path + ) + + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + written = list(tmp_path.glob("backup-manifest-*.yaml")) + assert len(written) == 1 + import yaml + + data = yaml.safe_load(written[0].read_text()) + assert data["summary"] == {"succeeded": 1, "failed": 1} + assert data["dispatched_at"] == "2026-04-09T14:22:01+00:00" + assert {r["app"] for r in data["results"]} == { + "keystone-mysql", + "glance-mysql", + } + + +class TestWriteBackupInventoryManifestStep: + def test_writes_inventory_manifest(self, step_context, tmp_path): + results = [ + BackupInventory( + app="keystone-mysql", + unit="keystone-mysql/1", + component=MYSQL_CHARM, + backups=[BackupOutcome("2026-07-15T00:00:00Z", success=True)], + ), + BackupInventory( + app="vault", + unit="vault/0", + component=VAULT_CHARM, + error="failed", + ), + ] + step = WriteBackupInventoryManifestStep( + results, "2026-07-15T00:04:28+00:00", manifest_dir=tmp_path + ) + + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + written = list(tmp_path.glob("backup-inventory-*.yaml")) + assert len(written) == 1 + + +class TestExtensibility: + """Adding a component is a registration change, not a code change (FR-021).""" + + def test_new_component_flows_through_generic_pipeline( + self, step_context, monkeypatch + ): + def _resolve_fake(jhelper, app, model, force): + return ActionTarget(app, f"{app}/0", "fake-charm", "create-backup") + + fake = BackupComponent( + name="fake-charm", + resolve_backup_target=_resolve_fake, + parse_backup_list=_parse_vault_backups, + parse_backup=_parse_backup, + build_backup_plan=_build_vault_backup_plan, + build_restore_plan=_build_vault_restore_plan, + ) + components = BACKUP_COMPONENTS + [fake] + monkeypatch.setattr( + "sunbeam.steps.backup_restore.BACKUP_COMPONENTS", components + ) + + jhelper = Mock() + jhelper.get_model_status.return_value = _model_status( + {"my-fake": _app_status("fake-charm")} + ) + jhelper.run_action.return_value = {"backup-id": "fake-backup"} + + discover = DiscoverBackupApplicationsStep(jhelper, components=components).run( + step_context + ) + assert discover.message["fake-charm"] == ["my-fake"] + + run = RunBackupsStep(jhelper, {"fake-charm": ["my-fake"]}).run(step_context) + assert run.message[0].component == "fake-charm" + assert run.message[0].backup is not None + assert run.message[0].backup.success is True + assert run.message[0].backup.backup_id == "fake-backup" diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py new file mode 100644 index 000000000..19481dc35 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py @@ -0,0 +1,222 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import dataclasses +from unittest.mock import Mock + +from sunbeam.core.common import ResultType +from sunbeam.core.juju import JujuException +from sunbeam.steps.backup_restore import ( + MYSQL_CHARM, + VAULT_CHARM, + ActionTarget, + RestoreStep, + _component_for, + _PauseAppStep, + _RestoreAppStep, + _ResumeAppStep, + _ScaleAppStep, +) + + +def _mysql_component(): + return _component_for(MYSQL_CHARM) + + +def _vault_component(): + return _component_for(VAULT_CHARM) + + +class TestGuardedSteps: + def test_pause_dispatches_action_without_is_skip_check(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-k8s/0" + + step = _PauseAppStep(jhelper, app="keystone-k8s") + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + jhelper.get_application_actions.assert_not_called() + jhelper.run_action.assert_called_once() + + def test_resume_dispatches_action(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-k8s/0" + + step = _ResumeAppStep(jhelper, app="keystone-k8s") + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + jhelper.run_action.assert_called_once() + + def test_restore_mysql_uses_latest_backup_id(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.run_action.side_effect = [ + { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + }, + {}, + ] + target = ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore" + ) + result = _RestoreAppStep(jhelper, _mysql_component(), target).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert jhelper.run_action.call_args_list[1].args[2] == "restore" + assert jhelper.run_action.call_args_list[1].args[3] == { + "backup-id": "2026-07-15T00:00:00Z" + } + + def test_restore_mysql_uses_restore_to_time(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + component = dataclasses.replace( + _mysql_component(), restore_to_time_param="restore-to-time" + ) + target = ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore" + ) + + result = _RestoreAppStep( + jhelper, + component, + target, + restore_to_time="2026-07-15 00:00:00", + ).run(step_context) + + assert result.result_type == ResultType.COMPLETED + jhelper.run_action.assert_called_once_with( + "keystone-mysql/0", + "openstack", + "restore", + {"restore-to-time": "2026-07-15 00:00:00"}, + timeout=1800, + ) + + def test_restore_vault_uses_latest_backup(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "vault/0" + jhelper.run_action.side_effect = [ + {"backup-ids": '["vault-backup-openstack-2026-07-15-00-03-28"]'}, + {}, + ] + target = ActionTarget("vault", "vault/0", VAULT_CHARM, "restore-backup") + + result = _RestoreAppStep(jhelper, _vault_component(), target).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert jhelper.run_action.call_args_list[1].args[2] == "restore-backup" + assert jhelper.run_action.call_args_list[1].args[3] == { + "backup-id": "vault-backup-openstack-2026-07-15-00-03-28" + } + + +class TestScaleMySQLStep: + def test_scales_and_waits(self, step_context): + jhelper = Mock() + result = _ScaleAppStep(jhelper, "keystone-mysql", 1).run(step_context) + assert result.result_type == ResultType.COMPLETED + jhelper.scale_application.assert_called_once_with( + "openstack", "keystone-mysql", 1 + ) + jhelper.wait_until_active.assert_called_once() + + def test_returns_failed_on_juju_error(self, step_context): + jhelper = Mock() + jhelper.scale_application.side_effect = JujuException("boom") + result = _ScaleAppStep(jhelper, "keystone-mysql", 1).run(step_context) + assert result.result_type == ResultType.FAILED + + +def _finished_backup_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + +class TestRestoreStepWrapper: + def test_prechecks_all_before_any_restore_work(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.return_value = Mock( + units={"a/0": Mock(), "a/1": Mock()} + ) + jhelper.get_application_actions.return_value = [] # no pause/resume + jhelper.run_action.side_effect = _finished_backup_action + + discovered = {MYSQL_CHARM: ["keystone-mysql"]} + + result = RestoreStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.FAILED + assert "pause/resume" in result.message + jhelper.scale_application.assert_not_called() + + def test_restores_each_target(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.return_value = Mock( + units={"a/0": Mock(), "a/1": Mock()} + ) + jhelper.get_application_actions.return_value = ["pause", "resume"] + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + return {"backup-ids": '["vault-backup-openstack-2026-07-15-00-03-28"]'} + return {} + + jhelper.run_action.side_effect = _run_action + + discovered = {VAULT_CHARM: ["vault"]} + + result = RestoreStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message[0].success is True + + def test_reverts_mysql_on_restore_failure(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.return_value = Mock( + units={"a/0": Mock(), "a/1": Mock()} + ) + jhelper.get_application_actions.return_value = ["pause", "resume"] + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + if action == "restore": + raise JujuException("restore failed") + return {} + + jhelper.run_action.side_effect = _run_action + + discovered = {MYSQL_CHARM: ["keystone-mysql"]} + + result = RestoreStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + outcome = result.message[0] + assert outcome.success is False + assert outcome.reverted is True + # scale down to 1, then revert scale back up + scale_calls = [call.args for call in jhelper.scale_application.call_args_list] + assert ("openstack", "keystone-mysql", 1) in scale_calls