diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 53a4d90c..beb66de3 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -51,3 +51,10 @@ jobs: run: | python3 -m unittest discover -s tests -p test_build_receipt.py -q python3 -m unittest discover -s tests -p test_current_build_policy.py -q + + - name: Test legacy CE recovery gate when changed (no backup) + shell: bash + run: | + if git diff --name-only HEAD^ HEAD -- tools/migrate/cf_recovery.py tests/test_cf_recovery.py | grep -q .; then + python3 -m unittest discover -s tests -p test_cf_recovery.py -q + fi diff --git a/tests/test_cf_recovery.py b/tests/test_cf_recovery.py index 31131358..61cc9985 100644 --- a/tests/test_cf_recovery.py +++ b/tests/test_cf_recovery.py @@ -1,5 +1,8 @@ """Offline safety checks. Synthetic bytes are NOT full Seafile restore acceptance.""" import argparse +import json +import os +import subprocess from pathlib import Path import sys import tempfile @@ -27,7 +30,8 @@ def setUp(self): self.restore = self.root / "restore" def args(self): - return argparse.Namespace(compose_root=str(self.compose), output=str(self.snapshot)) + return argparse.Namespace(compose_root=str(self.compose), output=str(self.snapshot), + legacy_ce_only=True) def test_backup_verify_restore_and_hash_protection(self): with mock.patch.object(cf_recovery, "stopped") as guard: @@ -53,16 +57,124 @@ def test_missing_required_and_symlink_refused(self): with self.assertRaisesRegex(cf_recovery.RecoveryError, "symbolic"): cf_recovery.backup(self.args()) + def test_bulk_backup_requires_explicit_legacy_ce_scope(self): + args = self.args() + args.legacy_ce_only = False + with mock.patch.object(cf_recovery, "stopped") as guard: + with self.assertRaisesRegex(cf_recovery.RecoveryError, "--legacy-ce-only"): + cf_recovery.backup(args) + guard.assert_not_called() + self.assertFalse(self.snapshot.exists()) + + def test_bulk_backup_rejects_auto_local_mode(self): + (self.compose / ".env").write_text( + "PRIVATE_KEY=hidden\nCF_LIBRARY_STORAGE=auto\n") + with mock.patch.object(cf_recovery, "stopped") as guard: + with self.assertRaisesRegex(cf_recovery.RecoveryError, "auto-local"): + cf_recovery.backup(self.args()) + guard.assert_not_called() + + def test_bulk_backup_rejects_managed_library_objects(self): + directory = self.compose / "data/libraries/custom-key/storage/commits" + directory.mkdir(parents=True) + with mock.patch.object(cf_recovery, "stopped") as guard: + with self.assertRaisesRegex(cf_recovery.RecoveryError, "per-library objects"): + cf_recovery.backup(self.args()) + guard.assert_not_called() + + def test_bulk_backup_rejects_persisted_v2_storage_config(self): + conf = self.compose / "data/seafile/seafile/conf/seafile.conf" + conf.parent.mkdir(parents=True) + conf.write_text("[cloudfile_v2]\nauto_local_root=/shared/seafile/library-storage\n") + with mock.patch.object(cf_recovery, "stopped") as guard: + with self.assertRaisesRegex(cf_recovery.RecoveryError, "per-library root"): + cf_recovery.backup(self.args()) + guard.assert_not_called() + + def test_bulk_backup_rejects_persisted_s3_storage_class(self): + conf = self.compose / "data/seafile/seafile/conf/seafile.conf" + conf.parent.mkdir(parents=True) + conf.write_text("[storage]\nenable_storage_classes=true\n") + with mock.patch.object(cf_recovery, "stopped") as guard: + with self.assertRaisesRegex(cf_recovery.RecoveryError, "storage classes"): + cf_recovery.backup(self.args()) + guard.assert_not_called() + + def test_bulk_backup_rejects_s3_environment(self): + (self.compose / ".env").write_text("CF_ENABLE_S3_STORAGE=true\n") + with mock.patch.object(cf_recovery, "stopped") as guard: + with self.assertRaisesRegex(cf_recovery.RecoveryError, "S3/multiple"): + cf_recovery.backup(self.args()) + guard.assert_not_called() + + def test_dotenv_detection_matches_compose_formats(self): + # Docker Compose honours `export`, quoted values and trailing comments. + # This fail-closed gate must read the same value those forms resolve to. + entries = ( + ("export CF_LIBRARY_STORAGE=auto\n", "auto-local"), + ('CF_LIBRARY_STORAGE="auto" # migrated\n', "auto-local"), + ("CF_LIBRARY_STORAGE=auto # trailing note\n", "auto-local"), + ("CF_ENABLE_S3_STORAGE=true # note\n", "S3/multiple"), + ("export COMPOSE_FILE=docker-compose.yml:docker-compose.auto-library-storage.yml\n", + "COMPOSE_FILE"), + ) + for entry, message in entries: + with self.subTest(entry=entry): + (self.compose / ".env").write_text("PRIVATE_KEY=hidden\n" + entry) + with mock.patch.object(cf_recovery, "stopped") as guard: + with self.assertRaisesRegex(cf_recovery.RecoveryError, message): + cf_recovery.backup(self.args()) + guard.assert_not_called() + self.assertFalse(self.snapshot.exists()) + + def test_legacy_dotenv_values_still_accepted(self): + (self.compose / ".env").write_text( + "PRIVATE_KEY=hidden\n" + "COMPOSE_FILE=docker-compose.yml\n" + "CF_ENABLE_S3_STORAGE=false # off\n" + "SEAF_SERVER_STORAGE_TYPE=\n" + "CF_LIBRARY_STORAGE=\n") + with mock.patch.object(cf_recovery, "stopped"): + cf_recovery.backup(self.args()) + self.assertTrue(self.snapshot.exists()) + + def resolved_mounts(self, *, db_source=None): + source = self.compose / "data" + def bind(where, inside): + return {"type": "bind", "source": str(where), "target": inside} + return json.dumps({"services": { + "db": {"volumes": [bind(db_source or source / "db", "/var/lib/mysql")]}, + "cloudfile": {"volumes": [bind(source / "seafile", "/shared")]}, + "cf-worker": {"volumes": [bind(source / "seafile", "/shared")]}, + }}) + def test_running_or_unverifiable_compose_refused(self): + for status, message in ((0, "active-container-id\n"), (1, "")): + with self.subTest(status=status), mock.patch.object(cf_recovery.subprocess, "run") as runner: + def response(command, **kwargs): + if "config" in command: + return subprocess.CompletedProcess(command, 0, self.resolved_mounts(), "") + return subprocess.CompletedProcess(command, status, message, "") + runner.side_effect = response + with self.assertRaisesRegex(cf_recovery.RecoveryError, "running"): + cf_recovery.stopped(self.compose) + + def test_effective_compose_mount_mismatch_refused(self): with mock.patch.object(cf_recovery.subprocess, "run") as runner: - runner.return_value.returncode = 0 - runner.return_value.stdout = "active-container-id\n" - with self.assertRaisesRegex(cf_recovery.RecoveryError, "running"): - cf_recovery.stopped(self.compose) - runner.return_value.stdout = "" - runner.return_value.returncode = 1 - with self.assertRaisesRegex(cf_recovery.RecoveryError, "running"): + def response(command, **kwargs): + if "config" in command: + return subprocess.CompletedProcess(command, 0, self.resolved_mounts( + db_source=self.root / "untracked-db"), "") + return subprocess.CompletedProcess(command, 0, "", "") + runner.side_effect = response + with self.assertRaisesRegex(cf_recovery.RecoveryError, "effective Compose mount"): cf_recovery.stopped(self.compose) + self.assertEqual(1, runner.call_count) + + def test_explicit_compose_override_is_not_silently_ignored(self): + with mock.patch.dict(os.environ, {"COMPOSE_FILE": "docker-compose.yml:other.yml"}): + with self.assertRaisesRegex(cf_recovery.RecoveryError, "custom COMPOSE_FILE"): + cf_recovery.reject_unsupported_bulk_backup(self.compose, True) def test_source_destination_overlap_refused(self): with self.assertRaisesRegex(cf_recovery.RecoveryError, "overlap"): diff --git a/tools/migrate/cf_recovery.py b/tools/migrate/cf_recovery.py index 563ee0bc..013e09af 100644 --- a/tools/migrate/cf_recovery.py +++ b/tools/migrate/cf_recovery.py @@ -9,6 +9,7 @@ import json import os from pathlib import Path +import re import shutil import subprocess import sys @@ -27,12 +28,30 @@ class RecoveryError(Exception): def stopped(compose): if not (compose / "docker-compose.yml").is_file(): raise RecoveryError("compose file is missing") + # Validate the effective Compose mounts, not a guess from .env. + # This legacy copier only understands two local bind sources. try: + resolved = subprocess.run( + ["docker", "compose", "-f", "docker-compose.yml", "config", "--format", "json"], + cwd=compose, text=True, capture_output=True, timeout=30) + if resolved.returncode: + raise RecoveryError("cannot resolve effective Compose mounts") + services = json.loads(resolved.stdout)["services"] + for service, target, source in ( + ("db", "/var/lib/mysql", "data/db"), + ("cloudfile", "/shared", "data/seafile"), + ("cf-worker", "/shared", "data/seafile"), + ): + mounts = [entry for entry in services[service]["volumes"] + if entry.get("target") == target] + if len(mounts) != 1 or mounts[0].get("type") != "bind" or ( + Path(mounts[0]["source"]).resolve() != (compose / source).resolve()): + raise RecoveryError("effective Compose mount differs from legacy backup scope") p = subprocess.run( ["docker", "compose", "-f", "docker-compose.yml", "ps", "--status", "running", "--quiet"], cwd=compose, text=True, capture_output=True, timeout=30) - except (OSError, subprocess.TimeoutExpired) as exc: - raise RecoveryError("cannot verify that all CloudFile services are stopped") from exc + except (OSError, subprocess.TimeoutExpired, ValueError, KeyError, TypeError) as exc: + raise RecoveryError("cannot verify effective mounts or stopped services") from exc if p.returncode != 0 or p.stdout.strip(): raise RecoveryError("CloudFile services may be running; stop ALL writers and retry") @@ -85,7 +104,9 @@ def check_manifest(snapshot): def copy_allowed(source, destination): destination.mkdir(mode=0o700) - for name in KINDS + OPTIONAL: + # A newly created legacy local-only snapshot never copies old S3 blobs. + # OPTIONAL is retained only for restoring historical snapshots verbatim. + for name in KINDS: src = source / name if name in KINDS and not src.is_dir(): raise RecoveryError("required compose volume missing: " + name) @@ -102,10 +123,97 @@ def disjoint(source, destination): raise RecoveryError("destination must not exist") +def dotenv_entry(line): + """Return (name, value) the way Docker Compose reads one ``.env`` line. + + Compose accepts an optional ``export `` prefix, single- or double-quoted + values and trailing ``#`` comments. This legacy gate is a safety check, so + it must not be bypassed by formatting that Compose still honours. + """ + text = line.strip() + if text.startswith("export ") or text.startswith("export\t"): + text = text[len("export"):].lstrip() + name, separator, value = text.partition("=") + if not separator: + return None, None + value = value.strip() + if not value: + return name.strip(), "" + quote = value[0] + if quote in ("'", '"'): + end = value.find(quote, 1) + return name.strip(), (value[1:end] if end != -1 else value[1:]) + # Unquoted values end at whitespace followed by '#'; '#' without preceding + # whitespace stays part of the value (same rule as Compose's dotenv parser). + comment = re.search(r"\s#", value) + if comment: + value = value[:comment.start()].rstrip() + return name.strip(), value + + +def reject_unsupported_bulk_backup(compose, legacy_ce_only): + """This legacy full-instance copier never backs up V2 per-library objects. + + Running this on auto-local could produce a misleading 'successful' archive. + Always require an explicit legacy-only acknowledgement; no implicit scan or + copy of all libraries is permitted. + """ + if not legacy_ce_only: + raise RecoveryError("whole-instance backup requires --legacy-ce-only; " + "use seafile-library-offline.py for a selected repo_id") + # -f docker-compose.yml below intentionally supports only the baseline. + # Refuse custom project overlays rather than quietly copy the wrong paths. + requested_compose = os.environ.get("COMPOSE_FILE", "") + if requested_compose and requested_compose not in ("docker-compose.yml", "./docker-compose.yml"): + raise RecoveryError("custom COMPOSE_FILE cannot use legacy backup") + managed = compose / "data/libraries" + if managed.is_symlink() or (managed.is_dir() and any(managed.iterdir())): + raise RecoveryError("V2 per-library objects are outside the legacy backup scope") + if os.environ.get("CF_LIBRARY_STORAGE") == "auto": + raise RecoveryError("CF_LIBRARY_STORAGE=auto is not a legacy CE installation") + if os.environ.get("SEAF_SERVER_STORAGE_TYPE") == "multiple" or os.environ.get("CF_ENABLE_S3_STORAGE", "").lower() == "true": + raise RecoveryError("S3/multiple storage cannot use legacy local-only backup") + environment = compose / ".env" + if environment.is_symlink(): + raise RecoveryError("compose .env must not be a symbolic link") + if environment.is_file(): + for entry in environment.read_text(encoding="utf-8").splitlines(): + name, value = dotenv_entry(entry) + if name is None: + continue + if name == "COMPOSE_FILE" and value not in ("", "docker-compose.yml", "./docker-compose.yml"): + raise RecoveryError("custom COMPOSE_FILE cannot use legacy backup") + if name == "CF_LIBRARY_STORAGE" and value == "auto": + raise RecoveryError("persisted V2 auto-local mode is not a legacy CE installation") + if ((name == "SEAF_SERVER_STORAGE_TYPE" and value == "multiple") or + (name == "CF_ENABLE_S3_STORAGE" and value.lower() == "true")): + raise RecoveryError("persisted S3/multiple storage is not a legacy CE backup") + # Known CE14 Compose config locations. Unrecognized external data roots + # are outside the contract of this explicitly acknowledged legacy tool. + for relative in ("data/seafile/seafile/conf/seafile.conf", + "data/seafile/conf/seafile.conf"): + conf = compose / relative + if conf.is_file(): + from configparser import ConfigParser, Error as ConfigError + cfg = ConfigParser(interpolation=None) + try: + cfg.read(conf) + if cfg.has_option("cloudfile_v2", "auto_local_root"): + raise RecoveryError("V2 per-library root is not in the legacy backup scope") + if cfg.getboolean("storage", "enable_storage_classes", fallback=False): + raise RecoveryError("storage classes are not supported by legacy local-only backup") + for section in ("commit_object_backend", "fs_object_backend", "block_backend"): + if cfg.get(section, "name", fallback="") in ("s3", "multiple"): + raise RecoveryError("S3/multiple backend is not a legacy local-only backup") + except (ValueError, ConfigError) as exc: + raise RecoveryError("invalid Seafile storage config: " + str(exc)) from exc + + def backup(args): compose = Path(args.compose_root).resolve(strict=True) dest = Path(args.output).absolute() disjoint(compose, dest) + reject_unsupported_bulk_backup(compose, getattr(args, 'legacy_ce_only', False)) stopped(compose) dest.parent.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix=".cf-snapshot-", dir=dest.parent) as tmp: @@ -157,6 +265,8 @@ def main(argv=None): b.add_argument("--compose-root", required=True) b.add_argument("--output", required=True) b.add_argument("--ack-writers-stopped", action="store_true", required=True) + b.add_argument("--legacy-ce-only", action="store_true", required=True, + help="explicitly acknowledge this is a full-instance legacy CE volume snapshot") b.set_defaults(func=backup) v = commands.add_parser("verify") v.add_argument("--snapshot", required=True)