From aa0eaf2ad2110acd6d9f4aba03c8bc950b70c270 Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 16:40:26 +0900 Subject: [PATCH 01/14] feat(storage): inspect and explicitly export one Seafile library offline without CloudFile services --- tools/ops/seafile-library-offline.py | 259 +++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 tools/ops/seafile-library-offline.py diff --git a/tools/ops/seafile-library-offline.py b/tools/ops/seafile-library-offline.py new file mode 100644 index 00000000..55bee82f --- /dev/null +++ b/tools/ops/seafile-library-offline.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Read-only, service-independent Seafile library discovery and explicit export. + +A library is stored in the standard Seafile object layout under either the +native data root or one managed per-library directory. No CloudFile SQL or Hub +API is consulted. Logical files are exported by the original seaf-fsck binary; +this tool never interprets Seafile Commit/FS/Block contents itself. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import sys +import tempfile +from uuid import UUID + +KINDS = ("commits", "fs", "blocks") +KEY = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,78}[a-z0-9])?\Z") + + +class LibraryError(Exception): + pass + + +def repository_id(raw): + try: + value = UUID(raw) + except (TypeError, ValueError, AttributeError) as exc: + raise LibraryError("--repo-id must be a complete UUID") from exc + if str(value) != raw: + raise LibraryError("--repo-id must be a canonical lowercase UUID") + return raw + + +def real_dir(path, *, optional=False): + """Keep managed children strictly physical, but allow native CE root aliases.""" + path = Path(path) + if not path.is_absolute(): + raise LibraryError("storage roots must be absolute") + if optional and not path.exists() and not path.is_symlink(): + return None + try: + root = path.resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise LibraryError("storage root is not accessible") from exc + if not root.is_dir(): + raise LibraryError("storage root must be a directory") + return root + + +def physical_dir(path): + """Return true only for a real directory, never a symlink (no mutation).""" + try: + return stat.S_ISDIR(path.lstat().st_mode) + except FileNotFoundError: + return False + + +def candidate(root, repo_id, backend, key=None): + if root is None: + return None + base = root / "storage" + if not physical_dir(base): + return None + commit_parent = base / "commits" + if not physical_dir(commit_parent): + return None + commits = commit_parent / repo_id + if not physical_dir(commits): + return None + paths = {} + for kind in KINDS: + parent = base / kind + if parent.exists() or parent.is_symlink(): + if not physical_dir(parent): + raise LibraryError("unsafe Seafile object parent for " + kind) + path = parent / repo_id + if path.exists() or path.is_symlink(): + if not physical_dir(path): + raise LibraryError("unsafe Seafile library object directory for " + kind) + paths[kind] = str(path) + else: + paths[kind] = None # A newly created/empty library can omit fs/blocks. + return {"repo_id": repo_id, "backend": backend, "repo_key": key, + "seafile_data_root": str(root), "object_dirs": paths} + + +def find_library(repo_id, native_root, managed_root): + repo_id = repository_id(repo_id) + native = real_dir(native_root) + managed = real_dir(managed_root, optional=True) + if managed and Path(managed_root).is_symlink(): + raise LibraryError("managed parent must not be a symbolic link") + found = [] + item = candidate(native, repo_id, "seafile-native") + if item is not None: + found.append(item) + if managed is not None: + for child in sorted(managed.iterdir()): + if not KEY.fullmatch(child.name): + continue + if not physical_dir(child): + continue # Never traverse links masquerading as libraries. + item = candidate(child, repo_id, "auto-local", child.name) + if item is not None: + found.append(item) + if not found: + raise LibraryError("repo_id was not found in the native or managed object directories") + if len(found) != 1: + raise LibraryError("ambiguous repo_id in multiple physical stores; inspect manually") + return found[0] + + +def output_directory(raw, sources): + target = Path(raw) + if not target.is_absolute(): + raise LibraryError("export destination must be an absolute path") + if target.exists() or target.is_symlink(): + raise LibraryError("export destination already exists; overwrite is not supported") + parent = target.parent.resolve(strict=True) + if not parent.is_dir() or not physical_dir(parent): + raise LibraryError("export destination parent must be a real existing directory") + dest = parent / target.name + for root in sources: + source = Path(root).resolve() + if source == dest or source in dest.parents or dest in source.parents: + raise LibraryError("export directory must not overlap an object storage root") + return dest + + +def export_library(args): + item = find_library(args.repo_id, args.seafile_data, args.library_root) + fsck = Path(args.seaf_fsck) + if not fsck.is_absolute() or not fsck.is_file() or not os.access(fsck, os.X_OK): + raise LibraryError("--seaf-fsck must name an executable Seafile fsck binary") + target = output_directory(args.output_dir, (args.seafile_data, args.library_root)) + # Temp staging lives on the destination volume, not in /tmp and not in the + # object store. Failed export is deleted; no target is published on error. + with tempfile.TemporaryDirectory(prefix=".seafile-export-", dir=target.parent) as temporary: + sandbox = Path(temporary) + data = sandbox / "seafile-data" + conf = sandbox / "conf" + out = sandbox / "out" + conf.mkdir() + (data / "tmpfiles").mkdir(parents=True) + out.mkdir() + (conf / "seafile.conf").write_text( + "[commit_object_backend]\nname = filesystem\n" + "[fs_object_backend]\nname = filesystem\n" + "[block_backend]\nname = filesystem\n", encoding="utf-8") + for kind in KINDS: + parent = data / "storage" / kind + parent.mkdir(parents=True) + original = item["object_dirs"][kind] + target_dir = parent / item["repo_id"] + if original is not None: + target_dir.symlink_to(original, target_is_directory=True) + else: + target_dir.mkdir() # No source-tree writes for an empty kind. + # Stock Seafile CE filesystem routing, without cloudfile_v2 or a DB. + command = [str(fsck), "-c", str(conf), "-d", str(data), "-F", str(conf), + "--export", str(out), item["repo_id"]] + result = subprocess.run(command, cwd=sandbox, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, errors="replace") + if result.returncode != 0: + raise LibraryError("seaf-fsck export failed: " + result.stdout[-1500:]) + children = list(out.iterdir()) + if len(children) != 1 or not physical_dir(children[0]) or \ + not children[0].name.startswith(item["repo_id"][:8] + "_"): + raise LibraryError("seaf-fsck did not produce exactly one library export " + "(encrypted, empty/corrupt or unsupported library)") + # Only the requested repo's result is published. No source bytes are + # copied by inspect and no bulk backup is ever implicit. + if target.exists() or target.is_symlink(): + raise LibraryError("export destination was created concurrently") + target.mkdir(mode=0o700) # Atomic no-overwrite reservation. + try: + children[0].rename(target / children[0].name) + except Exception: + target.rmdir() + raise + return {"action": "export", "repo_id": item["repo_id"], + "backend": item["backend"], "export_dir": str(target), + "status": "EXPORTED_UNVERIFIED", + "warning": "Export is not a database backup or proof of full source integrity; " + "run verify-export explicitly when needed."} + + +def verify_export(args): + repo_id = repository_id(args.repo_id) + root = real_dir(args.output_dir) + libraries = list(root.iterdir()) + if len(libraries) != 1 or not physical_dir(libraries[0]) or \ + not libraries[0].name.startswith(repo_id[:8] + "_"): + raise LibraryError("export directory must contain one library matching repo_id") + base = libraries[0] + count = 0 + total = 0 + digest = hashlib.sha256() if args.hash else None + for here, directories, names in os.walk(base, followlinks=False): + for name in directories: + if not physical_dir(Path(here) / name): + raise LibraryError("export contains symlink or unsupported directory") + for name in sorted(names): + p = Path(here) / name + if not stat.S_ISREG(p.lstat().st_mode): + raise LibraryError("export contains symlink or non-regular file") + count += 1 + total += p.stat().st_size + if digest is not None: + relative = p.relative_to(base).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(4, "big") + relative) + filehash = hashlib.sha256() + with p.open("rb") as stream: + for piece in iter(lambda: stream.read(1024 * 1024), b""): + filehash.update(piece) + digest.update(filehash.digest()) + return {"action": "verify-export", "repo_id": repo_id, "export_dir": str(root), + "file_count": count, "total_bytes": total, "tree_sha256": digest.hexdigest() if digest else None, + "note": "Local export inventory and readability only; source completeness is not proven."} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="action", required=True) + for op in ("inspect", "export", "verify-export"): + cli = commands.add_parser(op) + cli.add_argument("--repo-id", required=True, help="one exact Seafile UUID; never all libraries") + if op != "verify-export": + cli.add_argument("--seafile-data", required=True, help="absolute path to native seafile-data") + cli.add_argument("--library-root", required=True, help="absolute managed parent directory") + if op == "export": + cli.add_argument("--seaf-fsck", required=True, help="absolute path to native seaf-fsck executable") + cli.add_argument("--output-dir", required=True, help="unused absolute destination directory") + if op == "verify-export": + cli.add_argument("--output-dir", required=True, help="absolute prior export directory") + cli.add_argument("--hash", action="store_true", help="explicitly hash all exported file bytes") + args = parser.parse_args(argv) + try: + if args.action == "inspect": + result = {"action": "inspect", "status": "FOUND", **find_library(args.repo_id, args.seafile_data, args.library_root)} + elif args.action == "export": + result = export_library(args) + else: + result = verify_export(args) + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + return 0 + except (LibraryError, OSError, subprocess.SubprocessError) as exc: + print("seafile-library-offline: " + str(exc), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) From 1e681df268b77cad1227d062e108496342171cea Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 16:41:21 +0900 Subject: [PATCH 02/14] test(storage): check offline per-repo discovery, safe export and opt-in verification --- tests/test_seafile_library_offline.py | 119 ++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 tests/test_seafile_library_offline.py diff --git a/tests/test_seafile_library_offline.py b/tests/test_seafile_library_offline.py new file mode 100644 index 00000000..d9913a86 --- /dev/null +++ b/tests/test_seafile_library_offline.py @@ -0,0 +1,119 @@ +"""Standalone offline Seafile library tests: no DB, Docker or CloudFile service.""" +import importlib.util +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / "tools" / "ops" / "seafile-library-offline.py" +spec = importlib.util.spec_from_file_location("offline", SCRIPT) +offline = importlib.util.module_from_spec(spec) +spec.loader.exec_module(offline) + +REPO = "f2af601c-8f44-4eb1-ac10-0402783d142d" + + +class OfflineLibraryTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.top = Path(self.tmp.name) + self.native = self.top / "seafile-data" + self.managed = self.top / "libraries" + self.native.mkdir() + self.managed.mkdir() + self.export_parent = self.top / "exports" + self.export_parent.mkdir() + self.fsck = self.top / "fake-fsck" + self.fsck.write_text("""#!/usr/bin/env python3 +from pathlib import Path +import sys +args = sys.argv[1:] +get = lambda k: args[args.index(k) + 1] +repo = args[-1] +data = Path(get('-d')) +assert (data / 'storage/commits' / repo).is_dir() +assert 'cloudfile_v2' not in (Path(get('-F')) / 'seafile.conf').read_text() +path = Path(get('--export')) / (repo[:8] + '_test_owner') +path.mkdir() +(path / 'file.txt').write_text('Hello Seafile!') +""") + self.fsck.chmod(0o700) + + def layout(self, root, *, key=None): + base = root / key if key else root + for kind in offline.KINDS: + (base / "storage" / kind / REPO).mkdir(parents=True) + (base / "storage/commits" / REPO / "ab").mkdir() + (base / "storage/commits" / REPO / "ab" / "cd").write_bytes(b"commit") + + def locate(self): + return offline.find_library(REPO, str(self.native), str(self.managed)) + + def test_native_and_managed_layout_without_cloudfile_database(self): + self.layout(self.native) + self.assertEqual("seafile-native", self.locate()["backend"]) + import shutil + shutil.rmtree(self.native / "storage") + self.layout(self.managed, key="project-a") + self.assertEqual("auto-local", self.locate()["backend"]) + self.assertEqual("project-a", self.locate()["repo_key"]) + + def test_ambiguous_and_missing_repo_rejected(self): + with self.assertRaisesRegex(offline.LibraryError, "not found"): + self.locate() + self.layout(self.managed, key="project-a") + self.layout(self.native) + with self.assertRaisesRegex(offline.LibraryError, "ambiguous"): + self.locate() + with self.assertRaisesRegex(offline.LibraryError, "complete UUID"): + offline.find_library("not-a-repo", str(self.native), str(self.managed)) + + def test_symlinks_never_rename_the_object_source(self): + target = self.top / "elsewhere" + (target / "storage/commits" / REPO).mkdir(parents=True) + (self.managed / "project-a").symlink_to(target) + with self.assertRaisesRegex(offline.LibraryError, "not found"): + self.locate() + (self.managed / "project-a").unlink() + alt = self.top / "managed-link" + alt.symlink_to(self.managed) + with self.assertRaisesRegex(offline.LibraryError, "parent must not be a symbolic"): + offline.find_library(REPO, str(self.native), str(alt)) + + def test_explicit_single_repo_export_and_verify(self): + self.layout(self.managed, key="custom-project") + output = self.export_parent / "repo-one" + cmd = ["export", "--repo-id", REPO, "--seafile-data", str(self.native), + "--library-root", str(self.managed), "--seaf-fsck", str(self.fsck), + "--output-dir", str(output)] + run = subprocess.run([sys.executable, str(SCRIPT)] + cmd, + capture_output=True, text=True) + self.assertEqual(0, run.returncode, run.stderr) + self.assertTrue((output / (REPO[:8] + "_test_owner") / "file.txt").exists()) + self.assertFalse((self.managed / "custom-project/tmpfiles").exists()) + self.assertEqual(b"commit", (self.managed / "custom-project/storage/commits" / REPO / "ab/cd").read_bytes()) + with self.assertRaisesRegex(offline.LibraryError, "exists"): + offline.export_library(type("A", (), dict( + repo_id=REPO, seafile_data=str(self.native), + library_root=str(self.managed), seaf_fsck=str(self.fsck), + output_dir=str(output)))) + summary = offline.verify_export(type("A", (), dict(repo_id=REPO, output_dir=str(output), hash=True))) + self.assertEqual(1, summary["file_count"]) + self.assertEqual(len(b"Hello Seafile!"), summary["total_bytes"]) + self.assertEqual(64, len(summary["tree_sha256"])) + + def test_inspect_does_not_export_or_copy_all_libraries(self): + self.layout(self.native) + run = subprocess.run([sys.executable, str(SCRIPT), "inspect", + "--seafile-data", str(self.native), + "--library-root", str(self.managed)], capture_output=True) + self.assertNotEqual(0, run.returncode) # exact UUID is mandatory + self.assertEqual([], list(self.export_parent.iterdir())) + self.assertEqual("seafile-native", self.locate()["backend"]) + self.assertEqual([], list(self.export_parent.iterdir())) + + +if __name__ == "__main__": + unittest.main() From 9470c39ad27491c4fbdb1c3313f57efcbd852fdd Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 16:43:11 +0900 Subject: [PATCH 03/14] docs(storage): document per-repo offline Seafile inspection and explicit export only --- tools/ops/README-seafile-library-offline.md | 73 +++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 tools/ops/README-seafile-library-offline.md diff --git a/tools/ops/README-seafile-library-offline.md b/tools/ops/README-seafile-library-offline.md new file mode 100644 index 00000000..7e069b0e --- /dev/null +++ b/tools/ops/README-seafile-library-offline.md @@ -0,0 +1,73 @@ +# Seafile 单库离线检查与导出(无需 CloudFile 服务) + +本工具只负责**定位某一个** Seafile CE/CloudFile 库的物理对象目录,以及显式调用 Seafile 原生 \`seaf-fsck --export\` 导出其逻辑文件。无需 Hub、CloudFile Web/API、\`cf_library_storage\` SQL,也不建立新的存储元数据或同步引擎。 + +**默认不备份、不复制、不扫描全库内容。** 所有操作都必须传入**完整且唯一的 \`repo_id\`**,没有 \`--all\`。此工具也不会通过定时器、Compose 启动脚本自动运行。 + +## 存储结构 + +Seafile 原生老库: + +\`\`\`text +/storage/{commits,fs,blocks}//... +\`\`\` + +CloudFile V2 每库本地目录: + +\`\`\`text +//storage/{commits,fs,blocks}//... +\`\`\` + +\`repo_key\` 可以是与 \`repo_id\` 不同的受控名称。离线定位只检查一级库目录中的 \`storage/commits/\`,不需要知道 \`cf_library_storage\` 内容。如果同一 UUID 同时在原生目录和多个库目录中出现,**拒绝自动选择**,由运维人工核对。这些目录是 Seafile 对象存储,**不是用户文件目录**。 + +## 1. 只读定位(推荐的第一步) + +\`\`\`sh +python3 tools/ops/seafile-library-offline.py inspect \ + --repo-id \ + --seafile-data /shared/seafile/seafile-data \ + --library-root /shared/seafile/library-storage +\`\`\` + +只枚举一级目录并核对 Commit/FS/Block 根,不调用服务、不读数据库、不导出任何字节,不更改文件。它**不是**对象哈希完整性检查:完整修复/检查应按照 Seafile 原生工具的约束另行执行。 + +## 2. 显式导出单个库(耗时、耗磁盘空间) + +**操作前确保目标磁盘有足够空间,使用未加密库并明确选择一个 \`repo_id\` 和全新目标目录。** + +\`\`\`sh +python3 tools/ops/seafile-library-offline.py export \ + --repo-id \ + --seafile-data /shared/seafile/seafile-data \ + --library-root /shared/seafile/library-storage \ + --seaf-fsck /opt/seafile/seafile-server-latest/seafile/bin/seaf-fsck \ + --output-dir /mnt/recovery/ +\`\`\` + +此操作: +1. 先按 Seafile 物理目录确定唯一库,不使用 CloudFile 服务或 SQL。 +2. 在目标磁盘创建临时、只用于本次操作的原生 Seafile 配置和对象视图;通过符号链接**引用**该库的 Commit/FS/Block 目录,不复制它们。 +3. 调用 **Seafile 原生** \`seaf-fsck -d ... -F ... --export ... \`。原生工具读取 Commit/FS/Block,生成普通文件。 +4. 导出完成后发布结果;目标目录如果已存在则拒绝覆盖,执行异常时不声称成功。 + +独立的离线临时配置只使用原生 filesystem backend(去掉 \`cloudfile_v2\` 路由层);**不修改原来服务器的 \`seafile.conf\`**。需要可执行的 \`seaf-fsck\` 以及适合该二进制的动态库环境。加密库或损坏库可能无法成功导出;即便返回成功,也不能据此保证所有历史版本都已恢复。Seafile 原生导出读取选出的可用 Commit,而不是完整的数据库备份。 + +## 3. 显式检查已导出的文件 + +\`\`\`sh +python3 tools/ops/seafile-library-offline.py verify-export \ + --repo-id \ + --output-dir /mnt/recovery/ +\`\`\` + +这会读取目录树元数据并计算文件数量/总字节数;如果确有需要再传 \`--hash\`,逐文件读完整内容生成树校验摘要,可能消耗较长时间。 + +**检查导出文件 ≠ 证明全部原对象完整**。完整性验收仍可按 Seafile 官方的 per-repo \`seaf-fsck\` 检查/修复流程进行,涉及库 DB 与对象健康时按其要求执行。官方说明:https://manual.seafile.com/14.0/administration/seafile_fsck/ + +## 4. 严格限制与独立职责 + +- 不提供无库 ID 的批量导出;不启动自动备份、定时同步、全盘复制。 +- 不直接读写用户逻辑文件到 Seafile 对象目录;导出的文件只能进入单独的新目标目录。 +- 不直接修复/变更存储映射;\`RepoStorageId\` 仍是在线后端路由事实。 +- 只支持**本地 filesystem** 的原生布局和 auto-local;S3/RustFS/MinIO 需要其独立工具及配置,绝不猜路径或回退到错误本地目录。 +- 现有 \`cf_recovery.py\` 是**另一个显式调用的、全实例离线备份工具**,不能视为普通的单库导出,也不得加入自动启动/发布路径;如果存在 auto-local/S3,应重新审查其完整备份范围,避免漏数据或误报备份成功。 From 547fa0fed6f24e10d782f8fd22c03699be029cbf Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 16:43:23 +0900 Subject: [PATCH 04/14] docs(storage): render executable examples and code fences correctly --- tools/ops/README-seafile-library-offline.md | 40 ++++++++++----------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tools/ops/README-seafile-library-offline.md b/tools/ops/README-seafile-library-offline.md index 7e069b0e..21e5ae15 100644 --- a/tools/ops/README-seafile-library-offline.md +++ b/tools/ops/README-seafile-library-offline.md @@ -1,73 +1,73 @@ # Seafile 单库离线检查与导出(无需 CloudFile 服务) -本工具只负责**定位某一个** Seafile CE/CloudFile 库的物理对象目录,以及显式调用 Seafile 原生 \`seaf-fsck --export\` 导出其逻辑文件。无需 Hub、CloudFile Web/API、\`cf_library_storage\` SQL,也不建立新的存储元数据或同步引擎。 +本工具只负责**定位某一个** Seafile CE/CloudFile 库的物理对象目录,以及显式调用 Seafile 原生 `seaf-fsck --export` 导出其逻辑文件。无需 Hub、CloudFile Web/API、`cf_library_storage` SQL,也不建立新的存储元数据或同步引擎。 -**默认不备份、不复制、不扫描全库内容。** 所有操作都必须传入**完整且唯一的 \`repo_id\`**,没有 \`--all\`。此工具也不会通过定时器、Compose 启动脚本自动运行。 +**默认不备份、不复制、不扫描全库内容。** 所有操作都必须传入**完整且唯一的 `repo_id`**,没有 `--all`。此工具也不会通过定时器、Compose 启动脚本自动运行。 ## 存储结构 Seafile 原生老库: -\`\`\`text +```text /storage/{commits,fs,blocks}//... -\`\`\` +``` CloudFile V2 每库本地目录: -\`\`\`text +```text //storage/{commits,fs,blocks}//... -\`\`\` +``` -\`repo_key\` 可以是与 \`repo_id\` 不同的受控名称。离线定位只检查一级库目录中的 \`storage/commits/\`,不需要知道 \`cf_library_storage\` 内容。如果同一 UUID 同时在原生目录和多个库目录中出现,**拒绝自动选择**,由运维人工核对。这些目录是 Seafile 对象存储,**不是用户文件目录**。 +`repo_key` 可以是与 `repo_id` 不同的受控名称。离线定位只检查一级库目录中的 `storage/commits/`,不需要知道 `cf_library_storage` 内容。如果同一 UUID 同时在原生目录和多个库目录中出现,**拒绝自动选择**,由运维人工核对。这些目录是 Seafile 对象存储,**不是用户文件目录**。 ## 1. 只读定位(推荐的第一步) -\`\`\`sh +```sh python3 tools/ops/seafile-library-offline.py inspect \ --repo-id \ --seafile-data /shared/seafile/seafile-data \ --library-root /shared/seafile/library-storage -\`\`\` +``` 只枚举一级目录并核对 Commit/FS/Block 根,不调用服务、不读数据库、不导出任何字节,不更改文件。它**不是**对象哈希完整性检查:完整修复/检查应按照 Seafile 原生工具的约束另行执行。 ## 2. 显式导出单个库(耗时、耗磁盘空间) -**操作前确保目标磁盘有足够空间,使用未加密库并明确选择一个 \`repo_id\` 和全新目标目录。** +**操作前确保目标磁盘有足够空间,使用未加密库并明确选择一个 `repo_id` 和全新目标目录。** -\`\`\`sh +```sh python3 tools/ops/seafile-library-offline.py export \ --repo-id \ --seafile-data /shared/seafile/seafile-data \ --library-root /shared/seafile/library-storage \ --seaf-fsck /opt/seafile/seafile-server-latest/seafile/bin/seaf-fsck \ --output-dir /mnt/recovery/ -\`\`\` +``` 此操作: 1. 先按 Seafile 物理目录确定唯一库,不使用 CloudFile 服务或 SQL。 2. 在目标磁盘创建临时、只用于本次操作的原生 Seafile 配置和对象视图;通过符号链接**引用**该库的 Commit/FS/Block 目录,不复制它们。 -3. 调用 **Seafile 原生** \`seaf-fsck -d ... -F ... --export ... \`。原生工具读取 Commit/FS/Block,生成普通文件。 +3. 调用 **Seafile 原生** `seaf-fsck -d ... -F ... --export ... `。原生工具读取 Commit/FS/Block,生成普通文件。 4. 导出完成后发布结果;目标目录如果已存在则拒绝覆盖,执行异常时不声称成功。 -独立的离线临时配置只使用原生 filesystem backend(去掉 \`cloudfile_v2\` 路由层);**不修改原来服务器的 \`seafile.conf\`**。需要可执行的 \`seaf-fsck\` 以及适合该二进制的动态库环境。加密库或损坏库可能无法成功导出;即便返回成功,也不能据此保证所有历史版本都已恢复。Seafile 原生导出读取选出的可用 Commit,而不是完整的数据库备份。 +独立的离线临时配置只使用原生 filesystem backend(去掉 `cloudfile_v2` 路由层);**不修改原来服务器的 `seafile.conf`**。需要可执行的 `seaf-fsck` 以及适合该二进制的动态库环境。加密库或损坏库可能无法成功导出;即便返回成功,也不能据此保证所有历史版本都已恢复。Seafile 原生导出读取选出的可用 Commit,而不是完整的数据库备份。 ## 3. 显式检查已导出的文件 -\`\`\`sh +```sh python3 tools/ops/seafile-library-offline.py verify-export \ --repo-id \ --output-dir /mnt/recovery/ -\`\`\` +``` -这会读取目录树元数据并计算文件数量/总字节数;如果确有需要再传 \`--hash\`,逐文件读完整内容生成树校验摘要,可能消耗较长时间。 +这会读取目录树元数据并计算文件数量/总字节数;如果确有需要再传 `--hash`,逐文件读完整内容生成树校验摘要,可能消耗较长时间。 -**检查导出文件 ≠ 证明全部原对象完整**。完整性验收仍可按 Seafile 官方的 per-repo \`seaf-fsck\` 检查/修复流程进行,涉及库 DB 与对象健康时按其要求执行。官方说明:https://manual.seafile.com/14.0/administration/seafile_fsck/ +**检查导出文件 ≠ 证明全部原对象完整**。完整性验收仍可按 Seafile 官方的 per-repo `seaf-fsck` 检查/修复流程进行,涉及库 DB 与对象健康时按其要求执行。官方说明:https://manual.seafile.com/14.0/administration/seafile_fsck/ ## 4. 严格限制与独立职责 - 不提供无库 ID 的批量导出;不启动自动备份、定时同步、全盘复制。 - 不直接读写用户逻辑文件到 Seafile 对象目录;导出的文件只能进入单独的新目标目录。 -- 不直接修复/变更存储映射;\`RepoStorageId\` 仍是在线后端路由事实。 +- 不直接修复/变更存储映射;`RepoStorageId` 仍是在线后端路由事实。 - 只支持**本地 filesystem** 的原生布局和 auto-local;S3/RustFS/MinIO 需要其独立工具及配置,绝不猜路径或回退到错误本地目录。 -- 现有 \`cf_recovery.py\` 是**另一个显式调用的、全实例离线备份工具**,不能视为普通的单库导出,也不得加入自动启动/发布路径;如果存在 auto-local/S3,应重新审查其完整备份范围,避免漏数据或误报备份成功。 +- 现有 `cf_recovery.py` 是**另一个显式调用的、全实例离线备份工具**,不能视为普通的单库导出,也不得加入自动启动/发布路径;如果存在 auto-local/S3,应重新审查其完整备份范围,避免漏数据或误报备份成功。 From b09e54431604ebfa9ecbc9aead072e646894de09 Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 16:46:32 +0900 Subject: [PATCH 05/14] ci(storage): run bounded standalone per-repo inspect/export tests only when relevant --- .github/workflows/dev.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 53a4d90c..ee91d0e3 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -51,3 +51,11 @@ jobs: run: | python3 -m unittest discover -s tests -p test_build_receipt.py -q python3 -m unittest discover -s tests -p test_current_build_policy.py -q + + - name: Test per-library offline CLI when changed (no services / no backup) + shell: bash + run: | + if git diff --name-only HEAD^ HEAD -- \ + tools/ops/seafile-library-offline.py tests/test_seafile_library_offline.py | grep -q .; then + python3 -m unittest discover -s tests -p test_seafile_library_offline.py -q + fi From fef9725d7cd174d1ba534b1ded0bf89646110a6f Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 16:53:15 +0900 Subject: [PATCH 06/14] docs(agents): prohibit automatic bulk file backup and require explicit repo id --- AGENTS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 0f437713..94b0d0a1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,3 +1,9 @@ +## 存储维护默认操作(2026-10-24) + +**只读、单库、Seafile 原生优先。** 存储识别请先使用 [Seafile 单库离线工具](tools/ops/README-seafile-library-offline.md) 的 `inspect --repo-id`;逻辑文件导出只在明确指定 `repo_id`、新的目标目录和原生 `seaf-fsck` 时执行;校验已导出文件使用显式 `verify-export`,是否全文件哈希由 `--hash` 再确认。默认构建、安装、容器启动、CI 和 Agent 操作都不得触发批量备份、文件导出或原生对象根复制。 + +`cf_recovery.py` 属于**另外的旧 CE 全实例离线灾备**,不是 CloudFile V2 正常单库工作流;不得调用它为 V2 自动备份,检测到 V2 数据时应失败拒绝,不能返回不完整快照的成功结果。遇到 S3/加密库、重复 UUID、多存储冲突,停下核查,不能猜目录或盲目重试。 + # AGENTS.md — cloudfile-docker > **当前版本规则**:`cloudfile-docker/dev` 就是 CloudFile **V2 当前开发主线**,以 Seafile CE14 为基础。与 `cloudfile-server/dev`、`cloudfile-hub/dev` 统一,不存在另一条“当前 V0.x dev”。本规则适用于 AI 开发 Agent 和维护者。 From ba78cc94650245fef0a77a4f763e3d15d126fbf9 Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 17:11:20 +0900 Subject: [PATCH 07/14] fix(offline): require explicit mounted root and bound native export logs --- tools/ops/seafile-library-offline.py | 30 +++++++++++++++++----------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tools/ops/seafile-library-offline.py b/tools/ops/seafile-library-offline.py index 55bee82f..85a13c67 100644 --- a/tools/ops/seafile-library-offline.py +++ b/tools/ops/seafile-library-offline.py @@ -37,13 +37,11 @@ def repository_id(raw): return raw -def real_dir(path, *, optional=False): - """Keep managed children strictly physical, but allow native CE root aliases.""" +def real_dir(path): + """Require mounted roots; never treat an explicitly supplied missing mount as absent.""" path = Path(path) if not path.is_absolute(): raise LibraryError("storage roots must be absolute") - if optional and not path.exists() and not path.is_symlink(): - return None try: root = path.resolve(strict=True) except (OSError, RuntimeError) as exc: @@ -93,8 +91,8 @@ def candidate(root, repo_id, backend, key=None): def find_library(repo_id, native_root, managed_root): repo_id = repository_id(repo_id) native = real_dir(native_root) - managed = real_dir(managed_root, optional=True) - if managed and Path(managed_root).is_symlink(): + managed = real_dir(managed_root) if managed_root is not None else None + if managed is not None and Path(managed_root).is_symlink(): raise LibraryError("managed parent must not be a symbolic link") found = [] item = candidate(native, repo_id, "seafile-native") @@ -138,7 +136,8 @@ def export_library(args): fsck = Path(args.seaf_fsck) if not fsck.is_absolute() or not fsck.is_file() or not os.access(fsck, os.X_OK): raise LibraryError("--seaf-fsck must name an executable Seafile fsck binary") - target = output_directory(args.output_dir, (args.seafile_data, args.library_root)) + roots = (args.seafile_data,) + ((args.library_root,) if args.library_root else ()) + target = output_directory(args.output_dir, roots) # Temp staging lives on the destination volume, not in /tmp and not in the # object store. Failed export is deleted; no target is published on error. with tempfile.TemporaryDirectory(prefix=".seafile-export-", dir=target.parent) as temporary: @@ -165,10 +164,17 @@ def export_library(args): # Stock Seafile CE filesystem routing, without cloudfile_v2 or a DB. command = [str(fsck), "-c", str(conf), "-d", str(data), "-F", str(conf), "--export", str(out), item["repo_id"]] - result = subprocess.run(command, cwd=sandbox, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, text=True, errors="replace") - if result.returncode != 0: - raise LibraryError("seaf-fsck export failed: " + result.stdout[-1500:]) + # Native export may print thousands of file messages; do not keep all + # of them in RAM just to report the last error if it fails. + with tempfile.TemporaryFile(dir=sandbox) as output_log: + result = subprocess.run(command, cwd=sandbox, stdout=output_log, + stderr=subprocess.STDOUT) + if result.returncode != 0: + output_log.seek(0, os.SEEK_END) + length = output_log.tell() + output_log.seek(max(0, length - 2048)) + tail = output_log.read().decode("utf-8", "replace") + raise LibraryError("seaf-fsck export failed: " + tail[-1500:]) children = list(out.iterdir()) if len(children) != 1 or not physical_dir(children[0]) or \ not children[0].name.startswith(item["repo_id"][:8] + "_"): @@ -233,7 +239,7 @@ def main(argv=None): cli.add_argument("--repo-id", required=True, help="one exact Seafile UUID; never all libraries") if op != "verify-export": cli.add_argument("--seafile-data", required=True, help="absolute path to native seafile-data") - cli.add_argument("--library-root", required=True, help="absolute managed parent directory") + cli.add_argument("--library-root", help="existing absolute managed root; omit for native CE only") if op == "export": cli.add_argument("--seaf-fsck", required=True, help="absolute path to native seaf-fsck executable") cli.add_argument("--output-dir", required=True, help="unused absolute destination directory") From 4096dc88da6f9133d8ba5c2e340613c2ac45507b Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 17:11:43 +0900 Subject: [PATCH 08/14] test(offline): reject explicitly missing mounts while allowing CE-only operation --- tests/test_seafile_library_offline.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_seafile_library_offline.py b/tests/test_seafile_library_offline.py index d9913a86..43ce6047 100644 --- a/tests/test_seafile_library_offline.py +++ b/tests/test_seafile_library_offline.py @@ -60,6 +60,26 @@ def test_native_and_managed_layout_without_cloudfile_database(self): self.assertEqual("auto-local", self.locate()["backend"]) self.assertEqual("project-a", self.locate()["repo_key"]) + def test_explicit_missing_managed_mount_never_falls_back(self): + self.layout(self.native) + missing = self.top / "missing-mount" + with self.assertRaisesRegex(offline.LibraryError, "storage root is not accessible"): + offline.find_library(REPO, str(self.native), str(missing)) + # A CE-only operator can omit the managed root instead of asking the + # tool to guess whether an explicitly named mount is missing. + self.assertEqual("seafile-native", + offline.find_library(REPO, str(self.native), None)["backend"]) + + def test_cli_inspect_native_ce_without_managed_root(self): + import json + self.layout(self.native) + proc = subprocess.run( + [sys.executable, str(SCRIPT), "inspect", "--repo-id", REPO, + "--seafile-data", str(self.native)], + capture_output=True, text=True) + self.assertEqual(0, proc.returncode, proc.stderr) + self.assertEqual("seafile-native", json.loads(proc.stdout)["backend"]) + def test_ambiguous_and_missing_repo_rejected(self): with self.assertRaisesRegex(offline.LibraryError, "not found"): self.locate() From 029b857b06bb6c789df00145173af499d71206bb Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 17:12:09 +0900 Subject: [PATCH 09/14] docs(offline): require present managed mount when explicitly selected --- tools/ops/README-seafile-library-offline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ops/README-seafile-library-offline.md b/tools/ops/README-seafile-library-offline.md index 21e5ae15..e7582b55 100644 --- a/tools/ops/README-seafile-library-offline.md +++ b/tools/ops/README-seafile-library-offline.md @@ -29,7 +29,7 @@ python3 tools/ops/seafile-library-offline.py inspect \ --library-root /shared/seafile/library-storage ``` -只枚举一级目录并核对 Commit/FS/Block 根,不调用服务、不读数据库、不导出任何字节,不更改文件。它**不是**对象哈希完整性检查:完整修复/检查应按照 Seafile 原生工具的约束另行执行。 +只枚举一级目录并核对 Commit/FS/Block 根,不调用服务、不读数据库、不导出任何字节,不更改文件。**只有旧版 CE 原生库时,可以省略 `--library-root`;一旦显式提供该参数,目录不存在或挂载丢失必须报错,不得静默回退到旧库目录。** 它不是对象哈希完整性检查:完整修复/检查应按照 Seafile 原生工具的约束另行执行。 ## 2. 显式导出单个库(耗时、耗磁盘空间) From b96648968720920103f92e618cd875688a3f0037 Mon Sep 17 00:00:00 2001 From: review-bot Date: Sat, 24 Oct 2026 17:03:36 +0800 Subject: [PATCH 10/14] fix(offline): make verify-export tree digest independent of directory order os.walk descends in filesystem readdir order, so the same logical export hashed differently after a copy/re-export, which makes tree_sha256 useless for comparison. Sort each directory list before recursing and cover it with a deterministic regression test. --- tests/test_seafile_library_offline.py | 30 +++++++++++++++++++++++++++ tools/ops/seafile-library-offline.py | 4 ++++ 2 files changed, 34 insertions(+) diff --git a/tests/test_seafile_library_offline.py b/tests/test_seafile_library_offline.py index 43ce6047..f6543be5 100644 --- a/tests/test_seafile_library_offline.py +++ b/tests/test_seafile_library_offline.py @@ -1,10 +1,12 @@ """Standalone offline Seafile library tests: no DB, Docker or CloudFile service.""" import importlib.util +import os from pathlib import Path import subprocess import sys import tempfile import unittest +from unittest import mock SCRIPT = Path(__file__).resolve().parents[1] / "tools" / "ops" / "seafile-library-offline.py" spec = importlib.util.spec_from_file_location("offline", SCRIPT) @@ -124,6 +126,34 @@ def test_explicit_single_repo_export_and_verify(self): self.assertEqual(len(b"Hello Seafile!"), summary["total_bytes"]) self.assertEqual(64, len(summary["tree_sha256"])) + def test_verify_export_hash_is_directory_order_independent(self): + # A file-tree digest must be canonical: os.walk descends in readdir + # order, which differs between identical exports on other filesystems. + root = self.top / "hash-exports" + library = root / (REPO[:8] + "_library") + for name in ("zz-dir", "aa-dir", "mm-dir"): + (library / name).mkdir(parents=True) + (library / name / "file.txt").write_text(name) + args = type("A", (), dict(repo_id=REPO, output_dir=str(root), hash=True)) + + def fake_walk(order): + def walk(top, **kwargs): + base = Path(top) + directories = list(order) + yield str(base), directories, [] + # Mirrors os.walk: the caller's in-place sort of ``directories`` + # decides the child order. + for name in directories: + yield str(base / name), [], ["file.txt"] + return walk + + with mock.patch.object(os, "walk", fake_walk(("zz-dir", "aa-dir", "mm-dir"))): + first = offline.verify_export(args) + with mock.patch.object(os, "walk", fake_walk(("aa-dir", "zz-dir", "mm-dir"))): + second = offline.verify_export(args) + self.assertEqual(3, first["file_count"]) + self.assertEqual(first["tree_sha256"], second["tree_sha256"]) + def test_inspect_does_not_export_or_copy_all_libraries(self): self.layout(self.native) run = subprocess.run([sys.executable, str(SCRIPT), "inspect", diff --git a/tools/ops/seafile-library-offline.py b/tools/ops/seafile-library-offline.py index 85a13c67..b56108f0 100644 --- a/tools/ops/seafile-library-offline.py +++ b/tools/ops/seafile-library-offline.py @@ -209,6 +209,10 @@ def verify_export(args): total = 0 digest = hashlib.sha256() if args.hash else None for here, directories, names in os.walk(base, followlinks=False): + # Canonical order: os.walk descends in readdir order, so a tree digest + # would differ between identical exports on another filesystem/copy + # unless directory order is pinned before recursing. + directories.sort() for name in directories: if not physical_dir(Path(here) / name): raise LibraryError("export contains symlink or unsupported directory") From 20b10b4bd2998897e2d6ce6178fbb89856495c60 Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 19:01:31 +0900 Subject: [PATCH 11/14] fix(offline): refuse partial native per-file export despite zero exit status --- tools/ops/seafile-library-offline.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/ops/seafile-library-offline.py b/tools/ops/seafile-library-offline.py index b56108f0..d2269361 100644 --- a/tools/ops/seafile-library-offline.py +++ b/tools/ops/seafile-library-offline.py @@ -175,6 +175,17 @@ def export_library(args): output_log.seek(max(0, length - 2048)) tail = output_log.read().decode("utf-8", "replace") raise LibraryError("seaf-fsck export failed: " + tail[-1500:]) + # CE's export_file()/create_file() can log a per-file write error + # yet return 0 after exporting the other files. Never publish + # such a partial result as success. Scan the explicit export log + # incrementally, not into memory, and keep the native reader. + output_log.seek(0) + for line in output_log: + if (b"Failed to export file " in line or + b"Failed to mkdir " in line or + b"Failed to open block " in line): + raise LibraryError("seaf-fsck reported an incomplete export; " + "no destination was published") children = list(out.iterdir()) if len(children) != 1 or not physical_dir(children[0]) or \ not children[0].name.startswith(item["repo_id"][:8] + "_"): From b4b6c4e6551c3ab001166e1d1f5a81c1af91b3ce Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 19:01:51 +0900 Subject: [PATCH 12/14] test(offline): refuse zero-exit native export with per-file failure --- tests/test_seafile_library_offline.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_seafile_library_offline.py b/tests/test_seafile_library_offline.py index f6543be5..d75de481 100644 --- a/tests/test_seafile_library_offline.py +++ b/tests/test_seafile_library_offline.py @@ -154,6 +154,24 @@ def walk(top, **kwargs): self.assertEqual(3, first["file_count"]) self.assertEqual(first["tree_sha256"], second["tree_sha256"]) + def test_native_export_partial_failure_never_publishes(self): + # Seafile CE export_file()/create_file() log some per-file errors + # without propagating an error exit status. The wrapper must refuse. + self.layout(self.managed, key="managed-a") + self.fsck.write_text(self.fsck.read_text() + + "\\nprint('Failed to export file /missing/block')\\n") + output = self.export_parent / "must-not-publish" + cmd = [sys.executable, str(SCRIPT), "export", "--repo-id", REPO, + "--seafile-data", str(self.native), + "--library-root", str(self.managed), + "--seaf-fsck", str(self.fsck), + "--output-dir", str(output)] + run = subprocess.run(cmd, capture_output=True, text=True) + self.assertNotEqual(0, run.returncode) + self.assertIn("incomplete export", run.stderr) + self.assertFalse(output.exists()) + self.assertTrue((self.managed / "managed-a/storage/commits" / REPO).is_dir()) + def test_inspect_does_not_export_or_copy_all_libraries(self): self.layout(self.native) run = subprocess.run([sys.executable, str(SCRIPT), "inspect", From 82b40784526e5cfd2753f0301cb92bcb76fa76e0 Mon Sep 17 00:00:00 2001 From: servicememeai Date: Sat, 24 Oct 2026 19:02:20 +0900 Subject: [PATCH 13/14] test(offline): emit actual newlines in simulated seaf-fsck output --- tests/test_seafile_library_offline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_seafile_library_offline.py b/tests/test_seafile_library_offline.py index d75de481..4777a890 100644 --- a/tests/test_seafile_library_offline.py +++ b/tests/test_seafile_library_offline.py @@ -159,7 +159,7 @@ def test_native_export_partial_failure_never_publishes(self): # without propagating an error exit status. The wrapper must refuse. self.layout(self.managed, key="managed-a") self.fsck.write_text(self.fsck.read_text() + - "\\nprint('Failed to export file /missing/block')\\n") + "\nprint('Failed to export file /missing/block')\n") output = self.export_parent / "must-not-publish" cmd = [sys.executable, str(SCRIPT), "export", "--repo-id", REPO, "--seafile-data", str(self.native), From e05a7f2ad5dbe50d4bed132ce17b556ace00be8c Mon Sep 17 00:00:00 2001 From: review-bot Date: Sat, 24 Oct 2026 19:20:52 +0800 Subject: [PATCH 14/14] fix(offline): fail closed on native export warnings and missing objects seaf-fsck's export_file()/create_file() are void: CE can skip one file, a whole directory subtree (unreadable FS dir object) or an uncreatable destination file, still print "Finish exporting files" and exit 0. The previous guard only matched three per-file messages, so a missing subdir/root dir object or a g_open() failure (e.g. ENOSPC) was published as EXPORTED_UNVERIFIED. Treat any native [WARNING]/[ERROR] record plus the known INFO failure signatures ("Failed to export file", "Failed to mkdir", "... is missing.") as failure: no destination is published, exit code is 2, and the matched native log lines are surfaced for diagnosis. The scan stays incremental. Also mirror the shipped seaf-fsck.sh: prepend the seafile/lib and seafile/lib64 siblings of the given binary to LD_LIBRARY_PATH (keeping any inherited value). A direct call into 14.0.8 images otherwise dies with "libevent-2.1.so.7: cannot open shared object file". No-op when that install layout is absent. Six regression tests use log lines captured verbatim from real CE14 seaf-fsck --export runs plus the library-path handoff; a benign INFO-only progress log still publishes. README states the precise publish conditions, the direct-binary invocation and its library-path handling, and that log-based detection cannot prove content completeness. Verified against real 14.0 images: valid export publishes; missing block, missing subdir object and missing root dir object are refused; the 14.0.8 image now exports without any caller LD_LIBRARY_PATH. --- tests/test_seafile_library_offline.py | 129 ++++++++++++++++++++ tools/ops/README-seafile-library-offline.md | 6 +- tools/ops/seafile-library-offline.py | 46 +++++-- 3 files changed, 169 insertions(+), 12 deletions(-) diff --git a/tests/test_seafile_library_offline.py b/tests/test_seafile_library_offline.py index 4777a890..18226d5c 100644 --- a/tests/test_seafile_library_offline.py +++ b/tests/test_seafile_library_offline.py @@ -53,6 +53,74 @@ def layout(self, root, *, key=None): def locate(self): return offline.find_library(REPO, str(self.native), str(self.managed)) + def add_fsck_output(self, *lines): + """Append print() lines to the fake fsck so they land on stdout.""" + self.fsck.write_text(self.fsck.read_text() + "".join( + "\nprint(%r)\n" % line for line in lines)) + + def export_cmd(self, output, fsck=None): + return [sys.executable, str(SCRIPT), "export", "--repo-id", REPO, + "--seafile-data", str(self.native), + "--library-root", str(self.managed), + "--seaf-fsck", str(fsck or self.fsck), + "--output-dir", str(output)] + + def bundle_fsck(self): + """Fake seaf-fsck installed as /seafile/bin/seaf-fsck. + + The script records the LD_LIBRARY_PATH it was started with, which is + how the shipped seaf-fsck.sh passes the bundled libraries. + """ + bundle = self.top / "seafile" + (bundle / "bin").mkdir(parents=True) + (bundle / "lib").mkdir() + (bundle / "lib64").mkdir() + fake = bundle / "bin" / "fake-fsck" + fake.write_text("""#!/usr/bin/env python3 +import os +from pathlib import Path +import sys +args = sys.argv[1:] +get = lambda k: args[args.index(k) + 1] +repo = args[-1] +assert (Path(get('-d')) / 'storage/commits' / repo).is_dir() +path = Path(get('--export')) / (repo[:8] + '_test_owner') +path.mkdir() +(path / 'file.txt').write_text('Hello Seafile!') +Path(__file__).with_name('ld_library_path.txt').write_text( + os.environ.get('LD_LIBRARY_PATH', '')) +""") + fake.chmod(0o700) + return fake, bundle / "bin" / "ld_library_path.txt" + + def test_seaf_fsck_bundle_libraries_are_prepended(self): + # seaf-fsck.sh exports SEAFILE_LD_LIBRARY_PATH; the direct call must + # provide the same sibling libraries or some builds fail to load. + self.layout(self.managed, key="managed-a") + fake, record = self.bundle_fsck() + output = self.export_parent / "bundle-libraries" + run = subprocess.run( + self.export_cmd(output, fsck=fake), capture_output=True, text=True, + env={**os.environ, "LD_LIBRARY_PATH": "/inherited"}) + self.assertEqual(0, run.returncode, run.stderr) + bundle = fake.parent.parent + expected = os.pathsep.join( + [str(bundle / "lib"), str(bundle / "lib64"), "/inherited"]) + self.assertEqual(expected, record.read_text()) + self.assertTrue((output / (REPO[:8] + "_test_owner") / "file.txt").exists()) + + def test_seaf_fsck_without_bundle_libraries_keeps_env(self): + self.layout(self.managed, key="managed-a") + fake, record = self.bundle_fsck() + (fake.parent.parent / "lib").rmdir() + (fake.parent.parent / "lib64").rmdir() + output = self.export_parent / "no-bundle-libraries" + run = subprocess.run( + self.export_cmd(output, fsck=fake), capture_output=True, text=True, + env={**os.environ, "LD_LIBRARY_PATH": "/only"}) + self.assertEqual(0, run.returncode, run.stderr) + self.assertEqual("/only", record.read_text()) + def test_native_and_managed_layout_without_cloudfile_database(self): self.layout(self.native) self.assertEqual("seafile-native", self.locate()["backend"]) @@ -172,6 +240,67 @@ def test_native_export_partial_failure_never_publishes(self): self.assertFalse(output.exists()) self.assertTrue((self.managed / "managed-a/storage/commits" / REPO).is_dir()) + def test_native_export_native_warning_never_publishes(self): + # Real CE14 output for an unreadable FS dir object: a single WARNING, + # then "Finish exporting", exit status 0 and a silently partial tree. + self.layout(self.managed, key="managed-a") + self.add_fsck_output( + "[2026-10-24 11:14:03] [WARNING] ../../common/fs-mgr.c(1895): " + "[fs mgr] Failed to read dir " + "f4b3106f4deb9c74fab57bcb8864c9fac3237411.") + output = self.export_parent / "warning-must-not-publish" + run = subprocess.run(self.export_cmd(output), capture_output=True, text=True) + self.assertNotEqual(0, run.returncode) + self.assertIn("incomplete export", run.stderr) + self.assertIn("Failed to read dir", run.stderr) # matched line is surfaced + self.assertFalse(output.exists()) + + def test_native_export_missing_dir_object_never_publishes(self): + # Real CE14 output when the root FS dir object is gone: an INFO + # "is missing." line and an empty library dir that would otherwise + # pass the exactly-one-export check with exit status 0. + self.layout(self.managed, key="managed-a") + self.add_fsck_output( + "[2026-10-24 11:14:08] [INFO] fsck.c(55): Dir " + "e4bdf4e07d7a8652dd1dba8747f22c938d96936c is missing.") + output = self.export_parent / "missing-dir-must-not-publish" + run = subprocess.run(self.export_cmd(output), capture_output=True, text=True) + self.assertNotEqual(0, run.returncode) + self.assertIn("incomplete export", run.stderr) + self.assertFalse(output.exists()) + + def test_native_export_open_file_failure_never_publishes(self): + # create_file()'s g_open() failure (e.g. ENOSPC, EACCES on the + # destination) is a WARNING only; that file is skipped, exit stays 0. + self.layout(self.managed, key="managed-a") + self.add_fsck_output( + "[2026-10-24 11:14:03] [WARNING] fsck.c(1117): Open file " + "/mnt/recovery/out/file.txt failed: No space left on device.") + output = self.export_parent / "no-space-must-not-publish" + run = subprocess.run(self.export_cmd(output), capture_output=True, text=True) + self.assertNotEqual(0, run.returncode) + self.assertIn("incomplete export", run.stderr) + self.assertFalse(output.exists()) + + def test_native_export_benign_progress_log_still_publishes(self): + # A normal export logs INFO progress only; detection must not reject it. + self.layout(self.managed, key="managed-a") + self.add_fsck_output( + "[2026-10-24 11:13:58] [INFO] fsck.c(1208): Scanning available " + "commits for repo " + REPO + "...", + "[2026-10-24 11:13:58] [INFO] fsck.c(1247): Find available commit " + "1d4999db(created at 2023-11-14 22:13:20), will export files from it.", + "[2026-10-24 11:13:58] [INFO] fsck.c(1283): Start to export files " + "for repo f2af601c(review-repo).", + "[2026-10-24 11:13:58] [INFO] fsck.c(1151): Export file " + "/out/f2af601c_test_owner/file.txt.", + "[2026-10-24 11:13:58] [INFO] fsck.c(1301): Finish exporting files " + "for repo f2af601c.") + output = self.export_parent / "benign-log-publishes" + run = subprocess.run(self.export_cmd(output), capture_output=True, text=True) + self.assertEqual(0, run.returncode, run.stderr) + self.assertTrue((output / (REPO[:8] + "_test_owner") / "file.txt").exists()) + def test_inspect_does_not_export_or_copy_all_libraries(self): self.layout(self.native) run = subprocess.run([sys.executable, str(SCRIPT), "inspect", diff --git a/tools/ops/README-seafile-library-offline.md b/tools/ops/README-seafile-library-offline.md index e7582b55..497d9ebb 100644 --- a/tools/ops/README-seafile-library-offline.md +++ b/tools/ops/README-seafile-library-offline.md @@ -48,9 +48,11 @@ python3 tools/ops/seafile-library-offline.py export \ 1. 先按 Seafile 物理目录确定唯一库,不使用 CloudFile 服务或 SQL。 2. 在目标磁盘创建临时、只用于本次操作的原生 Seafile 配置和对象视图;通过符号链接**引用**该库的 Commit/FS/Block 目录,不复制它们。 3. 调用 **Seafile 原生** `seaf-fsck -d ... -F ... --export ... `。原生工具读取 Commit/FS/Block,生成普通文件。 -4. 导出完成后发布结果;目标目录如果已存在则拒绝覆盖,执行异常时不声称成功。 +4. 只有在**全部**条件满足时才发布结果:原生工具退出码为 0、日志中没有 `[WARNING]`/`[ERROR]` 记录、没有已知失败标记(`Failed to export file`、`Failed to mkdir`、`Dir/File ... is missing.` 等)、且恰好产出一个库目录。否则整体失败:不创建/不保留目标目录,返回非零退出码,并在 stderr 附带命中的原生日志行;目标目录已存在时同样拒绝覆盖。 -独立的离线临时配置只使用原生 filesystem backend(去掉 `cloudfile_v2` 路由层);**不修改原来服务器的 `seafile.conf`**。需要可执行的 `seaf-fsck` 以及适合该二进制的动态库环境。加密库或损坏库可能无法成功导出;即便返回成功,也不能据此保证所有历史版本都已恢复。Seafile 原生导出读取选出的可用 Commit,而不是完整的数据库备份。 +独立的离线临时配置只使用原生 filesystem backend(去掉 `cloudfile_v2` 路由层);**不修改原来服务器的 `seafile.conf`**。需要可执行的 `seaf-fsck` 以及适合该二进制的动态库环境。**本工具直接执行所给的 `seaf-fsck` 二进制,不经过发行包的 `seaf-fsck.sh` 包装脚本**;与包装脚本一致,它会自动把该二进制同级安装树中的 `seafile/lib`、`seafile/lib64` 前置到 `LD_LIBRARY_PATH`(保留调用方已有值;二进制不在安装树中时保持原环境),因此 14.0.8 这类依赖随包 `libevent` 的镜像也可直接执行。若二进制位于非标准位置或仍有库依赖缺失,需由调用方自行设置 `LD_LIBRARY_PATH`。加密库或损坏库可能无法成功导出;即便返回成功,也不能据此保证所有历史版本都已恢复。Seafile 原生导出读取选出的可用 Commit,而不是完整的数据库备份。 + +原生导出只在日志里报告**部分**内部失败(C 层 `export_file()` 为 `void`:缺块、目录对象读不到、目标文件无法创建,都可能只留一行日志后继续)。本工具按日志级别与已知失败标记判断,无法保证覆盖未来版本新增的静默失败;**成功导出仍不等于内容完整**,完整性仍需 `verify-export` 与官方 per-repo fsck。 ## 3. 显式检查已导出的文件 diff --git a/tools/ops/seafile-library-offline.py b/tools/ops/seafile-library-offline.py index d2269361..83682c81 100644 --- a/tools/ops/seafile-library-offline.py +++ b/tools/ops/seafile-library-offline.py @@ -164,28 +164,54 @@ def export_library(args): # Stock Seafile CE filesystem routing, without cloudfile_v2 or a DB. command = [str(fsck), "-c", str(conf), "-d", str(data), "-F", str(conf), "--export", str(out), item["repo_id"]] + # The shipped seaf-fsck.sh exports SEAFILE_LD_LIBRARY_PATH before + # calling the binary. A direct call must do the same: some builds + # (e.g. 14.0.8 images) link libevent/libsearpc from seafile/lib{,64} + # and fail to load without it. Prepend the sibling libraries of the + # given binary; keep any inherited value and leave env untouched when + # that layout does not exist (system loader paths still apply). + environment = dict(os.environ) + bundle = fsck.parent.parent + library_dirs = [path for path in (bundle / "lib", bundle / "lib64") + if path.is_dir()] + if library_dirs: + prefix = os.pathsep.join(str(path) for path in library_dirs) + inherited = environment.get("LD_LIBRARY_PATH") + environment["LD_LIBRARY_PATH"] = ( + prefix + (os.pathsep + inherited if inherited else "")) # Native export may print thousands of file messages; do not keep all # of them in RAM just to report the last error if it fails. with tempfile.TemporaryFile(dir=sandbox) as output_log: - result = subprocess.run(command, cwd=sandbox, stdout=output_log, - stderr=subprocess.STDOUT) + result = subprocess.run(command, cwd=sandbox, env=environment, + stdout=output_log, stderr=subprocess.STDOUT) if result.returncode != 0: output_log.seek(0, os.SEEK_END) length = output_log.tell() output_log.seek(max(0, length - 2048)) tail = output_log.read().decode("utf-8", "replace") raise LibraryError("seaf-fsck export failed: " + tail[-1500:]) - # CE's export_file()/create_file() can log a per-file write error - # yet return 0 after exporting the other files. Never publish - # such a partial result as success. Scan the explicit export log - # incrementally, not into memory, and keep the native reader. + # CE's export_file()/create_file() are void: they can log a + # per-file failure, skip a whole subtree when an FS dir object is + # unreadable, or fail to create a destination file, yet still print + # "Finish exporting files" and exit 0. Never publish such a partial + # result. Fail closed unless the native log holds only normal + # progress: a WARNING/ERROR record or a known failure signature + # means the tree cannot be claimed complete. Banner lines are + # matched regardless of locale (CE log records are English) and + # the scan is incremental so a huge export log is never held in RAM. output_log.seek(0) + reasons = [] for line in output_log: - if (b"Failed to export file " in line or + if (b"[WARNING]" in line or b"[ERROR]" in line or + b"Failed to export file " in line or b"Failed to mkdir " in line or - b"Failed to open block " in line): - raise LibraryError("seaf-fsck reported an incomplete export; " - "no destination was published") + b" is missing." in line): + if len(reasons) < 8: + reasons.append(line[:400].decode("utf-8", "replace").rstrip()) + if reasons: + raise LibraryError( + "seaf-fsck reported an incomplete export; no destination " + "was published: " + " | ".join(reasons)) children = list(out.iterdir()) if len(children) != 1 or not physical_dir(children[0]) or \ not children[0].name.startswith(item["repo_id"][:8] + "_"):