diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index a326ce6a..3654651d 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -55,6 +55,14 @@ jobs: python3 -m unittest discover -s tests -p test_build_receipt.py -q python3 -m unittest discover -s tests -p test_current_build_policy.py -q + - name: Test per-library offline CLI when changed (no services / no backup) + shell: bash + run: | + if git diff --name-only HEAD^ HEAD -- \ + tools/ops/seafile-library-offline.py tests/test_seafile_library_offline.py | grep -q .; then + python3 -m unittest discover -s tests -p test_seafile_library_offline.py -q + fi + - name: Test legacy CE recovery gate when changed (no backup) shell: bash run: | diff --git a/AGENTS.md b/AGENTS.md index 11517e82..a1390b5d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,3 +1,9 @@ +## 存储维护默认操作(2026-10-24) + +**只读、单库、Seafile 原生优先。** 存储识别请先使用 [Seafile 单库离线工具](tools/ops/README-seafile-library-offline.md) 的 `inspect --repo-id`;逻辑文件导出只在明确指定 `repo_id`、新的目标目录和原生 `seaf-fsck` 时执行;校验已导出文件使用显式 `verify-export`,是否全文件哈希由 `--hash` 再确认。默认构建、安装、容器启动、CI 和 Agent 操作都不得触发批量备份、文件导出或原生对象根复制。 + +`cf_recovery.py` 属于**另外的旧 CE 全实例离线灾备**,不是 CloudFile V2 正常单库工作流;不得调用它为 V2 自动备份,检测到 V2 数据时应失败拒绝,不能返回不完整快照的成功结果。遇到 S3/加密库、重复 UUID、多存储冲突,停下核查,不能猜目录或盲目重试。 + # AGENTS.md — cloudfile-docker > **当前版本规则**:`cloudfile-docker/dev` 就是 CloudFile **V2 当前开发主线**,以 Seafile CE14 为基础。与 `cloudfile-server/dev`、`cloudfile-hub/dev` 统一,不存在另一条“当前 V0.x dev”。本规则适用于 AI 开发 Agent 和维护者。 diff --git a/tests/test_seafile_library_offline.py b/tests/test_seafile_library_offline.py new file mode 100644 index 00000000..18226d5c --- /dev/null +++ b/tests/test_seafile_library_offline.py @@ -0,0 +1,316 @@ +"""Standalone offline Seafile library tests: no DB, Docker or CloudFile service.""" +import importlib.util +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +SCRIPT = Path(__file__).resolve().parents[1] / "tools" / "ops" / "seafile-library-offline.py" +spec = importlib.util.spec_from_file_location("offline", SCRIPT) +offline = importlib.util.module_from_spec(spec) +spec.loader.exec_module(offline) + +REPO = "f2af601c-8f44-4eb1-ac10-0402783d142d" + + +class OfflineLibraryTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.top = Path(self.tmp.name) + self.native = self.top / "seafile-data" + self.managed = self.top / "libraries" + self.native.mkdir() + self.managed.mkdir() + self.export_parent = self.top / "exports" + self.export_parent.mkdir() + self.fsck = self.top / "fake-fsck" + self.fsck.write_text("""#!/usr/bin/env python3 +from pathlib import Path +import sys +args = sys.argv[1:] +get = lambda k: args[args.index(k) + 1] +repo = args[-1] +data = Path(get('-d')) +assert (data / 'storage/commits' / repo).is_dir() +assert 'cloudfile_v2' not in (Path(get('-F')) / 'seafile.conf').read_text() +path = Path(get('--export')) / (repo[:8] + '_test_owner') +path.mkdir() +(path / 'file.txt').write_text('Hello Seafile!') +""") + self.fsck.chmod(0o700) + + def layout(self, root, *, key=None): + base = root / key if key else root + for kind in offline.KINDS: + (base / "storage" / kind / REPO).mkdir(parents=True) + (base / "storage/commits" / REPO / "ab").mkdir() + (base / "storage/commits" / REPO / "ab" / "cd").write_bytes(b"commit") + + def locate(self): + return offline.find_library(REPO, str(self.native), str(self.managed)) + + def add_fsck_output(self, *lines): + """Append print() lines to the fake fsck so they land on stdout.""" + self.fsck.write_text(self.fsck.read_text() + "".join( + "\nprint(%r)\n" % line for line in lines)) + + def export_cmd(self, output, fsck=None): + return [sys.executable, str(SCRIPT), "export", "--repo-id", REPO, + "--seafile-data", str(self.native), + "--library-root", str(self.managed), + "--seaf-fsck", str(fsck or self.fsck), + "--output-dir", str(output)] + + def bundle_fsck(self): + """Fake seaf-fsck installed as /seafile/bin/seaf-fsck. + + The script records the LD_LIBRARY_PATH it was started with, which is + how the shipped seaf-fsck.sh passes the bundled libraries. + """ + bundle = self.top / "seafile" + (bundle / "bin").mkdir(parents=True) + (bundle / "lib").mkdir() + (bundle / "lib64").mkdir() + fake = bundle / "bin" / "fake-fsck" + fake.write_text("""#!/usr/bin/env python3 +import os +from pathlib import Path +import sys +args = sys.argv[1:] +get = lambda k: args[args.index(k) + 1] +repo = args[-1] +assert (Path(get('-d')) / 'storage/commits' / repo).is_dir() +path = Path(get('--export')) / (repo[:8] + '_test_owner') +path.mkdir() +(path / 'file.txt').write_text('Hello Seafile!') +Path(__file__).with_name('ld_library_path.txt').write_text( + os.environ.get('LD_LIBRARY_PATH', '')) +""") + fake.chmod(0o700) + return fake, bundle / "bin" / "ld_library_path.txt" + + def test_seaf_fsck_bundle_libraries_are_prepended(self): + # seaf-fsck.sh exports SEAFILE_LD_LIBRARY_PATH; the direct call must + # provide the same sibling libraries or some builds fail to load. + self.layout(self.managed, key="managed-a") + fake, record = self.bundle_fsck() + output = self.export_parent / "bundle-libraries" + run = subprocess.run( + self.export_cmd(output, fsck=fake), capture_output=True, text=True, + env={**os.environ, "LD_LIBRARY_PATH": "/inherited"}) + self.assertEqual(0, run.returncode, run.stderr) + bundle = fake.parent.parent + expected = os.pathsep.join( + [str(bundle / "lib"), str(bundle / "lib64"), "/inherited"]) + self.assertEqual(expected, record.read_text()) + self.assertTrue((output / (REPO[:8] + "_test_owner") / "file.txt").exists()) + + def test_seaf_fsck_without_bundle_libraries_keeps_env(self): + self.layout(self.managed, key="managed-a") + fake, record = self.bundle_fsck() + (fake.parent.parent / "lib").rmdir() + (fake.parent.parent / "lib64").rmdir() + output = self.export_parent / "no-bundle-libraries" + run = subprocess.run( + self.export_cmd(output, fsck=fake), capture_output=True, text=True, + env={**os.environ, "LD_LIBRARY_PATH": "/only"}) + self.assertEqual(0, run.returncode, run.stderr) + self.assertEqual("/only", record.read_text()) + + def test_native_and_managed_layout_without_cloudfile_database(self): + self.layout(self.native) + self.assertEqual("seafile-native", self.locate()["backend"]) + import shutil + shutil.rmtree(self.native / "storage") + self.layout(self.managed, key="project-a") + self.assertEqual("auto-local", self.locate()["backend"]) + self.assertEqual("project-a", self.locate()["repo_key"]) + + def test_explicit_missing_managed_mount_never_falls_back(self): + self.layout(self.native) + missing = self.top / "missing-mount" + with self.assertRaisesRegex(offline.LibraryError, "storage root is not accessible"): + offline.find_library(REPO, str(self.native), str(missing)) + # A CE-only operator can omit the managed root instead of asking the + # tool to guess whether an explicitly named mount is missing. + self.assertEqual("seafile-native", + offline.find_library(REPO, str(self.native), None)["backend"]) + + def test_cli_inspect_native_ce_without_managed_root(self): + import json + self.layout(self.native) + proc = subprocess.run( + [sys.executable, str(SCRIPT), "inspect", "--repo-id", REPO, + "--seafile-data", str(self.native)], + capture_output=True, text=True) + self.assertEqual(0, proc.returncode, proc.stderr) + self.assertEqual("seafile-native", json.loads(proc.stdout)["backend"]) + + def test_ambiguous_and_missing_repo_rejected(self): + with self.assertRaisesRegex(offline.LibraryError, "not found"): + self.locate() + self.layout(self.managed, key="project-a") + self.layout(self.native) + with self.assertRaisesRegex(offline.LibraryError, "ambiguous"): + self.locate() + with self.assertRaisesRegex(offline.LibraryError, "complete UUID"): + offline.find_library("not-a-repo", str(self.native), str(self.managed)) + + def test_symlinks_never_rename_the_object_source(self): + target = self.top / "elsewhere" + (target / "storage/commits" / REPO).mkdir(parents=True) + (self.managed / "project-a").symlink_to(target) + with self.assertRaisesRegex(offline.LibraryError, "not found"): + self.locate() + (self.managed / "project-a").unlink() + alt = self.top / "managed-link" + alt.symlink_to(self.managed) + with self.assertRaisesRegex(offline.LibraryError, "parent must not be a symbolic"): + offline.find_library(REPO, str(self.native), str(alt)) + + def test_explicit_single_repo_export_and_verify(self): + self.layout(self.managed, key="custom-project") + output = self.export_parent / "repo-one" + cmd = ["export", "--repo-id", REPO, "--seafile-data", str(self.native), + "--library-root", str(self.managed), "--seaf-fsck", str(self.fsck), + "--output-dir", str(output)] + run = subprocess.run([sys.executable, str(SCRIPT)] + cmd, + capture_output=True, text=True) + self.assertEqual(0, run.returncode, run.stderr) + self.assertTrue((output / (REPO[:8] + "_test_owner") / "file.txt").exists()) + self.assertFalse((self.managed / "custom-project/tmpfiles").exists()) + self.assertEqual(b"commit", (self.managed / "custom-project/storage/commits" / REPO / "ab/cd").read_bytes()) + with self.assertRaisesRegex(offline.LibraryError, "exists"): + offline.export_library(type("A", (), dict( + repo_id=REPO, seafile_data=str(self.native), + library_root=str(self.managed), seaf_fsck=str(self.fsck), + output_dir=str(output)))) + summary = offline.verify_export(type("A", (), dict(repo_id=REPO, output_dir=str(output), hash=True))) + self.assertEqual(1, summary["file_count"]) + self.assertEqual(len(b"Hello Seafile!"), summary["total_bytes"]) + self.assertEqual(64, len(summary["tree_sha256"])) + + def test_verify_export_hash_is_directory_order_independent(self): + # A file-tree digest must be canonical: os.walk descends in readdir + # order, which differs between identical exports on other filesystems. + root = self.top / "hash-exports" + library = root / (REPO[:8] + "_library") + for name in ("zz-dir", "aa-dir", "mm-dir"): + (library / name).mkdir(parents=True) + (library / name / "file.txt").write_text(name) + args = type("A", (), dict(repo_id=REPO, output_dir=str(root), hash=True)) + + def fake_walk(order): + def walk(top, **kwargs): + base = Path(top) + directories = list(order) + yield str(base), directories, [] + # Mirrors os.walk: the caller's in-place sort of ``directories`` + # decides the child order. + for name in directories: + yield str(base / name), [], ["file.txt"] + return walk + + with mock.patch.object(os, "walk", fake_walk(("zz-dir", "aa-dir", "mm-dir"))): + first = offline.verify_export(args) + with mock.patch.object(os, "walk", fake_walk(("aa-dir", "zz-dir", "mm-dir"))): + second = offline.verify_export(args) + self.assertEqual(3, first["file_count"]) + self.assertEqual(first["tree_sha256"], second["tree_sha256"]) + + def test_native_export_partial_failure_never_publishes(self): + # Seafile CE export_file()/create_file() log some per-file errors + # without propagating an error exit status. The wrapper must refuse. + self.layout(self.managed, key="managed-a") + self.fsck.write_text(self.fsck.read_text() + + "\nprint('Failed to export file /missing/block')\n") + output = self.export_parent / "must-not-publish" + cmd = [sys.executable, str(SCRIPT), "export", "--repo-id", REPO, + "--seafile-data", str(self.native), + "--library-root", str(self.managed), + "--seaf-fsck", str(self.fsck), + "--output-dir", str(output)] + run = subprocess.run(cmd, capture_output=True, text=True) + self.assertNotEqual(0, run.returncode) + self.assertIn("incomplete export", run.stderr) + self.assertFalse(output.exists()) + self.assertTrue((self.managed / "managed-a/storage/commits" / REPO).is_dir()) + + def test_native_export_native_warning_never_publishes(self): + # Real CE14 output for an unreadable FS dir object: a single WARNING, + # then "Finish exporting", exit status 0 and a silently partial tree. + self.layout(self.managed, key="managed-a") + self.add_fsck_output( + "[2026-10-24 11:14:03] [WARNING] ../../common/fs-mgr.c(1895): " + "[fs mgr] Failed to read dir " + "f4b3106f4deb9c74fab57bcb8864c9fac3237411.") + output = self.export_parent / "warning-must-not-publish" + run = subprocess.run(self.export_cmd(output), capture_output=True, text=True) + self.assertNotEqual(0, run.returncode) + self.assertIn("incomplete export", run.stderr) + self.assertIn("Failed to read dir", run.stderr) # matched line is surfaced + self.assertFalse(output.exists()) + + def test_native_export_missing_dir_object_never_publishes(self): + # Real CE14 output when the root FS dir object is gone: an INFO + # "is missing." line and an empty library dir that would otherwise + # pass the exactly-one-export check with exit status 0. + self.layout(self.managed, key="managed-a") + self.add_fsck_output( + "[2026-10-24 11:14:08] [INFO] fsck.c(55): Dir " + "e4bdf4e07d7a8652dd1dba8747f22c938d96936c is missing.") + output = self.export_parent / "missing-dir-must-not-publish" + run = subprocess.run(self.export_cmd(output), capture_output=True, text=True) + self.assertNotEqual(0, run.returncode) + self.assertIn("incomplete export", run.stderr) + self.assertFalse(output.exists()) + + def test_native_export_open_file_failure_never_publishes(self): + # create_file()'s g_open() failure (e.g. ENOSPC, EACCES on the + # destination) is a WARNING only; that file is skipped, exit stays 0. + self.layout(self.managed, key="managed-a") + self.add_fsck_output( + "[2026-10-24 11:14:03] [WARNING] fsck.c(1117): Open file " + "/mnt/recovery/out/file.txt failed: No space left on device.") + output = self.export_parent / "no-space-must-not-publish" + run = subprocess.run(self.export_cmd(output), capture_output=True, text=True) + self.assertNotEqual(0, run.returncode) + self.assertIn("incomplete export", run.stderr) + self.assertFalse(output.exists()) + + def test_native_export_benign_progress_log_still_publishes(self): + # A normal export logs INFO progress only; detection must not reject it. + self.layout(self.managed, key="managed-a") + self.add_fsck_output( + "[2026-10-24 11:13:58] [INFO] fsck.c(1208): Scanning available " + "commits for repo " + REPO + "...", + "[2026-10-24 11:13:58] [INFO] fsck.c(1247): Find available commit " + "1d4999db(created at 2023-11-14 22:13:20), will export files from it.", + "[2026-10-24 11:13:58] [INFO] fsck.c(1283): Start to export files " + "for repo f2af601c(review-repo).", + "[2026-10-24 11:13:58] [INFO] fsck.c(1151): Export file " + "/out/f2af601c_test_owner/file.txt.", + "[2026-10-24 11:13:58] [INFO] fsck.c(1301): Finish exporting files " + "for repo f2af601c.") + output = self.export_parent / "benign-log-publishes" + run = subprocess.run(self.export_cmd(output), capture_output=True, text=True) + self.assertEqual(0, run.returncode, run.stderr) + self.assertTrue((output / (REPO[:8] + "_test_owner") / "file.txt").exists()) + + def test_inspect_does_not_export_or_copy_all_libraries(self): + self.layout(self.native) + run = subprocess.run([sys.executable, str(SCRIPT), "inspect", + "--seafile-data", str(self.native), + "--library-root", str(self.managed)], capture_output=True) + self.assertNotEqual(0, run.returncode) # exact UUID is mandatory + self.assertEqual([], list(self.export_parent.iterdir())) + self.assertEqual("seafile-native", self.locate()["backend"]) + self.assertEqual([], list(self.export_parent.iterdir())) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ops/README-seafile-library-offline.md b/tools/ops/README-seafile-library-offline.md new file mode 100644 index 00000000..497d9ebb --- /dev/null +++ b/tools/ops/README-seafile-library-offline.md @@ -0,0 +1,75 @@ +# Seafile 单库离线检查与导出(无需 CloudFile 服务) + +本工具只负责**定位某一个** Seafile CE/CloudFile 库的物理对象目录,以及显式调用 Seafile 原生 `seaf-fsck --export` 导出其逻辑文件。无需 Hub、CloudFile Web/API、`cf_library_storage` SQL,也不建立新的存储元数据或同步引擎。 + +**默认不备份、不复制、不扫描全库内容。** 所有操作都必须传入**完整且唯一的 `repo_id`**,没有 `--all`。此工具也不会通过定时器、Compose 启动脚本自动运行。 + +## 存储结构 + +Seafile 原生老库: + +```text +/storage/{commits,fs,blocks}//... +``` + +CloudFile V2 每库本地目录: + +```text +//storage/{commits,fs,blocks}//... +``` + +`repo_key` 可以是与 `repo_id` 不同的受控名称。离线定位只检查一级库目录中的 `storage/commits/`,不需要知道 `cf_library_storage` 内容。如果同一 UUID 同时在原生目录和多个库目录中出现,**拒绝自动选择**,由运维人工核对。这些目录是 Seafile 对象存储,**不是用户文件目录**。 + +## 1. 只读定位(推荐的第一步) + +```sh +python3 tools/ops/seafile-library-offline.py inspect \ + --repo-id \ + --seafile-data /shared/seafile/seafile-data \ + --library-root /shared/seafile/library-storage +``` + +只枚举一级目录并核对 Commit/FS/Block 根,不调用服务、不读数据库、不导出任何字节,不更改文件。**只有旧版 CE 原生库时,可以省略 `--library-root`;一旦显式提供该参数,目录不存在或挂载丢失必须报错,不得静默回退到旧库目录。** 它不是对象哈希完整性检查:完整修复/检查应按照 Seafile 原生工具的约束另行执行。 + +## 2. 显式导出单个库(耗时、耗磁盘空间) + +**操作前确保目标磁盘有足够空间,使用未加密库并明确选择一个 `repo_id` 和全新目标目录。** + +```sh +python3 tools/ops/seafile-library-offline.py export \ + --repo-id \ + --seafile-data /shared/seafile/seafile-data \ + --library-root /shared/seafile/library-storage \ + --seaf-fsck /opt/seafile/seafile-server-latest/seafile/bin/seaf-fsck \ + --output-dir /mnt/recovery/ +``` + +此操作: +1. 先按 Seafile 物理目录确定唯一库,不使用 CloudFile 服务或 SQL。 +2. 在目标磁盘创建临时、只用于本次操作的原生 Seafile 配置和对象视图;通过符号链接**引用**该库的 Commit/FS/Block 目录,不复制它们。 +3. 调用 **Seafile 原生** `seaf-fsck -d ... -F ... --export ... `。原生工具读取 Commit/FS/Block,生成普通文件。 +4. 只有在**全部**条件满足时才发布结果:原生工具退出码为 0、日志中没有 `[WARNING]`/`[ERROR]` 记录、没有已知失败标记(`Failed to export file`、`Failed to mkdir`、`Dir/File ... is missing.` 等)、且恰好产出一个库目录。否则整体失败:不创建/不保留目标目录,返回非零退出码,并在 stderr 附带命中的原生日志行;目标目录已存在时同样拒绝覆盖。 + +独立的离线临时配置只使用原生 filesystem backend(去掉 `cloudfile_v2` 路由层);**不修改原来服务器的 `seafile.conf`**。需要可执行的 `seaf-fsck` 以及适合该二进制的动态库环境。**本工具直接执行所给的 `seaf-fsck` 二进制,不经过发行包的 `seaf-fsck.sh` 包装脚本**;与包装脚本一致,它会自动把该二进制同级安装树中的 `seafile/lib`、`seafile/lib64` 前置到 `LD_LIBRARY_PATH`(保留调用方已有值;二进制不在安装树中时保持原环境),因此 14.0.8 这类依赖随包 `libevent` 的镜像也可直接执行。若二进制位于非标准位置或仍有库依赖缺失,需由调用方自行设置 `LD_LIBRARY_PATH`。加密库或损坏库可能无法成功导出;即便返回成功,也不能据此保证所有历史版本都已恢复。Seafile 原生导出读取选出的可用 Commit,而不是完整的数据库备份。 + +原生导出只在日志里报告**部分**内部失败(C 层 `export_file()` 为 `void`:缺块、目录对象读不到、目标文件无法创建,都可能只留一行日志后继续)。本工具按日志级别与已知失败标记判断,无法保证覆盖未来版本新增的静默失败;**成功导出仍不等于内容完整**,完整性仍需 `verify-export` 与官方 per-repo fsck。 + +## 3. 显式检查已导出的文件 + +```sh +python3 tools/ops/seafile-library-offline.py verify-export \ + --repo-id \ + --output-dir /mnt/recovery/ +``` + +这会读取目录树元数据并计算文件数量/总字节数;如果确有需要再传 `--hash`,逐文件读完整内容生成树校验摘要,可能消耗较长时间。 + +**检查导出文件 ≠ 证明全部原对象完整**。完整性验收仍可按 Seafile 官方的 per-repo `seaf-fsck` 检查/修复流程进行,涉及库 DB 与对象健康时按其要求执行。官方说明:https://manual.seafile.com/14.0/administration/seafile_fsck/ + +## 4. 严格限制与独立职责 + +- 不提供无库 ID 的批量导出;不启动自动备份、定时同步、全盘复制。 +- 不直接读写用户逻辑文件到 Seafile 对象目录;导出的文件只能进入单独的新目标目录。 +- 不直接修复/变更存储映射;`RepoStorageId` 仍是在线后端路由事实。 +- 只支持**本地 filesystem** 的原生布局和 auto-local;S3/RustFS/MinIO 需要其独立工具及配置,绝不猜路径或回退到错误本地目录。 +- 现有 `cf_recovery.py` 是**另一个显式调用的、全实例离线备份工具**,不能视为普通的单库导出,也不得加入自动启动/发布路径;如果存在 auto-local/S3,应重新审查其完整备份范围,避免漏数据或误报备份成功。 diff --git a/tools/ops/seafile-library-offline.py b/tools/ops/seafile-library-offline.py new file mode 100644 index 00000000..83682c81 --- /dev/null +++ b/tools/ops/seafile-library-offline.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""Read-only, service-independent Seafile library discovery and explicit export. + +A library is stored in the standard Seafile object layout under either the +native data root or one managed per-library directory. No CloudFile SQL or Hub +API is consulted. Logical files are exported by the original seaf-fsck binary; +this tool never interprets Seafile Commit/FS/Block contents itself. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import sys +import tempfile +from uuid import UUID + +KINDS = ("commits", "fs", "blocks") +KEY = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,78}[a-z0-9])?\Z") + + +class LibraryError(Exception): + pass + + +def repository_id(raw): + try: + value = UUID(raw) + except (TypeError, ValueError, AttributeError) as exc: + raise LibraryError("--repo-id must be a complete UUID") from exc + if str(value) != raw: + raise LibraryError("--repo-id must be a canonical lowercase UUID") + return raw + + +def real_dir(path): + """Require mounted roots; never treat an explicitly supplied missing mount as absent.""" + path = Path(path) + if not path.is_absolute(): + raise LibraryError("storage roots must be absolute") + try: + root = path.resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise LibraryError("storage root is not accessible") from exc + if not root.is_dir(): + raise LibraryError("storage root must be a directory") + return root + + +def physical_dir(path): + """Return true only for a real directory, never a symlink (no mutation).""" + try: + return stat.S_ISDIR(path.lstat().st_mode) + except FileNotFoundError: + return False + + +def candidate(root, repo_id, backend, key=None): + if root is None: + return None + base = root / "storage" + if not physical_dir(base): + return None + commit_parent = base / "commits" + if not physical_dir(commit_parent): + return None + commits = commit_parent / repo_id + if not physical_dir(commits): + return None + paths = {} + for kind in KINDS: + parent = base / kind + if parent.exists() or parent.is_symlink(): + if not physical_dir(parent): + raise LibraryError("unsafe Seafile object parent for " + kind) + path = parent / repo_id + if path.exists() or path.is_symlink(): + if not physical_dir(path): + raise LibraryError("unsafe Seafile library object directory for " + kind) + paths[kind] = str(path) + else: + paths[kind] = None # A newly created/empty library can omit fs/blocks. + return {"repo_id": repo_id, "backend": backend, "repo_key": key, + "seafile_data_root": str(root), "object_dirs": paths} + + +def find_library(repo_id, native_root, managed_root): + repo_id = repository_id(repo_id) + native = real_dir(native_root) + managed = real_dir(managed_root) if managed_root is not None else None + if managed is not None and Path(managed_root).is_symlink(): + raise LibraryError("managed parent must not be a symbolic link") + found = [] + item = candidate(native, repo_id, "seafile-native") + if item is not None: + found.append(item) + if managed is not None: + for child in sorted(managed.iterdir()): + if not KEY.fullmatch(child.name): + continue + if not physical_dir(child): + continue # Never traverse links masquerading as libraries. + item = candidate(child, repo_id, "auto-local", child.name) + if item is not None: + found.append(item) + if not found: + raise LibraryError("repo_id was not found in the native or managed object directories") + if len(found) != 1: + raise LibraryError("ambiguous repo_id in multiple physical stores; inspect manually") + return found[0] + + +def output_directory(raw, sources): + target = Path(raw) + if not target.is_absolute(): + raise LibraryError("export destination must be an absolute path") + if target.exists() or target.is_symlink(): + raise LibraryError("export destination already exists; overwrite is not supported") + parent = target.parent.resolve(strict=True) + if not parent.is_dir() or not physical_dir(parent): + raise LibraryError("export destination parent must be a real existing directory") + dest = parent / target.name + for root in sources: + source = Path(root).resolve() + if source == dest or source in dest.parents or dest in source.parents: + raise LibraryError("export directory must not overlap an object storage root") + return dest + + +def export_library(args): + item = find_library(args.repo_id, args.seafile_data, args.library_root) + fsck = Path(args.seaf_fsck) + if not fsck.is_absolute() or not fsck.is_file() or not os.access(fsck, os.X_OK): + raise LibraryError("--seaf-fsck must name an executable Seafile fsck binary") + roots = (args.seafile_data,) + ((args.library_root,) if args.library_root else ()) + target = output_directory(args.output_dir, roots) + # Temp staging lives on the destination volume, not in /tmp and not in the + # object store. Failed export is deleted; no target is published on error. + with tempfile.TemporaryDirectory(prefix=".seafile-export-", dir=target.parent) as temporary: + sandbox = Path(temporary) + data = sandbox / "seafile-data" + conf = sandbox / "conf" + out = sandbox / "out" + conf.mkdir() + (data / "tmpfiles").mkdir(parents=True) + out.mkdir() + (conf / "seafile.conf").write_text( + "[commit_object_backend]\nname = filesystem\n" + "[fs_object_backend]\nname = filesystem\n" + "[block_backend]\nname = filesystem\n", encoding="utf-8") + for kind in KINDS: + parent = data / "storage" / kind + parent.mkdir(parents=True) + original = item["object_dirs"][kind] + target_dir = parent / item["repo_id"] + if original is not None: + target_dir.symlink_to(original, target_is_directory=True) + else: + target_dir.mkdir() # No source-tree writes for an empty kind. + # Stock Seafile CE filesystem routing, without cloudfile_v2 or a DB. + command = [str(fsck), "-c", str(conf), "-d", str(data), "-F", str(conf), + "--export", str(out), item["repo_id"]] + # The shipped seaf-fsck.sh exports SEAFILE_LD_LIBRARY_PATH before + # calling the binary. A direct call must do the same: some builds + # (e.g. 14.0.8 images) link libevent/libsearpc from seafile/lib{,64} + # and fail to load without it. Prepend the sibling libraries of the + # given binary; keep any inherited value and leave env untouched when + # that layout does not exist (system loader paths still apply). + environment = dict(os.environ) + bundle = fsck.parent.parent + library_dirs = [path for path in (bundle / "lib", bundle / "lib64") + if path.is_dir()] + if library_dirs: + prefix = os.pathsep.join(str(path) for path in library_dirs) + inherited = environment.get("LD_LIBRARY_PATH") + environment["LD_LIBRARY_PATH"] = ( + prefix + (os.pathsep + inherited if inherited else "")) + # Native export may print thousands of file messages; do not keep all + # of them in RAM just to report the last error if it fails. + with tempfile.TemporaryFile(dir=sandbox) as output_log: + result = subprocess.run(command, cwd=sandbox, env=environment, + stdout=output_log, stderr=subprocess.STDOUT) + if result.returncode != 0: + output_log.seek(0, os.SEEK_END) + length = output_log.tell() + output_log.seek(max(0, length - 2048)) + tail = output_log.read().decode("utf-8", "replace") + raise LibraryError("seaf-fsck export failed: " + tail[-1500:]) + # CE's export_file()/create_file() are void: they can log a + # per-file failure, skip a whole subtree when an FS dir object is + # unreadable, or fail to create a destination file, yet still print + # "Finish exporting files" and exit 0. Never publish such a partial + # result. Fail closed unless the native log holds only normal + # progress: a WARNING/ERROR record or a known failure signature + # means the tree cannot be claimed complete. Banner lines are + # matched regardless of locale (CE log records are English) and + # the scan is incremental so a huge export log is never held in RAM. + output_log.seek(0) + reasons = [] + for line in output_log: + if (b"[WARNING]" in line or b"[ERROR]" in line or + b"Failed to export file " in line or + b"Failed to mkdir " in line or + b" is missing." in line): + if len(reasons) < 8: + reasons.append(line[:400].decode("utf-8", "replace").rstrip()) + if reasons: + raise LibraryError( + "seaf-fsck reported an incomplete export; no destination " + "was published: " + " | ".join(reasons)) + children = list(out.iterdir()) + if len(children) != 1 or not physical_dir(children[0]) or \ + not children[0].name.startswith(item["repo_id"][:8] + "_"): + raise LibraryError("seaf-fsck did not produce exactly one library export " + "(encrypted, empty/corrupt or unsupported library)") + # Only the requested repo's result is published. No source bytes are + # copied by inspect and no bulk backup is ever implicit. + if target.exists() or target.is_symlink(): + raise LibraryError("export destination was created concurrently") + target.mkdir(mode=0o700) # Atomic no-overwrite reservation. + try: + children[0].rename(target / children[0].name) + except Exception: + target.rmdir() + raise + return {"action": "export", "repo_id": item["repo_id"], + "backend": item["backend"], "export_dir": str(target), + "status": "EXPORTED_UNVERIFIED", + "warning": "Export is not a database backup or proof of full source integrity; " + "run verify-export explicitly when needed."} + + +def verify_export(args): + repo_id = repository_id(args.repo_id) + root = real_dir(args.output_dir) + libraries = list(root.iterdir()) + if len(libraries) != 1 or not physical_dir(libraries[0]) or \ + not libraries[0].name.startswith(repo_id[:8] + "_"): + raise LibraryError("export directory must contain one library matching repo_id") + base = libraries[0] + count = 0 + total = 0 + digest = hashlib.sha256() if args.hash else None + for here, directories, names in os.walk(base, followlinks=False): + # Canonical order: os.walk descends in readdir order, so a tree digest + # would differ between identical exports on another filesystem/copy + # unless directory order is pinned before recursing. + directories.sort() + for name in directories: + if not physical_dir(Path(here) / name): + raise LibraryError("export contains symlink or unsupported directory") + for name in sorted(names): + p = Path(here) / name + if not stat.S_ISREG(p.lstat().st_mode): + raise LibraryError("export contains symlink or non-regular file") + count += 1 + total += p.stat().st_size + if digest is not None: + relative = p.relative_to(base).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(4, "big") + relative) + filehash = hashlib.sha256() + with p.open("rb") as stream: + for piece in iter(lambda: stream.read(1024 * 1024), b""): + filehash.update(piece) + digest.update(filehash.digest()) + return {"action": "verify-export", "repo_id": repo_id, "export_dir": str(root), + "file_count": count, "total_bytes": total, "tree_sha256": digest.hexdigest() if digest else None, + "note": "Local export inventory and readability only; source completeness is not proven."} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="action", required=True) + for op in ("inspect", "export", "verify-export"): + cli = commands.add_parser(op) + cli.add_argument("--repo-id", required=True, help="one exact Seafile UUID; never all libraries") + if op != "verify-export": + cli.add_argument("--seafile-data", required=True, help="absolute path to native seafile-data") + cli.add_argument("--library-root", help="existing absolute managed root; omit for native CE only") + if op == "export": + cli.add_argument("--seaf-fsck", required=True, help="absolute path to native seaf-fsck executable") + cli.add_argument("--output-dir", required=True, help="unused absolute destination directory") + if op == "verify-export": + cli.add_argument("--output-dir", required=True, help="absolute prior export directory") + cli.add_argument("--hash", action="store_true", help="explicitly hash all exported file bytes") + args = parser.parse_args(argv) + try: + if args.action == "inspect": + result = {"action": "inspect", "status": "FOUND", **find_library(args.repo_id, args.seafile_data, args.library_root)} + elif args.action == "export": + result = export_library(args) + else: + result = verify_export(args) + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + return 0 + except (LibraryError, OSError, subprocess.SubprocessError) as exc: + print("seafile-library-offline: " + str(exc), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main())