From 39ac734b9c89ef330fc2ffe98c543cbc5b631a87 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 30 May 2026 01:57:01 +0200 Subject: [PATCH 1/4] support "rest:" repository URLs, fixes #9593 That is borgstore's REST http over stdio (over ssh, if a host is given). --- docs/usage/general/repository-urls.rst.inc | 8 ++- src/borg/archiver/_common.py | 2 +- src/borg/helpers/parseformat.py | 23 ++++++- .../testsuite/helpers/parseformat_test.py | 65 +++++++++++++++++++ 4 files changed, 95 insertions(+), 3 deletions(-) diff --git a/docs/usage/general/repository-urls.rst.inc b/docs/usage/general/repository-urls.rst.inc index 238598bd59..a53bb4de90 100644 --- a/docs/usage/general/repository-urls.rst.inc +++ b/docs/usage/general/repository-urls.rst.inc @@ -12,7 +12,13 @@ expanded by your shell). Note: You may also prepend ``file://`` to a filesystem path to use URL style. -**Remote repositories** accessed via SSH user@host: +**Remote repositories** accessed via SSH user@host (REST http over stdio): + +``rest://user@host:port//abs/path/to/repo`` — absolute path + +``rest://user@host:port/rel/path/to/repo`` — path relative to the current directory + +**Remote repositories** accessed via SSH user@host (legacy borg RPC protocol): ``ssh://user@host:port//abs/path/to/repo`` — absolute path diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 7aba725e28..c2642155cb 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -42,7 +42,7 @@ def get_repository(location, *, create, exclusive, lock_wait, lock, args, v1_leg ) elif ( - location.proto in ("sftp", "file", "http", "https", "rclone", "s3", "b2") and not v1_legacy + location.proto in ("rest", "sftp", "file", "http", "https", "rclone", "s3", "b2") and not v1_legacy ): # stuff directly supported by borgstore repository = Repository(location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock) diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index d70eb5781d..c8b6c417b3 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -552,6 +552,19 @@ class Location: re.VERBOSE, ) + # REST http via stdio (via ssh, if host given): + rest_re = re.compile( + r"(?P(rest))://" + + r"(" + + optional_user_re + + host_re + + optional_port_re + + r")?" + + r"/" # this is the separator, not part of the path! + + abs_or_rel_path_re, + re.VERBOSE, + ) + # BorgStore REST server # (http|https)://user:pass@host:port/ http_re = re.compile( @@ -624,6 +637,14 @@ def parse(self, text, overrides={}): def _parse(self, text): m = self.ssh_or_sftp_re.match(text) + if m: + self.proto = m.group("proto") + self.user = m.group("user") + self._host = m.group("host") + self.port = m.group("port") and int(m.group("port")) or None + self.path = os.path.normpath(m.group("path")) + return True + m = self.rest_re.match(text) if m: self.proto = m.group("proto") self.user = m.group("user") @@ -692,7 +713,7 @@ def canonical_path(self): return self.path if self.proto == "rclone": return f"{self.proto}:{self.path}" - if self.proto in ("sftp", "ssh", "s3", "b2", "http", "https"): + if self.proto in ("rest", "sftp", "ssh", "s3", "b2", "http", "https"): return ( f"{self.proto}://" f"{(self.user + '@') if self.user else ''}" diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index 0c9281f08f..0d6ab162c7 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -133,6 +133,71 @@ def test_ssh(self, monkeypatch): "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='relative/path')" ) + def test_rest(self, monkeypatch): + monkeypatch.delenv("BORG_REPO", raising=False) + assert ( + repr(Location("rest://user@host:1234//absolute/path")) + == "Location(proto='rest', user='user', pass=None, host='host', port=1234, path='/absolute/path')" + ) + assert ( + repr(Location("rest://user@host:1234/relative/path")) + == "Location(proto='rest', user='user', pass=None, host='host', port=1234, path='relative/path')" + ) + assert ( + repr(Location("rest://user@host/relative/path")) + == "Location(proto='rest', user='user', pass=None, host='host', port=None, path='relative/path')" + ) + assert ( + repr(Location("rest://user@[::]:1234/relative/path")) + == "Location(proto='rest', user='user', pass=None, host='::', port=1234, path='relative/path')" + ) + assert ( + repr(Location("rest://user@[::]/relative/path")) + == "Location(proto='rest', user='user', pass=None, host='::', port=None, path='relative/path')" + ) + assert ( + repr(Location("rest://user@[2001:db8::]:1234/relative/path")) + == "Location(proto='rest', user='user', pass=None, host='2001:db8::', port=1234, path='relative/path')" + ) + assert ( + repr(Location("rest://user@[2001:db8::]/relative/path")) + == "Location(proto='rest', user='user', pass=None, host='2001:db8::', port=None, path='relative/path')" + ) + assert ( + repr(Location("rest://user@[2001:db8::c0:ffee]:1234/relative/path")) + == "Location(proto='rest', user='user', pass=None, host='2001:db8::c0:ffee', port=1234, path='relative/path')" # noqa: E501 + ) + assert ( + repr(Location("rest://user@[2001:db8::c0:ffee]/relative/path")) + == "Location(proto='rest', user='user', pass=None, host='2001:db8::c0:ffee', port=None, path='relative/path')" # noqa: E501 + ) + assert ( + repr(Location("rest://user@[2001:db8::192.0.2.1]:1234/relative/path")) + == "Location(proto='rest', user='user', pass=None, host='2001:db8::192.0.2.1', port=1234, path='relative/path')" # noqa: E501 + ) + assert ( + repr(Location("rest://user@[2001:db8::192.0.2.1]/relative/path")) + == "Location(proto='rest', user='user', pass=None, host='2001:db8::192.0.2.1', port=None, path='relative/path')" # noqa: E501 + ) + assert ( + repr(Location("rest://user@[2a02:0001:0002:0003:0004:0005:0006:0007]/relative/path")) + == "Location(proto='rest', user='user', pass=None, " + "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=None, path='relative/path')" + ) + assert ( + repr(Location("rest://user@[2a02:0001:0002:0003:0004:0005:0006:0007]:1234/relative/path")) + == "Location(proto='rest', user='user', pass=None, " + "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='relative/path')" + ) + assert ( + repr(Location("rest:///relative/path")) + == "Location(proto='rest', user=None, pass=None, host=None, port=None, path='relative/path')" + ) + assert ( + repr(Location("rest:////absolute/path")) + == "Location(proto='rest', user=None, pass=None, host=None, port=None, path='/absolute/path')" + ) + def test_s3(self, monkeypatch): monkeypatch.delenv("BORG_REPO", raising=False) assert ( From f53dd6c3c452bf551ae46de610fa920ab0581304 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 1 Jun 2026 23:58:28 +0200 Subject: [PATCH 2/4] remote archiver tests: use rest:/// rather than ssh:// --- src/borg/conftest.py | 4 ++-- src/borg/testsuite/archiver/__init__.py | 5 ++++- src/borg/testsuite/archiver/checks_test.py | 4 ++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/borg/conftest.py b/src/borg/conftest.py index 074ace1793..e9ea95f61b 100644 --- a/src/borg/conftest.py +++ b/src/borg/conftest.py @@ -96,7 +96,7 @@ def __init__(self): self.patterns_file_path: str | None = None def get_kind(self) -> str: - if self.repository_location.startswith("ssh://__testsuite__"): + if self.repository_location.startswith("ssh://__testsuite__") or self.repository_location.startswith("rest://"): return "remote" elif self.EXE == "borg.exe": return "binary" @@ -152,7 +152,7 @@ def maybe_clear_flags_and_retry(func, path, _exc_info): @pytest.fixture() def remote_archiver(archiver): - archiver.repository_location = "ssh://__testsuite__/" + str(archiver.repository_path) + archiver.repository_location = "rest://" + "/" + str(archiver.repository_path) yield archiver diff --git a/src/borg/testsuite/archiver/__init__.py b/src/borg/testsuite/archiver/__init__.py index ebb27f7904..3169844e4f 100644 --- a/src/borg/testsuite/archiver/__init__.py +++ b/src/borg/testsuite/archiver/__init__.py @@ -178,7 +178,10 @@ def open_archive(repo_path, name): def open_repository(archiver): if archiver.get_kind() == "remote": - return RemoteRepository(Location(archiver.repository_location)) + location = Location(archiver.repository_location) + if location.proto == "rest": + return Repository(location, exclusive=True) + return RemoteRepository(location) else: return Repository(archiver.repository_path, exclusive=True) diff --git a/src/borg/testsuite/archiver/checks_test.py b/src/borg/testsuite/archiver/checks_test.py index eff167027f..88efa31c50 100644 --- a/src/borg/testsuite/archiver/checks_test.py +++ b/src/borg/testsuite/archiver/checks_test.py @@ -278,6 +278,8 @@ def test_unknown_mandatory_feature_in_cache(archivers, request): # Begin Remote Tests def test_remote_repo_restrict_to_path(remote_archiver): + if remote_archiver.repository_location.startswith("rest://"): + pytest.skip("Not applicable for rest:// protocol") original_location, repo_path = remote_archiver.repository_location, remote_archiver.repository_path # restricted to repo directory itself: with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-path", repo_path]): @@ -306,6 +308,8 @@ def test_remote_repo_restrict_to_path(remote_archiver): def test_remote_repo_restrict_to_repository(remote_archiver): + if remote_archiver.repository_location.startswith("rest://"): + pytest.skip("Not applicable for rest:// protocol") repo_path = remote_archiver.repository_path # restricted to repo directory itself: with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-repository", repo_path]): From 8dbfc50cde9c5042e6f3b9d24bb02522f89a44cc Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 2 Jun 2026 00:54:36 +0200 Subject: [PATCH 3/4] require borgstore 0.5.1, always with rest extra --- .github/workflows/canary.yml | 2 +- .github/workflows/ci.yml | 12 +++++------ pyproject.toml | 39 ++++++++++++++++++------------------ 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index cfc57ec88f..0d100c35ef 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -136,7 +136,7 @@ jobs: run: | # build borg.exe . env/bin/activate - pip install -e ".[cockpit,s3,sftp,rest,rclone]" + pip install -e ".[cockpit,s3,sftp,rclone]" mkdir -p dist/binary pyinstaller -y --clean --distpath=dist/binary scripts/borg.exe.spec # build sdist and wheel in dist/... diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 464ad960e1..c997495b27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -281,13 +281,13 @@ jobs: - name: Install borgbackup run: | if [[ "$TOXENV" == *"llfuse"* ]]; then - pip install -ve ".[llfuse,cockpit,s3,sftp,rest,rclone]" + pip install -ve ".[llfuse,cockpit,s3,sftp,rclone]" elif [[ "$TOXENV" == *"pyfuse3"* ]]; then - pip install -ve ".[pyfuse3,cockpit,s3,sftp,rest,rclone]" + pip install -ve ".[pyfuse3,cockpit,s3,sftp,rclone]" elif [[ "$TOXENV" == *"mfusepy"* ]]; then - pip install -ve ".[mfusepy,cockpit,s3,sftp,rest,rclone]" + pip install -ve ".[mfusepy,cockpit,s3,sftp,rclone]" else - pip install -ve ".[cockpit,s3,sftp,rest,rclone]" + pip install -ve ".[cockpit,s3,sftp,rclone]" fi - name: Build Borg fat binaries (${{ matrix.binary }}) @@ -461,7 +461,7 @@ jobs: pip -V python -m pip install --upgrade pip wheel pip install -r requirements.d/development.lock.txt - pip install -e ".[mfusepy,cockpit,s3,sftp,rest,rclone]" + pip install -e ".[mfusepy,cockpit,s3,sftp,rclone]" tox -e py311-mfusepy if [[ "${{ matrix.do_binaries }}" == "true" && "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then @@ -659,7 +659,7 @@ jobs: run: | # build borg.exe . env/bin/activate - pip install -e ".[cockpit,s3,sftp,rest,rclone]" + pip install -e ".[cockpit,s3,sftp,rclone]" mkdir -p dist/binary pyinstaller -y --clean --distpath=dist/binary scripts/borg.exe.spec # build sdist and wheel in dist/... diff --git a/pyproject.toml b/pyproject.toml index a0e7d1b8f6..e66d11ce21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ license = "BSD-3-Clause" license-files = ["LICENSE", "AUTHORS"] dependencies = [ "borghash ~= 0.1.0", - "borgstore ~= 0.5.0", + "borgstore[rest] ~= 0.5.1", "msgpack >=1.0.3, <=1.1.2", "packaging", "platformdirs >=3.0.0, <5.0.0; sys_platform == 'darwin'", # for macOS: breaking changes in 3.0.0. @@ -51,10 +51,9 @@ mfusepy = ["mfusepy >= 3.1.0, <4.0.0"] # fuse 2+3, high-level # a pypi release of borgbackup can't contain a dependency on github! # mfusepym = ["mfusepy @ git+https://github.com/mxmlnkn/mfusepy.git@master"] nofuse = [] -s3 = ["borgstore[s3] ~= 0.5.0"] -sftp = ["borgstore[sftp] ~= 0.5.0"] -rclone = ["borgstore[rclone] ~= 0.5.0"] -rest = ["borgstore[rest] ~= 0.5.0"] +s3 = ["borgstore[rest,s3] ~= 0.5.1"] +sftp = ["borgstore[rest,sftp] ~= 0.5.1"] +rclone = ["borgstore[rest,rclone] ~= 0.5.1"] cockpit = ["textual>=6.8.0"] # might also work with older versions, untested [project.urls] @@ -192,71 +191,71 @@ pass_env = ["*"] # needed by tox4, so env vars are visible for building borg [tool.tox.env.py310-llfuse] set_env = {BORG_FUSE_IMPL = "llfuse"} -extras = ["llfuse", "sftp", "s3", "rest", "rclone"] +extras = ["llfuse", "sftp", "s3", "rclone"] [tool.tox.env.py310-pyfuse3] set_env = {BORG_FUSE_IMPL = "pyfuse3"} -extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"] +extras = ["pyfuse3", "sftp", "s3", "rclone"] [tool.tox.env.py310-mfusepy] set_env = {BORG_FUSE_IMPL = "mfusepy"} -extras = ["mfusepy", "sftp", "s3", "rest", "rclone"] +extras = ["mfusepy", "sftp", "s3", "rclone"] [tool.tox.env.py311-none] [tool.tox.env.py311-llfuse] set_env = {BORG_FUSE_IMPL = "llfuse"} -extras = ["llfuse", "sftp", "s3", "rest", "rclone"] +extras = ["llfuse", "sftp", "s3", "rclone"] [tool.tox.env.py311-pyfuse3] set_env = {BORG_FUSE_IMPL = "pyfuse3"} -extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"] +extras = ["pyfuse3", "sftp", "s3", "rclone"] [tool.tox.env.py311-mfusepy] set_env = {BORG_FUSE_IMPL = "mfusepy"} -extras = ["mfusepy", "sftp", "s3", "rest", "rclone"] +extras = ["mfusepy", "sftp", "s3", "rclone"] [tool.tox.env.py312-none] [tool.tox.env.py312-llfuse] set_env = {BORG_FUSE_IMPL = "llfuse"} -extras = ["llfuse", "sftp", "s3", "rest", "rclone"] +extras = ["llfuse", "sftp", "s3", "rclone"] [tool.tox.env.py312-pyfuse3] set_env = {BORG_FUSE_IMPL = "pyfuse3"} -extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"] +extras = ["pyfuse3", "sftp", "s3", "rclone"] [tool.tox.env.py312-mfusepy] set_env = {BORG_FUSE_IMPL = "mfusepy"} -extras = ["mfusepy", "sftp", "s3", "rest", "rclone"] +extras = ["mfusepy", "sftp", "s3", "rclone"] [tool.tox.env.py313-none] [tool.tox.env.py313-llfuse] set_env = {BORG_FUSE_IMPL = "llfuse"} -extras = ["llfuse", "sftp", "s3", "rest", "rclone"] +extras = ["llfuse", "sftp", "s3", "rclone"] [tool.tox.env.py313-pyfuse3] set_env = {BORG_FUSE_IMPL = "pyfuse3"} -extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"] +extras = ["pyfuse3", "sftp", "s3", "rclone"] [tool.tox.env.py313-mfusepy] set_env = {BORG_FUSE_IMPL = "mfusepy"} -extras = ["mfusepy", "sftp", "s3", "rest", "rclone"] +extras = ["mfusepy", "sftp", "s3", "rclone"] [tool.tox.env.py314-none] [tool.tox.env.py314-llfuse] set_env = {BORG_FUSE_IMPL = "llfuse"} -extras = ["llfuse", "sftp", "s3", "rest", "rclone"] +extras = ["llfuse", "sftp", "s3", "rclone"] [tool.tox.env.py314-pyfuse3] set_env = {BORG_FUSE_IMPL = "pyfuse3"} -extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"] +extras = ["pyfuse3", "sftp", "s3", "rclone"] [tool.tox.env.py314-mfusepy] set_env = {BORG_FUSE_IMPL = "mfusepy"} -extras = ["mfusepy", "sftp", "s3", "rest", "rclone"] +extras = ["mfusepy", "sftp", "s3", "rclone"] [tool.tox.env.ruff] skip_install = true From 5584bff830db9ae6cb96ae9400bb039e66bb6a6f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 2 Jun 2026 01:12:19 +0200 Subject: [PATCH 4/4] CI: add test_rest_repo_basics tests to localhost openssh server --- .github/workflows/ci.yml | 5 ++++ .../testsuite/archiver/remote_repo_test.py | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c997495b27..a24be3d595 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,8 +243,13 @@ jobs: # Start ssh-agent and add our key so paramiko can use the agent eval "$(ssh-agent -s)" ssh-add ~/.ssh/id_ed25519 + sudo python3 -m venv /opt/borgstore-venv + sudo /opt/borgstore-venv/bin/pip install -U pip setuptools wheel + sudo /opt/borgstore-venv/bin/pip install "borgstore[rest]" + sudo ln -sf /opt/borgstore-venv/bin/borgstore-server-rest /usr/local/bin/borgstore-server-rest # Export SFTP test URL for tox via GITHUB_ENV echo "BORG_TEST_SFTP_REPO=sftp://sftpuser@localhost:22/borg/sftp-repo" >> $GITHUB_ENV + echo "BORG_TEST_REST_REPO=rest://sftpuser@localhost:22/borg/rest-repo" >> $GITHUB_ENV - name: Install and configure MinIO S3 server (test only) if: ${{ runner.os == 'Linux' && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }} diff --git a/src/borg/testsuite/archiver/remote_repo_test.py b/src/borg/testsuite/archiver/remote_repo_test.py index 294aa4b3a1..2374a94368 100644 --- a/src/borg/testsuite/archiver/remote_repo_test.py +++ b/src/borg/testsuite/archiver/remote_repo_test.py @@ -10,6 +10,7 @@ SFTP_URL = os.environ.get("BORG_TEST_SFTP_REPO") +REST_URL = os.environ.get("BORG_TEST_REST_REPO") S3_URL = os.environ.get("BORG_TEST_S3_REPO") @@ -57,6 +58,30 @@ def test_rclone_repo_basics(archiver, tmp_path): cmd(archiver, "repo-delete") +@pytest.mark.skipif(not REST_URL, reason="BORG_TEST_REST_REPO not set.") +def test_rest_repo_basics(archiver): + create_regular_file(archiver.input_path, "file1", size=100 * 1024) + create_regular_file(archiver.input_path, "file2", size=10 * 1024) + archiver.repository_location = REST_URL + archive_name = "test-archive" + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", archive_name, "input") + list_output = cmd(archiver, "repo-list") + assert archive_name in list_output + archive_list_output = cmd(archiver, "list", archive_name) + assert "input/file1" in archive_list_output + assert "input/file2" in archive_list_output + with changedir("output"): + cmd(archiver, "extract", archive_name) + assert_dirs_equal( + archiver.input_path, os.path.join(archiver.output_path, "input"), ignore_flags=True, ignore_xattrs=True + ) + cmd(archiver, "delete", "-a", archive_name) + list_output = cmd(archiver, "repo-list") + assert archive_name not in list_output + cmd(archiver, "repo-delete") + + @pytest.mark.skipif(not SFTP_URL, reason="BORG_TEST_SFTP_REPO not set.") def test_sftp_repo_basics(archiver): create_regular_file(archiver.input_path, "file1", size=100 * 1024)