From d7ffe74801506586652a2d348fd7fa03fef38269 Mon Sep 17 00:00:00 2001 From: Mrityunjay Raj Date: Wed, 15 Jul 2026 19:59:41 +0530 Subject: [PATCH 1/4] check: support custom archive formatting, #9411 Add --format and BORG_CHECK_FORMAT to control the archive part of the "Analyzing archive ..." output, like repo-list and prune already do. Archives with damaged or missing metadata fall back to the plain description. --- docs/usage/general/environment.rst.inc | 2 + src/borg/archive.py | 18 +++++-- src/borg/archiver/check_cmd.py | 31 +++++++++++- src/borg/testsuite/archiver/check_cmd_test.py | 47 ++++++++++++++++++- 4 files changed, 93 insertions(+), 5 deletions(-) diff --git a/docs/usage/general/environment.rst.inc b/docs/usage/general/environment.rst.inc index fb3eaca53c..b7b37e8fca 100644 --- a/docs/usage/general/environment.rst.inc +++ b/docs/usage/general/environment.rst.inc @@ -142,6 +142,8 @@ General: Now you can init a fresh repo. Make sure you do not use the workaround any more. Output formatting: + BORG_CHECK_FORMAT + Giving the default value for ``borg check --format=X``. BORG_LIST_FORMAT Giving the default value for ``borg list --format=X``. BORG_REPO_LIST_FORMAT diff --git a/src/borg/archive.py b/src/borg/archive.py index 6e107896a7..de13a3ecfe 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -35,6 +35,7 @@ from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns from .helpers import parse_timestamp, archive_ts_now, CompressionSpec from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize +from .helpers import ArchiveFormatter from .helpers import safe_encode, make_path_safe, remove_surrogates, text_to_json, join_cmd, remove_dotdot_prefixes from .helpers import StableDict from .helpers import bin_to_hex @@ -1763,6 +1764,8 @@ def check( newer=None, oldest=None, newest=None, + format, + iec=False, ): """Perform a set of checks on 'repository' @@ -1773,6 +1776,8 @@ def check( :param older/newer: only check archives older/newer than timedelta from now :param oldest/newest: only check archives older/newer than timedelta from oldest/newest archive timestamp :param verify_data: integrity verification of data referenced by archives + :param format: format string used to describe an archive in the log output + :param iec: format sizes using IEC units (1KiB = 1024B) """ if not isinstance(repository, Repository): logger.error("Checking legacy repositories is not supported.") @@ -1780,6 +1785,8 @@ def check( logger.info("Starting archive consistency check...") self.check_all = not any((first, last, match, older, newer, oldest, newest)) self.repair = repair + self.format = format + self.iec = iec self.repository = repository # A normal (non-repair) archives check trusts the in-repo index: the repository check verified # each index object's sha256, and the index is the authoritative record of which chunks exist, @@ -2139,6 +2146,7 @@ def valid_item(obj): else: archive_infos = self.manifest.archives.list(sort_by=sort_by) num_archives = len(archive_infos) + formatter = ArchiveFormatter(self.format, self.repository, self.manifest, self.key, iec=self.iec) pi = ProgressIndicatorPercent( total=num_archives, msg="Checking archives %3.1f%%", step=0.1, msgid="check.rebuild_archives" @@ -2146,9 +2154,13 @@ def valid_item(obj): for i, info in enumerate(archive_infos): pi.show(i) archive_id, archive_id_hex = info.id, bin_to_hex(info.id) - logger.info( - f"Analyzing archive {info.name} {info.ts.astimezone()} {archive_id_hex} ({i + 1}/{num_archives})" - ) + try: + formatted = formatter.format_item(info, jsonline=False) + except (Archive.DoesNotExist, Repository.ObjectNotFound, IntegrityErrorBase): + # keys like {comment} need the archive metadata, which is damaged or missing here. + # use the values from the archive directory entry, they are always available. + formatted = f"{info.name} {info.ts.astimezone()} {archive_id_hex}" + logger.info(f"Analyzing archive {formatted} ({i + 1}/{num_archives})") if archive_id not in self.chunks: logger.error(f"Archive metadata block {archive_id_hex} is missing!") self.error_found = True diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index 353306bb93..5ea1b60e3b 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -1,8 +1,11 @@ +import os +from string import Formatter + from ._common import with_repository, Highlander from ..archive import ArchiveChecker from ..constants import * # NOQA from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, IntegrityError -from ..helpers import yes +from ..helpers import yes, ArchiveFormatter from ..helpers.argparsing import ArgumentParser from ..logger import create_logger @@ -51,6 +54,19 @@ def do_check(self, args, repository): archive_checker.key = archive_checker.make_key(repository, manifest_only=True) except IntegrityError: pass # will try to make key later again + if args.format is not None: + format = args.format + else: + format = os.environ.get("BORG_CHECK_FORMAT", "{archive} {time} {id}") + # check the format here, the archives check formats the first archive only after + # the repository check has finished, which can take hours. + try: + format_keys = {key for _, key, _, _ in Formatter().parse(format) if key} + except ValueError as err: + raise CommandError(f"Invalid format string: {err}") + unknown_keys = format_keys - set(ArchiveFormatter.KEY_DESCRIPTIONS) - set(ArchiveFormatter.FIXED_KEYS) + if unknown_keys: + raise CommandError(f"Invalid format keys: {', '.join(sorted(unknown_keys))}") if not args.archives_only: if not repository.check(repair=args.repair, max_duration=args.max_duration): set_ec(EXIT_WARNING) @@ -67,6 +83,8 @@ def do_check(self, args, repository): newer=args.newer, oldest=args.oldest, newest=args.newest, + format=format, + iec=args.iec, ): set_ec(EXIT_WARNING) return @@ -139,6 +157,10 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser): data from the repository and is thus very time-consuming. You cannot use ``--find-lost-archives`` with ``--repository-only``. + You can influence how the archive part of the ``Analyzing archive ...`` output is + formatted by giving a custom format using ``--format`` (see the ``borg repo-list`` + description for more details about the format string). + About repair mode +++++++++++++++++ @@ -210,4 +232,11 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser): action=Highlander, help="perform only a partial repository check for at most SECONDS seconds (default: unlimited)", ) + subparser.add_argument( + "--format", + metavar="FORMAT", + dest="format", + action=Highlander, + help="specify format for the archive part " '(default: "{archive} {time} {id}")', + ) define_archive_filters_group(subparser) diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py index 06f9339581..78357045f2 100644 --- a/src/borg/testsuite/archiver/check_cmd_test.py +++ b/src/borg/testsuite/archiver/check_cmd_test.py @@ -7,7 +7,7 @@ from ...archive import ChunkBuffer from ...constants import * # NOQA -from ...helpers import bin_to_hex, msgpack +from ...helpers import bin_to_hex, msgpack, CommandError from ...manifest import Manifest from ...repository import Repository from ..repository_test import fchunk, corrupt_chunk_on_disk @@ -215,6 +215,51 @@ def test_missing_archive_metadata(archivers, request): cmd(archiver, "check", exit_code=0) +def test_check_format(archivers, request): + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + output = cmd(archiver, "check", "-v", "--archives-only", "--format", "{archive}|{hostname}", exit_code=0) + assert "Analyzing archive archive1|" in output + + +def test_check_format_env_var(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + monkeypatch.setenv("BORG_CHECK_FORMAT", "{archive}|env") + output = cmd(archiver, "check", "-v", "--archives-only", exit_code=0) + assert "Analyzing archive archive1|env" in output + output = cmd(archiver, "check", "-v", "--archives-only", "--format", "{archive}|arg", exit_code=0) + assert "Analyzing archive archive1|arg" in output # --format overrides the env var + + +def test_check_format_invalid_key(archivers, request): + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + if archiver.FORK_DEFAULT: + expected_ec = CommandError().exit_code + output = cmd(archiver, "check", "--archives-only", "--format", "{nosuchkey}", exit_code=expected_ec) + assert "Invalid format keys: nosuchkey" in output + else: + with pytest.raises(CommandError, match="Invalid format keys: nosuchkey"): + cmd(archiver, "check", "--archives-only", "--format", "{nosuchkey}") + + +def test_check_format_missing_archive_metadata(archivers, request): + # {comment} needs the archive metadata, which is deleted below. + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + archive, repository = open_archive(archiver.repository_path, "archive1") + with repository: + repository.delete(archive.id) + archive_id_hex = bin_to_hex(archive.id) + output = cmd(archiver, "check", "-v", "--archives-only", "--format", "{archive} {comment}", exit_code=1) + # the archive directory entry has no name for it, only the id, which {archive} {comment} would not show: + assert "Analyzing archive archive-does-not-exist " in output + assert f"{archive_id_hex} (1/2)" in output + assert f"Archive metadata block {archive_id_hex} is missing!" in output + assert "Analyzing archive archive2" in output # the intact archive still uses the given format + + def test_missing_manifest(archivers, request): archiver = request.getfixturevalue(archivers) check_cmd_setup(archiver) From e853ccbc964dd8ade7849743f59f12d77ad390d3 Mon Sep 17 00:00:00 2001 From: Mrityunjay Raj Date: Fri, 17 Jul 2026 17:52:47 +0530 Subject: [PATCH 2/4] check: use OutputTimestamp in the archive format fallback, #9411 --- src/borg/archive.py | 2 +- src/borg/testsuite/archiver/check_cmd_test.py | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index de13a3ecfe..198435fdb3 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -2159,7 +2159,7 @@ def valid_item(obj): except (Archive.DoesNotExist, Repository.ObjectNotFound, IntegrityErrorBase): # keys like {comment} need the archive metadata, which is damaged or missing here. # use the values from the archive directory entry, they are always available. - formatted = f"{info.name} {info.ts.astimezone()} {archive_id_hex}" + formatted = f"{info.name} {OutputTimestamp(info.ts)} {archive_id_hex}" logger.info(f"Analyzing archive {formatted} ({i + 1}/{num_archives})") if archive_id not in self.chunks: logger.error(f"Archive metadata block {archive_id_hex} is missing!") diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py index 78357045f2..ed28bb3b51 100644 --- a/src/borg/testsuite/archiver/check_cmd_test.py +++ b/src/borg/testsuite/archiver/check_cmd_test.py @@ -1,5 +1,6 @@ from datetime import datetime, timezone, timedelta from pathlib import Path +import re import shutil from unittest.mock import patch @@ -244,6 +245,18 @@ def test_check_format_invalid_key(archivers, request): cmd(archiver, "check", "--archives-only", "--format", "{nosuchkey}") +def test_check_format_invalid_format_string(archivers, request): + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + if archiver.FORK_DEFAULT: + expected_ec = CommandError().exit_code + output = cmd(archiver, "check", "--archives-only", "--format", "{archive", exit_code=expected_ec) + assert "Invalid format string" in output + else: + with pytest.raises(CommandError, match="Invalid format string"): + cmd(archiver, "check", "--archives-only", "--format", "{archive") + + def test_check_format_missing_archive_metadata(archivers, request): # {comment} needs the archive metadata, which is deleted below. archiver = request.getfixturevalue(archivers) @@ -253,8 +266,9 @@ def test_check_format_missing_archive_metadata(archivers, request): repository.delete(archive.id) archive_id_hex = bin_to_hex(archive.id) output = cmd(archiver, "check", "-v", "--archives-only", "--format", "{archive} {comment}", exit_code=1) - # the archive directory entry has no name for it, only the id, which {archive} {comment} would not show: - assert "Analyzing archive archive-does-not-exist " in output + # the archive directory entry has no name for it, only the id, which {archive} {comment} would not show. + # the timestamp uses the same style as the formatter would produce, e.g. "Thu, 1970-01-01 00:00:00 +0000": + assert re.search(r"Analyzing archive archive-does-not-exist \w{3}, \d{4}-\d{2}-\d{2} ", output) assert f"{archive_id_hex} (1/2)" in output assert f"Archive metadata block {archive_id_hex} is missing!" in output assert "Analyzing archive archive2" in output # the intact archive still uses the given format From 38ee6ea38227d9ef19a2e421db525b0fe8b4edde Mon Sep 17 00:00:00 2001 From: Mrityunjay Raj Date: Fri, 17 Jul 2026 18:45:43 +0530 Subject: [PATCH 3/4] check: --repository-only contradicts --format, #9411 --- src/borg/archiver/check_cmd.py | 7 +++++-- src/borg/testsuite/archiver/check_cmd_test.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index 5ea1b60e3b..ef74dbcd97 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -34,9 +34,12 @@ def do_check(self, args, repository): env_var_override="BORG_CHECK_I_KNOW_WHAT_I_AM_DOING", ): raise CancelledByUser() - if args.repo_only and any((args.verify_data, args.first, args.last, args.match_archives)): + if args.repo_only and any( + (args.verify_data, args.first, args.last, args.match_archives, args.format is not None) + ): raise CommandError( - "--repository-only contradicts --first, --last, -a / --match-archives and --verify-data arguments." + "--repository-only contradicts --first, --last, -a / --match-archives, " + "--verify-data and --format arguments." ) if args.repo_only and args.find_lost_archives: raise CommandError("--repository-only contradicts the --find-lost-archives option.") diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py index ed28bb3b51..1e91ea84ba 100644 --- a/src/borg/testsuite/archiver/check_cmd_test.py +++ b/src/borg/testsuite/archiver/check_cmd_test.py @@ -245,6 +245,21 @@ def test_check_format_invalid_key(archivers, request): cmd(archiver, "check", "--archives-only", "--format", "{nosuchkey}") +def test_check_format_repository_only(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + if archiver.FORK_DEFAULT: + expected_ec = CommandError().exit_code + output = cmd(archiver, "check", "--repository-only", "--format", "{archive}", exit_code=expected_ec) + assert "--repository-only contradicts" in output + else: + with pytest.raises(CommandError, match="--repository-only contradicts"): + cmd(archiver, "check", "--repository-only", "--format", "{archive}") + # only the option contradicts, a set env var must not make the repository check fail: + monkeypatch.setenv("BORG_CHECK_FORMAT", "{archive}|env") + cmd(archiver, "check", "--repository-only", exit_code=0) + + def test_check_format_invalid_format_string(archivers, request): archiver = request.getfixturevalue(archivers) check_cmd_setup(archiver) From 5878040530c516c3f9baddaaf363aee49f4baa6d Mon Sep 17 00:00:00 2001 From: Mrityunjay Raj Date: Tue, 21 Jul 2026 07:34:10 +0530 Subject: [PATCH 4/4] check: move format validation to ArchiveFormatter.validate_format, #9411 --- src/borg/archive.py | 2 +- src/borg/archiver/check_cmd.py | 9 +-------- src/borg/helpers/parseformat.py | 13 ++++++++++++- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 198435fdb3..8be00e862c 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1753,6 +1753,7 @@ def check( self, repository, *, + format, verify_data=False, repair=False, find_lost_archives=False, @@ -1764,7 +1765,6 @@ def check( newer=None, oldest=None, newest=None, - format, iec=False, ): """Perform a set of checks on 'repository' diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index ef74dbcd97..1b7cf3d420 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -1,5 +1,4 @@ import os -from string import Formatter from ._common import with_repository, Highlander from ..archive import ArchiveChecker @@ -63,13 +62,7 @@ def do_check(self, args, repository): format = os.environ.get("BORG_CHECK_FORMAT", "{archive} {time} {id}") # check the format here, the archives check formats the first archive only after # the repository check has finished, which can take hours. - try: - format_keys = {key for _, key, _, _ in Formatter().parse(format) if key} - except ValueError as err: - raise CommandError(f"Invalid format string: {err}") - unknown_keys = format_keys - set(ArchiveFormatter.KEY_DESCRIPTIONS) - set(ArchiveFormatter.FIXED_KEYS) - if unknown_keys: - raise CommandError(f"Invalid format keys: {', '.join(sorted(unknown_keys))}") + ArchiveFormatter.validate_format(format) if not args.archives_only: if not repository.check(repair=args.repair, max_duration=args.max_duration): set_ec(EXIT_WARNING) diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 2cf2e8227f..7ccd464ccd 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -22,7 +22,7 @@ import yaml -from .errors import Error +from .errors import Error, CommandError from .fs import make_path_safe, slashify from .argparsing import Action, ArgumentError, ArgumentTypeError, register_type from .msgpack import Timestamp @@ -927,6 +927,17 @@ def keys_help(cls): assert not keys, str(keys) return "\n".join(help) + @classmethod + def validate_format(cls, format): + """raise a CommandError if format is malformed or uses keys this formatter does not know""" + try: + format_keys = {key for _, key, _, _ in Formatter().parse(format) if key} + except ValueError as err: + raise CommandError(f"Invalid format string: {err}") + unknown_keys = format_keys - set(cls.KEY_DESCRIPTIONS) - set(cls.FIXED_KEYS) + if unknown_keys: + raise CommandError(f"Invalid format keys: {', '.join(sorted(unknown_keys))}") + class ArchiveFormatter(BaseFormatter): KEY_DESCRIPTIONS = {