diff --git a/docs/usage/general/environment.rst.inc b/docs/usage/general/environment.rst.inc index fb3eaca53c..b7b37e8fca 100644 --- a/docs/usage/general/environment.rst.inc +++ b/docs/usage/general/environment.rst.inc @@ -142,6 +142,8 @@ General: Now you can init a fresh repo. Make sure you do not use the workaround any more. Output formatting: + BORG_CHECK_FORMAT + Giving the default value for ``borg check --format=X``. BORG_LIST_FORMAT Giving the default value for ``borg list --format=X``. BORG_REPO_LIST_FORMAT diff --git a/src/borg/archive.py b/src/borg/archive.py index 6e107896a7..8be00e862c 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -35,6 +35,7 @@ from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns from .helpers import parse_timestamp, archive_ts_now, CompressionSpec from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize +from .helpers import ArchiveFormatter from .helpers import safe_encode, make_path_safe, remove_surrogates, text_to_json, join_cmd, remove_dotdot_prefixes from .helpers import StableDict from .helpers import bin_to_hex @@ -1752,6 +1753,7 @@ def check( self, repository, *, + format, verify_data=False, repair=False, find_lost_archives=False, @@ -1763,6 +1765,7 @@ def check( newer=None, oldest=None, newest=None, + iec=False, ): """Perform a set of checks on 'repository' @@ -1773,6 +1776,8 @@ def check( :param older/newer: only check archives older/newer than timedelta from now :param oldest/newest: only check archives older/newer than timedelta from oldest/newest archive timestamp :param verify_data: integrity verification of data referenced by archives + :param format: format string used to describe an archive in the log output + :param iec: format sizes using IEC units (1KiB = 1024B) """ if not isinstance(repository, Repository): logger.error("Checking legacy repositories is not supported.") @@ -1780,6 +1785,8 @@ def check( logger.info("Starting archive consistency check...") self.check_all = not any((first, last, match, older, newer, oldest, newest)) self.repair = repair + self.format = format + self.iec = iec self.repository = repository # A normal (non-repair) archives check trusts the in-repo index: the repository check verified # each index object's sha256, and the index is the authoritative record of which chunks exist, @@ -2139,6 +2146,7 @@ def valid_item(obj): else: archive_infos = self.manifest.archives.list(sort_by=sort_by) num_archives = len(archive_infos) + formatter = ArchiveFormatter(self.format, self.repository, self.manifest, self.key, iec=self.iec) pi = ProgressIndicatorPercent( total=num_archives, msg="Checking archives %3.1f%%", step=0.1, msgid="check.rebuild_archives" @@ -2146,9 +2154,13 @@ def valid_item(obj): for i, info in enumerate(archive_infos): pi.show(i) archive_id, archive_id_hex = info.id, bin_to_hex(info.id) - logger.info( - f"Analyzing archive {info.name} {info.ts.astimezone()} {archive_id_hex} ({i + 1}/{num_archives})" - ) + try: + formatted = formatter.format_item(info, jsonline=False) + except (Archive.DoesNotExist, Repository.ObjectNotFound, IntegrityErrorBase): + # keys like {comment} need the archive metadata, which is damaged or missing here. + # use the values from the archive directory entry, they are always available. + formatted = f"{info.name} {OutputTimestamp(info.ts)} {archive_id_hex}" + logger.info(f"Analyzing archive {formatted} ({i + 1}/{num_archives})") if archive_id not in self.chunks: logger.error(f"Archive metadata block {archive_id_hex} is missing!") self.error_found = True diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index 353306bb93..1b7cf3d420 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -1,8 +1,10 @@ +import os + from ._common import with_repository, Highlander from ..archive import ArchiveChecker from ..constants import * # NOQA from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, IntegrityError -from ..helpers import yes +from ..helpers import yes, ArchiveFormatter from ..helpers.argparsing import ArgumentParser from ..logger import create_logger @@ -31,9 +33,12 @@ def do_check(self, args, repository): env_var_override="BORG_CHECK_I_KNOW_WHAT_I_AM_DOING", ): raise CancelledByUser() - if args.repo_only and any((args.verify_data, args.first, args.last, args.match_archives)): + if args.repo_only and any( + (args.verify_data, args.first, args.last, args.match_archives, args.format is not None) + ): raise CommandError( - "--repository-only contradicts --first, --last, -a / --match-archives and --verify-data arguments." + "--repository-only contradicts --first, --last, -a / --match-archives, " + "--verify-data and --format arguments." ) if args.repo_only and args.find_lost_archives: raise CommandError("--repository-only contradicts the --find-lost-archives option.") @@ -51,6 +56,13 @@ def do_check(self, args, repository): archive_checker.key = archive_checker.make_key(repository, manifest_only=True) except IntegrityError: pass # will try to make key later again + if args.format is not None: + format = args.format + else: + format = os.environ.get("BORG_CHECK_FORMAT", "{archive} {time} {id}") + # check the format here, the archives check formats the first archive only after + # the repository check has finished, which can take hours. + ArchiveFormatter.validate_format(format) if not args.archives_only: if not repository.check(repair=args.repair, max_duration=args.max_duration): set_ec(EXIT_WARNING) @@ -67,6 +79,8 @@ def do_check(self, args, repository): newer=args.newer, oldest=args.oldest, newest=args.newest, + format=format, + iec=args.iec, ): set_ec(EXIT_WARNING) return @@ -139,6 +153,10 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser): data from the repository and is thus very time-consuming. You cannot use ``--find-lost-archives`` with ``--repository-only``. + You can influence how the archive part of the ``Analyzing archive ...`` output is + formatted by giving a custom format using ``--format`` (see the ``borg repo-list`` + description for more details about the format string). + About repair mode +++++++++++++++++ @@ -210,4 +228,11 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser): action=Highlander, help="perform only a partial repository check for at most SECONDS seconds (default: unlimited)", ) + subparser.add_argument( + "--format", + metavar="FORMAT", + dest="format", + action=Highlander, + help="specify format for the archive part " '(default: "{archive} {time} {id}")', + ) define_archive_filters_group(subparser) diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 2cf2e8227f..7ccd464ccd 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -22,7 +22,7 @@ import yaml -from .errors import Error +from .errors import Error, CommandError from .fs import make_path_safe, slashify from .argparsing import Action, ArgumentError, ArgumentTypeError, register_type from .msgpack import Timestamp @@ -927,6 +927,17 @@ def keys_help(cls): assert not keys, str(keys) return "\n".join(help) + @classmethod + def validate_format(cls, format): + """raise a CommandError if format is malformed or uses keys this formatter does not know""" + try: + format_keys = {key for _, key, _, _ in Formatter().parse(format) if key} + except ValueError as err: + raise CommandError(f"Invalid format string: {err}") + unknown_keys = format_keys - set(cls.KEY_DESCRIPTIONS) - set(cls.FIXED_KEYS) + if unknown_keys: + raise CommandError(f"Invalid format keys: {', '.join(sorted(unknown_keys))}") + class ArchiveFormatter(BaseFormatter): KEY_DESCRIPTIONS = { diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py index 06f9339581..1e91ea84ba 100644 --- a/src/borg/testsuite/archiver/check_cmd_test.py +++ b/src/borg/testsuite/archiver/check_cmd_test.py @@ -1,5 +1,6 @@ from datetime import datetime, timezone, timedelta from pathlib import Path +import re import shutil from unittest.mock import patch @@ -7,7 +8,7 @@ from ...archive import ChunkBuffer from ...constants import * # NOQA -from ...helpers import bin_to_hex, msgpack +from ...helpers import bin_to_hex, msgpack, CommandError from ...manifest import Manifest from ...repository import Repository from ..repository_test import fchunk, corrupt_chunk_on_disk @@ -215,6 +216,79 @@ def test_missing_archive_metadata(archivers, request): cmd(archiver, "check", exit_code=0) +def test_check_format(archivers, request): + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + output = cmd(archiver, "check", "-v", "--archives-only", "--format", "{archive}|{hostname}", exit_code=0) + assert "Analyzing archive archive1|" in output + + +def test_check_format_env_var(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + monkeypatch.setenv("BORG_CHECK_FORMAT", "{archive}|env") + output = cmd(archiver, "check", "-v", "--archives-only", exit_code=0) + assert "Analyzing archive archive1|env" in output + output = cmd(archiver, "check", "-v", "--archives-only", "--format", "{archive}|arg", exit_code=0) + assert "Analyzing archive archive1|arg" in output # --format overrides the env var + + +def test_check_format_invalid_key(archivers, request): + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + if archiver.FORK_DEFAULT: + expected_ec = CommandError().exit_code + output = cmd(archiver, "check", "--archives-only", "--format", "{nosuchkey}", exit_code=expected_ec) + assert "Invalid format keys: nosuchkey" in output + else: + with pytest.raises(CommandError, match="Invalid format keys: nosuchkey"): + cmd(archiver, "check", "--archives-only", "--format", "{nosuchkey}") + + +def test_check_format_repository_only(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + if archiver.FORK_DEFAULT: + expected_ec = CommandError().exit_code + output = cmd(archiver, "check", "--repository-only", "--format", "{archive}", exit_code=expected_ec) + assert "--repository-only contradicts" in output + else: + with pytest.raises(CommandError, match="--repository-only contradicts"): + cmd(archiver, "check", "--repository-only", "--format", "{archive}") + # only the option contradicts, a set env var must not make the repository check fail: + monkeypatch.setenv("BORG_CHECK_FORMAT", "{archive}|env") + cmd(archiver, "check", "--repository-only", exit_code=0) + + +def test_check_format_invalid_format_string(archivers, request): + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + if archiver.FORK_DEFAULT: + expected_ec = CommandError().exit_code + output = cmd(archiver, "check", "--archives-only", "--format", "{archive", exit_code=expected_ec) + assert "Invalid format string" in output + else: + with pytest.raises(CommandError, match="Invalid format string"): + cmd(archiver, "check", "--archives-only", "--format", "{archive") + + +def test_check_format_missing_archive_metadata(archivers, request): + # {comment} needs the archive metadata, which is deleted below. + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) + archive, repository = open_archive(archiver.repository_path, "archive1") + with repository: + repository.delete(archive.id) + archive_id_hex = bin_to_hex(archive.id) + output = cmd(archiver, "check", "-v", "--archives-only", "--format", "{archive} {comment}", exit_code=1) + # the archive directory entry has no name for it, only the id, which {archive} {comment} would not show. + # the timestamp uses the same style as the formatter would produce, e.g. "Thu, 1970-01-01 00:00:00 +0000": + assert re.search(r"Analyzing archive archive-does-not-exist \w{3}, \d{4}-\d{2}-\d{2} ", output) + assert f"{archive_id_hex} (1/2)" in output + assert f"Archive metadata block {archive_id_hex} is missing!" in output + assert "Analyzing archive archive2" in output # the intact archive still uses the given format + + def test_missing_manifest(archivers, request): archiver = request.getfixturevalue(archivers) check_cmd_setup(archiver)