Add a "--match-archives" flag to the "check" action (#779).

This commit is contained in:
Dan Helfman 2023-10-29 16:22:39 -07:00
parent c5732aa4fc
commit b47088067c
5 changed files with 175 additions and 43 deletions

2
NEWS
View file

@ -9,6 +9,8 @@
support empty sections without erroring.
* #774: Disallow the "--dry-run" flag with the "borg" action, as borgmatic can't guarantee the Borg
command won't have side effects.
* #779: Add a "--match-archives" flag to the "check" action for selecting the archives to check,
overriding the existing "archive_name_format" and "match_archives" options in configuration.
1.8.3
* #665: BREAKING: Simplify logging logic as follows: Syslog verbosity is now disabled by

View file

@ -39,13 +39,10 @@ def run_check(
repository['path'],
config,
local_borg_version,
check_arguments,
global_arguments,
local_path=local_path,
remote_path=remote_path,
progress=check_arguments.progress,
repair=check_arguments.repair,
only_checks=check_arguments.only,
force=check_arguments.force,
)
borgmatic.hooks.command.execute_hook(
config.get('after_check'),

View file

@ -149,11 +149,13 @@ def filter_checks_on_frequency(
return tuple(filtered_checks)
def make_archive_filter_flags(local_borg_version, config, checks, check_last=None, prefix=None):
def make_archive_filter_flags(
local_borg_version, config, checks, check_arguments, check_last=None, prefix=None
):
'''
Given the local Borg version, a configuration dict, a parsed sequence of checks, the check last
value, and a consistency check prefix, transform the checks into tuple of command-line flags for
filtering archives in a check command.
Given the local Borg version, a configuration dict, a parsed sequence of checks, check arguments
as an argparse.Namespace instance, the check last value, and a consistency check prefix,
transform the checks into tuple of command-line flags for filtering archives in a check command.
If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if
a prefix value is given and "archives" is in checks, then include a "--match-archives" flag.
@ -168,7 +170,7 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_last=Non
if prefix
else (
flags.make_match_archives_flags(
config.get('match_archives'),
check_arguments.match_archives or config.get('match_archives'),
config.get('archive_name_format'),
local_borg_version,
)
@ -353,18 +355,15 @@ def check_archives(
repository_path,
config,
local_borg_version,
check_arguments,
global_arguments,
local_path='borg',
remote_path=None,
progress=None,
repair=None,
only_checks=None,
force=None,
):
'''
Given a local or remote repository path, a configuration dict, local/remote commands to run,
whether to include progress information, whether to attempt a repair, and an optional list of
checks to use instead of configured checks, check the contained Borg archives for consistency.
Given a local or remote repository path, a configuration dict, the local Borg version, check
arguments as an argparse.Namespace instance, global arguments, and local/remote commands to run,
check the contained Borg archives for consistency.
If there are no consistency checks to run, skip running them.
@ -389,11 +388,11 @@ def check_archives(
check_last = config.get('check_last', None)
prefix = config.get('prefix')
configured_checks = parse_checks(config, only_checks)
configured_checks = parse_checks(config, check_arguments.only_checks)
lock_wait = None
extra_borg_options = config.get('extra_borg_options', {}).get('check', '')
archive_filter_flags = make_archive_filter_flags(
local_borg_version, config, configured_checks, check_last, prefix
local_borg_version, config, configured_checks, check_arguments, check_last, prefix
)
archives_check_id = make_archives_check_id(archive_filter_flags)
@ -401,7 +400,7 @@ def check_archives(
config,
borg_repository_id,
configured_checks,
force,
check_arguments.force,
archives_check_id,
)
@ -416,13 +415,13 @@ def check_archives(
full_command = (
(local_path, 'check')
+ (('--repair',) if repair else ())
+ (('--repair',) if check_arguments.repair else ())
+ make_check_flags(checks, archive_filter_flags)
+ (('--remote-path', remote_path) if remote_path else ())
+ (('--log-json',) if global_arguments.log_json else ())
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
+ verbosity_flags
+ (('--progress',) if progress else ())
+ (('--progress',) if check_arguments.progress else ())
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+ flags.make_repository_flags(repository_path, local_borg_version)
)
@ -431,7 +430,7 @@ def check_archives(
# The Borg repair option triggers an interactive prompt, which won't work when output is
# captured. And progress messes with the terminal directly.
if repair or progress:
if check_arguments.repair or check_arguments.progress:
execute_command(
full_command, output_file=DO_NOT_CAPTURE, extra_environment=borg_environment
)

View file

@ -604,11 +604,18 @@ def make_parsers():
action='store_true',
help='Attempt to repair any inconsistencies found (for interactive use)',
)
check_group.add_argument(
'-a',
'--match-archives',
'--glob-archives',
metavar='PATTERN',
help='Only check archives with names matching this pattern',
)
check_group.add_argument(
'--only',
metavar='CHECK',
choices=('repository', 'archives', 'data', 'extract'),
dest='only',
dest='only_checks',
action='append',
help='Run a particular consistency check (repository, archives, data, or extract) instead of configured checks (subject to configured frequency, can specify flag multiple times)',
)

View file

@ -201,6 +201,7 @@ def test_make_archive_filter_flags_with_default_checks_and_prefix_returns_defaul
'1.2.3',
{},
('repository', 'archives'),
check_arguments=flexmock(match_archives=None),
prefix='foo',
)
@ -215,6 +216,7 @@ def test_make_archive_filter_flags_with_all_checks_and_prefix_returns_default_fl
'1.2.3',
{},
('repository', 'archives', 'extract'),
check_arguments=flexmock(match_archives=None),
prefix='foo',
)
@ -229,6 +231,7 @@ def test_make_archive_filter_flags_with_all_checks_and_prefix_without_borg_featu
'1.2.3',
{},
('repository', 'archives', 'extract'),
check_arguments=flexmock(match_archives=None),
prefix='foo',
)
@ -239,7 +242,9 @@ def test_make_archive_filter_flags_with_archives_check_and_last_includes_last_fl
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), check_last=3)
flags = module.make_archive_filter_flags(
'1.2.3', {}, ('archives',), check_arguments=flexmock(match_archives=None), check_last=3
)
assert flags == ('--last', '3')
@ -248,7 +253,9 @@ def test_make_archive_filter_flags_with_data_check_and_last_includes_last_flag()
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), check_last=3)
flags = module.make_archive_filter_flags(
'1.2.3', {}, ('data',), check_arguments=flexmock(match_archives=None), check_last=3
)
assert flags == ('--last', '3')
@ -257,7 +264,9 @@ def test_make_archive_filter_flags_with_repository_check_and_last_omits_last_fla
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), check_last=3)
flags = module.make_archive_filter_flags(
'1.2.3', {}, ('repository',), check_arguments=flexmock(match_archives=None), check_last=3
)
assert flags == ()
@ -266,7 +275,13 @@ def test_make_archive_filter_flags_with_default_checks_and_last_includes_last_fl
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), check_last=3)
flags = module.make_archive_filter_flags(
'1.2.3',
{},
('repository', 'archives'),
check_arguments=flexmock(match_archives=None),
check_last=3,
)
assert flags == ('--last', '3')
@ -275,7 +290,9 @@ def test_make_archive_filter_flags_with_archives_check_and_prefix_includes_match
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix='foo-')
flags = module.make_archive_filter_flags(
'1.2.3', {}, ('archives',), check_arguments=flexmock(match_archives=None), prefix='foo-'
)
assert flags == ('--match-archives', 'sh:foo-*')
@ -284,11 +301,30 @@ def test_make_archive_filter_flags_with_data_check_and_prefix_includes_match_arc
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), prefix='foo-')
flags = module.make_archive_filter_flags(
'1.2.3', {}, ('data',), check_arguments=flexmock(match_archives=None), prefix='foo-'
)
assert flags == ('--match-archives', 'sh:foo-*')
def test_make_archive_filter_flags_prefers_check_arguments_match_archives_to_config_match_archives():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
'baz-*', None, '1.2.3'
).and_return(('--match-archives', 'sh:baz-*'))
flags = module.make_archive_filter_flags(
'1.2.3',
{'match_archives': 'bar-{now}'}, # noqa: FS003
('archives',),
check_arguments=flexmock(match_archives='baz-*'),
prefix='',
)
assert flags == ('--match-archives', 'sh:baz-*')
def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
@ -296,7 +332,11 @@ def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_arc
).and_return(('--match-archives', 'sh:bar-*'))
flags = module.make_archive_filter_flags(
'1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003
'1.2.3',
{'archive_name_format': 'bar-{now}'}, # noqa: FS003
('archives',),
check_arguments=flexmock(match_archives=None),
prefix='',
)
assert flags == ('--match-archives', 'sh:bar-*')
@ -306,7 +346,9 @@ def test_make_archive_filter_flags_with_archives_check_and_none_prefix_omits_mat
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix=None)
flags = module.make_archive_filter_flags(
'1.2.3', {}, ('archives',), check_arguments=flexmock(match_archives=None), prefix=None
)
assert flags == ()
@ -315,7 +357,9 @@ def test_make_archive_filter_flags_with_repository_check_and_prefix_omits_match_
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), prefix='foo-')
flags = module.make_archive_filter_flags(
'1.2.3', {}, ('repository',), check_arguments=flexmock(match_archives=None), prefix='foo-'
)
assert flags == ()
@ -324,7 +368,13 @@ def test_make_archive_filter_flags_with_default_checks_and_prefix_includes_match
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-')
flags = module.make_archive_filter_flags(
'1.2.3',
{},
('repository', 'archives'),
check_arguments=flexmock(match_archives=None),
prefix='foo-',
)
assert flags == ('--match-archives', 'sh:foo-*')
@ -607,7 +657,7 @@ def test_upgrade_check_times_renames_stale_temporary_check_path():
module.upgrade_check_times(flexmock(), flexmock())
def test_check_archives_with_progress_calls_borg_with_progress_parameter():
def test_check_archives_with_progress_passes_through_to_borg():
checks = ('repository',)
config = {'check_last': None}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
@ -634,12 +684,14 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=True, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
progress=True,
)
def test_check_archives_with_repair_calls_borg_with_repair_parameter():
def test_check_archives_with_repair_passes_through_to_borg():
checks = ('repository',)
config = {'check_last': None}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
@ -666,8 +718,10 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=True, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
repair=True,
)
@ -701,6 +755,9 @@ def test_check_archives_calls_borg_with_parameters(checks):
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
)
@ -723,6 +780,9 @@ def test_check_archives_with_json_error_raises():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
)
@ -743,6 +803,9 @@ def test_check_archives_with_missing_json_keys_raises():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
)
@ -769,11 +832,14 @@ def test_check_archives_with_extract_check_calls_extract_only():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
)
def test_check_archives_with_log_info_calls_borg_with_info_parameter():
def test_check_archives_with_log_info_passes_through_to_borg():
checks = ('repository',)
config = {'check_last': None}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
@ -795,11 +861,14 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
)
def test_check_archives_with_log_debug_calls_borg_with_debug_parameter():
def test_check_archives_with_log_debug_passes_through_to_borg():
checks = ('repository',)
config = {'check_last': None}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
@ -821,6 +890,9 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
)
@ -841,6 +913,9 @@ def test_check_archives_without_any_checks_bails():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
)
@ -867,12 +942,15 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
local_path='borg1',
)
def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters():
def test_check_archives_with_remote_path_passes_through_to_borg():
checks = ('repository',)
check_last = flexmock()
config = {'check_last': check_last}
@ -894,12 +972,15 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
remote_path='borg1',
)
def test_check_archives_with_log_json_calls_borg_with_log_json_parameters():
def test_check_archives_with_log_json_passes_through_to_borg():
checks = ('repository',)
check_last = flexmock()
config = {'check_last': check_last}
@ -921,11 +1002,14 @@ def test_check_archives_with_log_json_calls_borg_with_log_json_parameters():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=True),
)
def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
def test_check_archives_with_lock_wait_passes_through_to_borg():
checks = ('repository',)
check_last = flexmock()
config = {'lock_wait': 5, 'check_last': check_last}
@ -947,6 +1031,9 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
)
@ -974,11 +1061,14 @@ def test_check_archives_with_retention_prefix():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
)
def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options():
def test_check_archives_with_extra_borg_options_passes_through_to_borg():
checks = ('repository',)
config = {'check_last': None, 'extra_borg_options': {'check': '--extra --options'}}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
@ -999,5 +1089,42 @@ def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options():
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives=None
),
global_arguments=flexmock(log_json=False),
)
def test_check_archives_with_match_archives_passes_through_to_borg():
checks = ('archives',)
config = {'check_last': None}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(
('--match-archives', 'foo-*')
)
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module).should_receive('make_check_flags').and_return(('--match-archives', 'foo-*'))
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'check', '--match-archives', 'foo-*', 'repo'),
extra_environment=None,
).once()
flexmock(module).should_receive('make_check_time_path')
flexmock(module).should_receive('write_check_time')
module.check_archives(
repository_path='repo',
config=config,
local_borg_version='1.2.3',
check_arguments=flexmock(
progress=None, repair=None, only_checks=None, force=None, match_archives='foo-*'
),
global_arguments=flexmock(log_json=False),
)