From 8fa90053cf1ec9a924f12967eb30f45722f0976b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 28 May 2022 19:29:33 -0700 Subject: [PATCH] Add "borgmatic check --force" flag to ignore configured check frequencies (#523). --- NEWS | 3 ++- borgmatic/borg/check.py | 17 +++++++++----- borgmatic/commands/arguments.py | 8 ++++++- borgmatic/commands/borgmatic.py | 1 + docs/how-to/deal-with-very-large-backups.md | 5 ++++- tests/unit/borg/test_check.py | 25 ++++++++++++++++++--- tests/unit/commands/test_borgmatic.py | 4 +++- 7 files changed, 51 insertions(+), 12 deletions(-) diff --git a/NEWS b/NEWS index f5888b0..f566233 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 1.6.2.dev0 * #523: Reduce the default consistency check frequency and support configuring the frequency - independently for each check. See the documentation for more information: + independently for each check. Also add "borgmatic check --force" flag to ignore configured + frequencies. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#check-frequency * #536: Fix generate-borgmatic-config to support more complex schema changes like the new Healthchecks configuration options when the "--source" flag is used. diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index f92e49a..0657b2a 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -81,8 +81,8 @@ def parse_frequency(frequency): time_unit += 's' if time_unit == 'months': - number *= 4 - time_unit = 'weeks' + number *= 30 + time_unit = 'days' elif time_unit == 'years': number *= 365 time_unit = 'days' @@ -93,11 +93,13 @@ def parse_frequency(frequency): raise ValueError(f"Could not parse consistency check frequency '{frequency}'") -def filter_checks_on_frequency(location_config, consistency_config, borg_repository_id, checks): +def filter_checks_on_frequency( + location_config, consistency_config, borg_repository_id, checks, force +): ''' Given a location config, a consistency config with a "checks" sequence of dicts, a Borg - repository ID, and sequence of checks, filter down those checks based on the configured - "frequency" for each check as compared to its check time file. + repository ID, a sequence of checks, and whether to force checks to run, filter down those + checks based on the configured "frequency" for each check as compared to its check time file. In other words, a check whose check time file's timestamp is too new (based on the configured frequency) will get cut from the returned sequence of checks. Example: @@ -119,6 +121,9 @@ def filter_checks_on_frequency(location_config, consistency_config, borg_reposit ''' filtered_checks = list(checks) + if force: + return tuple(filtered_checks) + for check_config in consistency_config.get('checks', DEFAULT_CHECKS): check = check_config['name'] if checks and check not in checks: @@ -240,6 +245,7 @@ def check_archives( progress=None, repair=None, only_checks=None, + force=None, ): ''' Given a local or remote repository path, a storage config dict, a consistency config dict, @@ -269,6 +275,7 @@ def check_archives( consistency_config, borg_repository_id, parse_checks(consistency_config, only_checks), + force, ) check_last = consistency_config.get('check_last', None) lock_wait = None diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index cb0c5dc..95f8a10 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -346,7 +346,7 @@ def make_parsers(): dest='repair', default=False, action='store_true', - help='Attempt to repair any inconsistencies found (experimental and only for interactive use)', + help='Attempt to repair any inconsistencies found (for interactive use)', ) check_group.add_argument( '--only', @@ -356,6 +356,12 @@ def make_parsers(): action='append', help='Run a particular consistency check (repository, archives, data, or extract) instead of configured checks (subject to configured frequency, can specify flag multiple times)', ) + check_group.add_argument( + '--force', + default=False, + action='store_true', + help='Ignore configured check frequencies and run checks unconditionally', + ) check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') extract_parser = subparsers.add_parser( diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index efc803f..4fac165 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -403,6 +403,7 @@ def run_actions( progress=arguments['check'].progress, repair=arguments['check'].repair, only_checks=arguments['check'].only, + force=arguments['check'].force, ) command.execute_hook( hooks.get('after_check'), diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index 447f90a..2a6bff4 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -96,6 +96,9 @@ within `~/.borgmatic/checks`). If it hasn't been long enough, the check is skipped. And you still have to run `borgmatic check` (or just `borgmatic`) in order for checks to run, even when a `frequency` is configured! +If you want to temporarily ignore your configured frequencies, you can invoke +`borgmatic check --force` to run checks unconditionally. + ### Disabling checks @@ -129,7 +132,7 @@ borgmatic check --only data --only extract This is useful for running slow consistency checks on an infrequent basis, separate from your regular checks. It is still subject to any configured -check frequencies. +check frequencies unless the `--force` flag is used. ## Troubleshooting diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 1f69d6a..6cd84b1 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -83,8 +83,8 @@ def test_parse_checks_with_override_data_check_also_injects_archives(): ('2 days', module.datetime.timedelta(days=2)), ('1 week', module.datetime.timedelta(weeks=1)), ('2 weeks', module.datetime.timedelta(weeks=2)), - ('1 month', module.datetime.timedelta(weeks=4)), - ('2 months', module.datetime.timedelta(weeks=8)), + ('1 month', module.datetime.timedelta(days=30)), + ('2 months', module.datetime.timedelta(days=60)), ('1 year', module.datetime.timedelta(days=365)), ('2 years', module.datetime.timedelta(days=365 * 2)), ), @@ -113,12 +113,17 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks(): consistency_config={}, borg_repository_id='repo', checks=('repository', 'archives'), + force=False, ) == ('repository', 'archives') def test_filter_checks_on_frequency_retains_unconfigured_check(): assert module.filter_checks_on_frequency( - location_config={}, consistency_config={}, borg_repository_id='repo', checks=('data',), + location_config={}, + consistency_config={}, + borg_repository_id='repo', + checks=('data',), + force=False, ) == ('data',) @@ -130,6 +135,7 @@ def test_filter_checks_on_frequency_retains_check_without_frequency(): consistency_config={'checks': [{'name': 'archives'}]}, borg_repository_id='repo', checks=('archives',), + force=False, ) == ('archives',) @@ -147,6 +153,7 @@ def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency(): consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, borg_repository_id='repo', checks=('archives',), + force=False, ) == ('archives',) @@ -162,6 +169,7 @@ def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file() consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, borg_repository_id='repo', checks=('archives',), + force=False, ) == ('archives',) @@ -178,11 +186,22 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency(): consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, borg_repository_id='repo', checks=('archives',), + force=False, ) == () ) +def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_force(): + assert module.filter_checks_on_frequency( + location_config={}, + consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, + borg_repository_id='repo', + checks=('archives',), + force=True, + ) == ('archives',) + + def test_make_check_flags_with_repository_check_returns_flag(): flags = module.make_check_flags(('repository',)) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 3a2c4f1..6918d4e 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -468,7 +468,9 @@ def test_run_actions_calls_hooks_for_check_action(): flexmock(module.command).should_receive('execute_hook').twice() arguments = { 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'check': flexmock(progress=flexmock(), repair=flexmock(), only=flexmock()), + 'check': flexmock( + progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock() + ), } list(