Only check archives with matching prefix.
This commit is contained in:
parent
0112407250
commit
c64d0100d5
8 changed files with 84 additions and 13 deletions
1
AUTHORS
1
AUTHORS
|
@ -8,3 +8,4 @@ newtonne: Read encryption password from external file
|
||||||
Robin `ypid` Schneider: Support additional options of Borg
|
Robin `ypid` Schneider: Support additional options of Borg
|
||||||
Scott Squires: Custom archive names
|
Scott Squires: Custom archive names
|
||||||
Thomas LÉVEIL: Support for a keep_minutely prune option
|
Thomas LÉVEIL: Support for a keep_minutely prune option
|
||||||
|
Nick Whyte: Support prefix filtering for archive consistency checks
|
||||||
|
|
|
@ -82,10 +82,13 @@ def check_archives(verbosity, repository, storage_config, consistency_config, lo
|
||||||
VERBOSITY_LOTS: ('--debug',),
|
VERBOSITY_LOTS: ('--debug',),
|
||||||
}.get(verbosity, ())
|
}.get(verbosity, ())
|
||||||
|
|
||||||
|
prefix = consistency_config.get('prefix', '{hostname}-')
|
||||||
|
prefix_flags = ('--prefix', prefix) if prefix else ()
|
||||||
|
|
||||||
full_command = (
|
full_command = (
|
||||||
local_path, 'check',
|
local_path, 'check',
|
||||||
repository,
|
repository,
|
||||||
) + _make_check_flags(checks, check_last) + remote_path_flags + lock_wait_flags + verbosity_flags
|
) + _make_check_flags(checks, check_last) + prefix_flags + remote_path_flags + lock_wait_flags + verbosity_flags
|
||||||
|
|
||||||
# The check command spews to stdout/stderr even without the verbose flag. Suppress it.
|
# The check command spews to stdout/stderr even without the verbose flag. Suppress it.
|
||||||
stdout = None if verbosity_flags else open(os.devnull, 'w')
|
stdout = None if verbosity_flags else open(os.devnull, 'w')
|
||||||
|
|
|
@ -218,6 +218,13 @@ map:
|
||||||
desc: Restrict the number of checked archives to the last n. Applies only to the
|
desc: Restrict the number of checked archives to the last n. Applies only to the
|
||||||
"archives" check.
|
"archives" check.
|
||||||
example: 3
|
example: 3
|
||||||
|
prefix:
|
||||||
|
type: scalar
|
||||||
|
desc: |
|
||||||
|
When performing consistency checks, only consider archive names starting with
|
||||||
|
this prefix. Borg placeholders can be used. See the output of
|
||||||
|
"borg help placeholders" for details. Default is "{hostname}-".
|
||||||
|
example: sourcehostname
|
||||||
hooks:
|
hooks:
|
||||||
desc: |
|
desc: |
|
||||||
Shell commands or scripts to execute before and after a backup or if an error has occurred.
|
Shell commands or scripts to execute before and after a backup or if an error has occurred.
|
||||||
|
|
|
@ -8,6 +8,8 @@ import pykwalify.errors
|
||||||
from ruamel import yaml
|
from ruamel import yaml
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
def schema_filename():
|
def schema_filename():
|
||||||
'''
|
'''
|
||||||
Path to the installed YAML configuration schema file, used to validate and parse the
|
Path to the installed YAML configuration schema file, used to validate and parse the
|
||||||
|
@ -50,6 +52,11 @@ def apply_logical_validation(config_filename, parsed_configuration):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix')
|
||||||
|
if archive_name_format and not consistency_prefix:
|
||||||
|
logger.warning('Since version 1.2.0, if you provide `archive_name_format`, you must also'
|
||||||
|
' specify `consistency.prefix`.')
|
||||||
|
|
||||||
|
|
||||||
def parse_configuration(config_filename, schema_filename):
|
def parse_configuration(config_filename, schema_filename):
|
||||||
'''
|
'''
|
||||||
|
|
|
@ -89,7 +89,7 @@ def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
|
||||||
)
|
)
|
||||||
def test_check_archives_calls_borg_with_parameters(checks):
|
def test_check_archives_calls_borg_with_parameters(checks):
|
||||||
check_last = flexmock()
|
check_last = flexmock()
|
||||||
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
consistency_config = {'check_last': check_last}
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
|
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
|
||||||
stdout = flexmock()
|
stdout = flexmock()
|
||||||
|
@ -111,7 +111,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
|
||||||
def test_check_archives_with_extract_check_calls_extract_only():
|
def test_check_archives_with_extract_check_calls_extract_only():
|
||||||
checks = ('extract',)
|
checks = ('extract',)
|
||||||
check_last = flexmock()
|
check_last = flexmock()
|
||||||
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
consistency_config = {'check_last': check_last}
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
flexmock(module).should_receive('_make_check_flags').never()
|
flexmock(module).should_receive('_make_check_flags').never()
|
||||||
flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
|
flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
|
||||||
|
@ -127,7 +127,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
|
||||||
|
|
||||||
def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
|
def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
|
||||||
checks = ('repository',)
|
checks = ('repository',)
|
||||||
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
consistency_config = {'check_last': None}
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
flexmock(module).should_receive('_make_check_flags').and_return(())
|
flexmock(module).should_receive('_make_check_flags').and_return(())
|
||||||
insert_subprocess_mock(
|
insert_subprocess_mock(
|
||||||
|
@ -145,7 +145,7 @@ def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
|
||||||
|
|
||||||
def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
|
def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
|
||||||
checks = ('repository',)
|
checks = ('repository',)
|
||||||
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
consistency_config = {'check_last': None}
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
flexmock(module).should_receive('_make_check_flags').and_return(())
|
flexmock(module).should_receive('_make_check_flags').and_return(())
|
||||||
insert_subprocess_mock(
|
insert_subprocess_mock(
|
||||||
|
@ -162,7 +162,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
|
||||||
|
|
||||||
|
|
||||||
def test_check_archives_without_any_checks_bails():
|
def test_check_archives_without_any_checks_bails():
|
||||||
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
consistency_config = {'check_last': None}
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(())
|
flexmock(module).should_receive('_parse_checks').and_return(())
|
||||||
insert_subprocess_never()
|
insert_subprocess_never()
|
||||||
|
|
||||||
|
@ -177,7 +177,7 @@ def test_check_archives_without_any_checks_bails():
|
||||||
def test_check_archives_with_local_path_calls_borg_via_local_path():
|
def test_check_archives_with_local_path_calls_borg_via_local_path():
|
||||||
checks = ('repository',)
|
checks = ('repository',)
|
||||||
check_last = flexmock()
|
check_last = flexmock()
|
||||||
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
consistency_config = {'check_last': check_last}
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
|
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
|
||||||
stdout = flexmock()
|
stdout = flexmock()
|
||||||
|
@ -200,7 +200,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
|
||||||
def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters():
|
def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters():
|
||||||
checks = ('repository',)
|
checks = ('repository',)
|
||||||
check_last = flexmock()
|
check_last = flexmock()
|
||||||
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
consistency_config = {'check_last': check_last}
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
|
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
|
||||||
stdout = flexmock()
|
stdout = flexmock()
|
||||||
|
@ -223,7 +223,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
|
||||||
def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
|
def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
|
||||||
checks = ('repository',)
|
checks = ('repository',)
|
||||||
check_last = flexmock()
|
check_last = flexmock()
|
||||||
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
consistency_config = {'check_last': check_last}
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
|
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
|
||||||
stdout = flexmock()
|
stdout = flexmock()
|
||||||
|
@ -240,3 +240,26 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
|
||||||
storage_config={'lock_wait': 5},
|
storage_config={'lock_wait': 5},
|
||||||
consistency_config=consistency_config,
|
consistency_config=consistency_config,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_archives_with_retention_prefix():
|
||||||
|
checks = ('repository',)
|
||||||
|
check_last = flexmock()
|
||||||
|
consistency_config = {'check_last': check_last, 'prefix': 'foo-'}
|
||||||
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
|
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
|
||||||
|
stdout = flexmock()
|
||||||
|
insert_subprocess_mock(
|
||||||
|
('borg', 'check', 'repo', '--prefix', 'foo-'),
|
||||||
|
stdout=stdout, stderr=STDOUT,
|
||||||
|
)
|
||||||
|
|
||||||
|
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
|
||||||
|
flexmock(module.os).should_receive('devnull')
|
||||||
|
|
||||||
|
module.check_archives(
|
||||||
|
verbosity=None,
|
||||||
|
repository='repo',
|
||||||
|
storage_config={},
|
||||||
|
consistency_config=consistency_config,
|
||||||
|
)
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import pytest
|
import pytest
|
||||||
|
from flexmock import flexmock
|
||||||
|
|
||||||
from borgmatic.config import validate as module
|
from borgmatic.config import validate as module
|
||||||
|
|
||||||
|
@ -23,13 +24,42 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_apply_logical_validation_raises_if_archive_name_format_present_without_retention_prefix():
|
||||||
|
with pytest.raises(module.Validation_error):
|
||||||
|
module.apply_logical_validation(
|
||||||
|
'config.yaml',
|
||||||
|
{
|
||||||
|
'storage': {'archive_name_format': '{hostname}-{now}'},
|
||||||
|
'retention': {'keep_daily': 7},
|
||||||
|
'consistency': {'prefix': '{hostname}-'}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_logical_validation_warns_if_archive_name_format_present_without_consistency_prefix():
|
||||||
|
logger = flexmock(module.logger)
|
||||||
|
logger.should_receive('warning').once()
|
||||||
|
|
||||||
def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present():
|
|
||||||
module.apply_logical_validation(
|
module.apply_logical_validation(
|
||||||
'config.yaml',
|
'config.yaml',
|
||||||
{
|
{
|
||||||
'storage': {'archive_name_format': '{hostname}-{now}'},
|
'storage': {'archive_name_format': '{hostname}-{now}'},
|
||||||
'retention': {'prefix': '{hostname}-'},
|
'retention': {'prefix': '{hostname}-'},
|
||||||
|
'consistency': {},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present():
|
||||||
|
logger = flexmock(module.logger)
|
||||||
|
logger.should_receive('warning').never()
|
||||||
|
|
||||||
|
module.apply_logical_validation(
|
||||||
|
'config.yaml',
|
||||||
|
{
|
||||||
|
'storage': {'archive_name_format': '{hostname}-{now}'},
|
||||||
|
'retention': {'prefix': '{hostname}-'},
|
||||||
|
'consistency': {'prefix': '{hostname}-'}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -7,5 +7,5 @@ def test_verbosity_to_log_level_maps_known_verbosity_to_log_level():
|
||||||
assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO
|
assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO
|
||||||
|
|
||||||
|
|
||||||
def test_verbosity_to_log_level_maps_unknown_verbosity_to_error_level():
|
def test_verbosity_to_log_level_maps_unknown_verbosity_to_warning_level():
|
||||||
assert module.verbosity_to_log_level('my pants') == logging.ERROR
|
assert module.verbosity_to_log_level('my pants') == logging.WARNING
|
||||||
|
|
|
@ -12,4 +12,4 @@ def verbosity_to_log_level(verbosity):
|
||||||
return {
|
return {
|
||||||
VERBOSITY_SOME: logging.INFO,
|
VERBOSITY_SOME: logging.INFO,
|
||||||
VERBOSITY_LOTS: logging.DEBUG,
|
VERBOSITY_LOTS: logging.DEBUG,
|
||||||
}.get(verbosity, logging.ERROR)
|
}.get(verbosity, logging.WARNING)
|
||||||
|
|
Loading…
Reference in a new issue