From 9c0687407311e1d01bc9ee08f7cc3f1a12d68719 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Jul 2015 21:47:52 -0700 Subject: [PATCH] #1: Add support for "borg check --last N" to Borg backend. --- NEWS | 4 ++ atticmatic/backends/attic.py | 2 +- atticmatic/backends/borg.py | 12 +++++ atticmatic/backends/shared.py | 42 ++++++++++++++-- atticmatic/command.py | 4 +- atticmatic/config.py | 48 ++++--------------- atticmatic/tests/unit/backends/test_shared.py | 26 +++++++--- atticmatic/tests/unit/test_config.py | 11 +++-- sample/config | 2 + 9 files changed, 94 insertions(+), 57 deletions(-) diff --git a/NEWS b/NEWS index db51ca3..207769e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.1.3 + + * #1: Add support for "borg check --last N" to Borg backend. + 0.1.2 * As a convenience to new users, allow a missing default excludes file. diff --git a/atticmatic/backends/attic.py b/atticmatic/backends/attic.py index a99c323..fcacce3 100644 --- a/atticmatic/backends/attic.py +++ b/atticmatic/backends/attic.py @@ -5,7 +5,7 @@ from atticmatic.backends import shared # An atticmatic backend that supports Attic for actually handling backups. COMMAND = 'attic' - +CONFIG_FORMAT = shared.CONFIG_FORMAT create_archive = partial(shared.create_archive, command=COMMAND) prune_archives = partial(shared.prune_archives, command=COMMAND) diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py index 5214ca1..bd6a386 100644 --- a/atticmatic/backends/borg.py +++ b/atticmatic/backends/borg.py @@ -1,10 +1,22 @@ from functools import partial +from atticmatic.config import Section_format, option from atticmatic.backends import shared # An atticmatic backend that supports Borg for actually handling backups. COMMAND = 'borg' +CONFIG_FORMAT = ( + shared.CONFIG_FORMAT[0], # location + shared.CONFIG_FORMAT[1], # retention + Section_format( + 'consistency', + ( + option('checks', required=False), + option('check_last', required=False), + ), + ) +) create_archive = partial(shared.create_archive, command=COMMAND) diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py index 8abd064..74f5533 100644 --- a/atticmatic/backends/shared.py +++ b/atticmatic/backends/shared.py @@ -3,6 +3,7 @@ import os import platform import subprocess +from atticmatic.config import Section_format, option from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS @@ -12,6 +13,34 @@ from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS # atticmatic.backends.borg. +CONFIG_FORMAT = ( + Section_format( + 'location', + ( + option('source_directories'), + option('repository'), + ), + ), + Section_format( + 'retention', + ( + option('keep_within', required=False), + option('keep_hourly', int, required=False), + option('keep_daily', int, required=False), + option('keep_weekly', int, required=False), + option('keep_monthly', int, required=False), + option('keep_yearly', int, required=False), + option('prefix', required=False), + ), + ), + Section_format( + 'consistency', + ( + option('checks', required=False), + ), + ) +) + def create_archive(excludes_filename, verbosity, source_directories, repository, command): ''' Given an excludes filename (or None), a vebosity flag, a space-separated list of source @@ -110,7 +139,7 @@ def _parse_checks(consistency_config): ) -def _make_check_flags(checks): +def _make_check_flags(checks, check_last=None): ''' Given a parsed sequence of checks, transform it into tuple of command-line flags. @@ -121,13 +150,17 @@ def _make_check_flags(checks): This will be returned as: ('--repository-only',) + + Additionally, if a check_last value is given, a "--last" flag will be added. Note that only + Borg supports this flag. ''' + last_flag = ('--last', check_last) if check_last else () if checks == DEFAULT_CHECKS: - return () + return last_flag return tuple( '--{}-only'.format(check) for check in checks - ) + ) + last_flag def check_archives(verbosity, repository, consistency_config, command): @@ -138,6 +171,7 @@ def check_archives(verbosity, repository, consistency_config, command): If there are no consistency checks to run, skip running them. ''' checks = _parse_checks(consistency_config) + check_last = consistency_config.get('check_last', None) if not checks: return @@ -149,7 +183,7 @@ def check_archives(verbosity, repository, consistency_config, command): full_command = ( command, 'check', repository, - ) + _make_check_flags(checks) + verbosity_flags + ) + _make_check_flags(checks, check_last) + verbosity_flags # The check command spews to stdout even without the verbose flag. Suppress it. stdout = None if verbosity_flags else open(os.devnull, 'w') diff --git a/atticmatic/command.py b/atticmatic/command.py index 13e7b43..0f512e1 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -60,9 +60,9 @@ def main(): try: command_name = os.path.basename(sys.argv[0]) args = parse_arguments(command_name, *sys.argv[1:]) - config = parse_configuration(args.config_filename) - repository = config.location['repository'] backend = load_backend(command_name) + config = parse_configuration(args.config_filename, backend.CONFIG_FORMAT) + repository = config.location['repository'] backend.create_archive(args.excludes_filename, args.verbosity, **config.location) backend.prune_archives(args.verbosity, repository, config.retention) diff --git a/atticmatic/config.py b/atticmatic/config.py index 7474f05..bc1a32d 100644 --- a/atticmatic/config.py +++ b/atticmatic/config.py @@ -20,35 +20,6 @@ def option(name, value_type=str, required=True): return Config_option(name, value_type, required) -CONFIG_FORMAT = ( - Section_format( - 'location', - ( - option('source_directories'), - option('repository'), - ), - ), - Section_format( - 'retention', - ( - option('keep_within', required=False), - option('keep_hourly', int, required=False), - option('keep_daily', int, required=False), - option('keep_weekly', int, required=False), - option('keep_monthly', int, required=False), - option('keep_yearly', int, required=False), - option('prefix', required=False), - ), - ), - Section_format( - 'consistency', - ( - option('checks', required=False), - ), - ) -) - - def validate_configuration_format(parser, config_format): ''' Given an open ConfigParser and an expected config file format, validate that the parsed @@ -110,11 +81,6 @@ def validate_configuration_format(parser, config_format): ) -# Describes a parsed configuration, where each attribute is the name of a configuration file section -# and each value is a dict of that section's parsed options. -Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT)) - - def parse_section_options(parser, section_format): ''' Given an open ConfigParser and an expected section format, return the option values from that @@ -135,21 +101,25 @@ def parse_section_options(parser, section_format): ) -def parse_configuration(config_filename): +def parse_configuration(config_filename, config_format): ''' - Given a config filename of the expected format, return the parsed configuration as Parsed_config - data structure. + Given a config filename and an expected config file format, return the parsed configuration + as a namedtuple with one attribute for each parsed section. Raise IOError if the file cannot be read, or ValueError if the format is not as expected. ''' parser = ConfigParser() parser.read(config_filename) - validate_configuration_format(parser, CONFIG_FORMAT) + validate_configuration_format(parser, config_format) + + # Describes a parsed configuration, where each attribute is the name of a configuration file + # section and each value is a dict of that section's parsed options. + Parsed_config = namedtuple('Parsed_config', (section_format.name for section_format in config_format)) return Parsed_config( *( parse_section_options(parser, section_format) - for section_format in CONFIG_FORMAT + for section_format in config_format ) ) diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py index 8fe236f..c742087 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -196,10 +196,24 @@ def test_make_check_flags_with_default_checks_returns_no_flags(): assert flags == () +def test_make_check_flags_with_checks_and_last_returns_flags_including_last(): + flags = module._make_check_flags(('foo', 'bar'), check_last=3) + + assert flags == ('--foo-only', '--bar-only', '--last', 3) + + +def test_make_check_flags_with_last_returns_last_flag(): + flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3) + + assert flags == ('--last', 3) + + def test_check_archives_should_call_attic_with_parameters(): - consistency_config = flexmock() - flexmock(module).should_receive('_parse_checks').and_return(flexmock()) - flexmock(module).should_receive('_make_check_flags').and_return(()) + checks = flexmock() + check_last = flexmock() + consistency_config = flexmock().should_receive('get').and_return(check_last).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) stdout = flexmock() insert_subprocess_mock( ('attic', 'check', 'repo'), @@ -219,7 +233,7 @@ def test_check_archives_should_call_attic_with_parameters(): def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter(): - consistency_config = flexmock() + consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( @@ -238,7 +252,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter(): - consistency_config = flexmock() + consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( @@ -257,7 +271,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param def test_check_archives_without_any_checks_should_bail(): - consistency_config = flexmock() + consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(()) insert_subprocess_never() diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py index 02f1f52..4c889b0 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/atticmatic/tests/unit/test_config.py @@ -205,17 +205,18 @@ def insert_mock_parser(): def test_parse_configuration_should_return_section_configs(): parser = insert_mock_parser() + config_format = (flexmock(name='items'), flexmock(name='things')) mock_module = flexmock(module) mock_module.should_receive('validate_configuration_format').with_args( - parser, module.CONFIG_FORMAT, + parser, config_format, ).once() - mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT) + mock_section_configs = (flexmock(), flexmock()) - for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs): + for section_format, section_config in zip(config_format, mock_section_configs): mock_module.should_receive('parse_section_options').with_args( parser, section_format, ).and_return(section_config).once() - parsed_config = module.parse_configuration('filename') + parsed_config = module.parse_configuration('filename', config_format) - assert parsed_config == module.Parsed_config(*mock_section_configs) + assert parsed_config == type(parsed_config)(*mock_section_configs) diff --git a/sample/config b/sample/config index c83c554..82d77d1 100644 --- a/sample/config +++ b/sample/config @@ -23,3 +23,5 @@ keep_yearly: 1 # checks. See https://attic-backup.org/usage.html#attic-check or # https://borgbackup.github.io/borgbackup/usage.html#borg-check for details. checks: repository archives +# For Borg only, you can restrict the number of checked archives to the last n. +#check_last: 3