psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

New configuration section for customizing which Attic consistency checks run, if any.

Browse source
This commit is contained in:
Dan Helfman 2015-05-10 22:00:31 -07:00
parent 301eb4926e
commit df2d059af2
9 changed files with 251 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
NEWS
View file

@ -1,3 +1,7 @@
0.0.6
* New configuration section for customizing which Attic consistency checks run, if any.
0.0.5
* Fixed regression with --verbose output being buffered. This means dropping the helpful error

3
README.md
View file

@ -26,6 +26,9 @@ Here's an example config file:
keep_weekly: 4
keep_monthly: 6
[consistency]
checks: repository archives
Additionally, exclude patterns can be specified in a separate excludes config
file, one pattern per line.

67
atticmatic/attic.py
View file

@ -26,7 +26,7 @@ def create_archive(excludes_filename, verbose, source_directories, repository):
subprocess.check_call(command)
def make_prune_flags(retention_config):
def _make_prune_flags(retention_config):
'''
Given a retention config dict mapping from option name to value, tranform it into an iterable of
command-line name-value flag pairs.
@ -58,22 +58,77 @@ def prune_archives(verbose, repository, retention_config):
repository,
) + tuple(
element
for pair in make_prune_flags(retention_config)
for pair in _make_prune_flags(retention_config)
for element in pair
) + (('--verbose',) if verbose else ())
subprocess.check_call(command)
def check_archives(verbose, repository):
DEFAULT_CHECKS = ('repository', 'archives')
def _parse_checks(consistency_config):
'''
Given a verbosity flag and a local or remote repository path, check the contained attic archives
for consistency.
Given a consistency config with a space-separated "checks" option, transform it to a tuple of
named checks to run.
For example, given a retention config of:
{'checks': 'repository archives'}
This will be returned as:
('repository', 'archives')
If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
"disabled", return an empty tuple, meaning that no checks should be run.
'''
checks = consistency_config.get('checks', '').strip()
if not checks:
return DEFAULT_CHECKS
return tuple(
check for check in consistency_config['checks'].split(' ')
if check.lower() not in ('disabled', '')
)
def _make_check_flags(checks):
'''
Given a parsed sequence of checks, transform it into tuple of command-line flags.
For example, given parsed checks of:
('repository',)
This will be returned as:
('--repository-only',)
'''
if checks == DEFAULT_CHECKS:
return ()
return tuple(
'--{}-only'.format(check) for check in checks
)
def check_archives(verbose, repository, consistency_config):
'''
Given a verbosity flag, a local or remote repository path, and a consistency config dict, check
the contained attic archives for consistency.
If there are no consistency checks to run, skip running them.
'''
checks = _parse_checks(consistency_config)
if not checks:
return
command = (
'attic', 'check',
repository,
) + (('--verbose',) if verbose else ())
) + _make_check_flags(checks) + (('--verbose',) if verbose else ())
# Attic's check command spews to stdout even without the verbose flag. Suppress it.
stdout = None if verbose else open(os.devnull, 'w')

10
atticmatic/command.py
View file

@ -40,12 +40,12 @@ def parse_arguments(*arguments):
def main():
try:
args = parse_arguments(*sys.argv[1:])
location_config, retention_config = parse_configuration(args.config_filename)
repository = location_config['repository']
config = parse_configuration(args.config_filename)
repository = config.location['repository']
create_archive(args.excludes_filename, args.verbose, **location_config)
prune_archives(args.verbose, repository, retention_config)
check_archives(args.verbose, repository)
create_archive(args.excludes_filename, args.verbose, **config.location)
prune_archives(args.verbose, repository, config.retention)
check_archives(args.verbose, repository, config.consistency)
except (ValueError, IOError, CalledProcessError) as error:
print(error, file=sys.stderr)
sys.exit(1)

49
atticmatic/config.py
View file

@ -39,6 +39,12 @@ CONFIG_FORMAT = (
option('keep_yearly', int, required=False),
option('prefix', required=False),
),
),
Section_format(
'consistency',
(
option('checks', required=False),
),
)
)
@ -49,20 +55,34 @@ def validate_configuration_format(parser, config_format):
configuration file has the expected sections, that any required options are present in those
sections, and that there aren't any unexpected options.
A section is required if any of its contained options are required.
Raise ValueError if anything is awry.
'''
section_names = parser.sections()
required_section_names = tuple(section.name for section in config_format)
if set(section_names) != set(required_section_names):
raise ValueError(
'Expected config sections {} but found sections: {}'.format(
', '.join(required_section_names),
', '.join(section_names)
section_names = set(parser.sections())
required_section_names = tuple(
section.name for section in config_format
if any(option.required for option in section.options)
)
unknown_section_names = section_names - set(
section_format.name for section_format in config_format
)
if unknown_section_names:
raise ValueError(
'Unknown config sections found: {}'.format(', '.join(unknown_section_names))
)
missing_section_names = set(required_section_names) - section_names
if missing_section_names:
raise ValueError(
'Missing config sections: {}'.format(', '.join(missing_section_names))
)
for section_format in config_format:
if section_format.name not in section_names:
continue
option_names = parser.options(section_format.name)
expected_options = section_format.options
@ -90,6 +110,11 @@ def validate_configuration_format(parser, config_format):
)
# Describes a parsed configuration, where each attribute is the name of a configuration file section
# and each value is a dict of that section's parsed options.
Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT))
def parse_section_options(parser, section_format):
'''
Given an open ConfigParser and an expected section format, return the option values from that
@ -112,8 +137,8 @@ def parse_section_options(parser, section_format):
def parse_configuration(config_filename):
'''
Given a config filename of the expected format, return the parsed configuration as a tuple of
(location config, retention config) where each config is a dict of that section's options.
Given a config filename of the expected format, return the parsed configuration as Parsed_config
data structure.
Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
'''
@ -122,7 +147,9 @@ def parse_configuration(config_filename):
validate_configuration_format(parser, CONFIG_FORMAT)
return tuple(
return Parsed_config(
*(
parse_section_options(parser, section_format)
for section_format in CONFIG_FORMAT
)
)

70
atticmatic/tests/unit/test_attic.py
View file

@ -11,6 +11,12 @@ def insert_subprocess_mock(check_call_command, **kwargs):
flexmock(module).subprocess = subprocess
def insert_subprocess_never():
subprocess = flexmock()
subprocess.should_receive('check_call').never()
flexmock(module).subprocess = subprocess
def insert_platform_mock():
flexmock(module).platform = flexmock().should_receive('node').and_return('host').mock
@ -70,14 +76,14 @@ def test_make_prune_flags_should_return_flags_from_config():
)
)
result = module.make_prune_flags(retention_config)
result = module._make_prune_flags(retention_config)
assert tuple(result) == BASE_PRUNE_FLAGS
def test_prune_archives_should_call_attic_with_parameters():
retention_config = flexmock()
flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS,
)
insert_subprocess_mock(
@ -96,7 +102,7 @@ def test_prune_archives_should_call_attic_with_parameters():
def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters():
retention_config = flexmock()
flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS,
)
insert_subprocess_mock(
@ -113,7 +119,46 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
)
def test_parse_checks_returns_them_as_tuple():
checks = module._parse_checks({'checks': 'foo disabled bar'})
assert checks == ('foo', 'bar')
def test_parse_checks_with_missing_value_returns_defaults():
checks = module._parse_checks({})
assert checks == module.DEFAULT_CHECKS
def test_parse_checks_with_blank_value_returns_defaults():
checks = module._parse_checks({'checks': ''})
assert checks == module.DEFAULT_CHECKS
def test_parse_checks_with_disabled_returns_no_checks():
checks = module._parse_checks({'checks': 'disabled'})
assert checks == ()
def test_make_check_flags_with_checks_returns_flags():
flags = module._make_check_flags(('foo', 'bar'))
assert flags == ('--foo-only', '--bar-only')
def test_make_check_flags_with_default_checks_returns_no_flags():
flags = module._make_check_flags(module.DEFAULT_CHECKS)
assert flags == ()
def test_check_archives_should_call_attic_with_parameters():
consistency_config = flexmock()
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
flexmock(module).should_receive('_make_check_flags').and_return(())
stdout = flexmock()
insert_subprocess_mock(
('attic', 'check', 'repo'),
@ -127,10 +172,14 @@ def test_check_archives_should_call_attic_with_parameters():
module.check_archives(
verbose=False,
repository='repo',
consistency_config=consistency_config,
)
def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters():
consistency_config = flexmock()
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock(
('attic', 'check', 'repo', '--verbose'),
stdout=None,
@ -141,4 +190,19 @@ def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters()
module.check_archives(
verbose=True,
repository='repo',
consistency_config=consistency_config,
)
def test_check_archives_without_any_checks_should_bail():
consistency_config = flexmock()
flexmock(module).should_receive('_parse_checks').and_return(())
insert_subprocess_never()
module.check_archives(
verbose=False,
repository='repo',
consistency_config=consistency_config,
)

70
atticmatic/tests/unit/test_config.py
View file

@ -41,19 +41,55 @@ def test_validate_configuration_format_with_valid_config_should_not_raise():
module.validate_configuration_format(parser, config_format)
def test_validate_configuration_format_with_missing_section_should_raise():
def test_validate_configuration_format_with_missing_required_section_should_raise():
parser = flexmock()
parser.should_receive('sections').and_return(('section',))
config_format = (
module.Section_format('section', options=()),
module.Section_format('missing', options=()),
module.Section_format(
'section',
options=(
module.Config_option('stuff', str, required=True),
),
),
# At least one option in this section is required, so the section is required.
module.Section_format(
'missing',
options=(
module.Config_option('such', str, required=False),
module.Config_option('things', str, required=True),
),
),
)
with assert_raises(ValueError):
module.validate_configuration_format(parser, config_format)
def test_validate_configuration_format_with_extra_section_should_raise():
def test_validate_configuration_format_with_missing_optional_section_should_not_raise():
parser = flexmock()
parser.should_receive('sections').and_return(('section',))
parser.should_receive('options').with_args('section').and_return(('stuff',))
config_format = (
module.Section_format(
'section',
options=(
module.Config_option('stuff', str, required=True),
),
),
# No options in the section are required, so the section is optional.
module.Section_format(
'missing',
options=(
module.Config_option('such', str, required=False),
module.Config_option('things', str, required=False),
),
),
)
module.validate_configuration_format(parser, config_format)
def test_validate_configuration_format_with_unknown_section_should_raise():
parser = flexmock()
parser.should_receive('sections').and_return(('section', 'extra'))
config_format = (
@ -139,6 +175,26 @@ def test_parse_section_options_should_return_section_options():
)
def test_parse_section_options_for_missing_section_should_return_empty_dict():
parser = flexmock()
parser.should_receive('get').never()
parser.should_receive('getint').never()
parser.should_receive('has_option').with_args('section', 'foo').and_return(False)
parser.should_receive('has_option').with_args('section', 'bar').and_return(False)
section_format = module.Section_format(
'section',
(
module.Config_option('foo', str, required=False),
module.Config_option('bar', int, required=False),
),
)
config = module.parse_section_options(parser, section_format)
assert config == OrderedDict()
def insert_mock_parser():
parser = flexmock()
parser.should_receive('readfp')
@ -154,13 +210,13 @@ def test_parse_configuration_should_return_section_configs():
mock_module.should_receive('validate_configuration_format').with_args(
parser, module.CONFIG_FORMAT,
).once()
mock_section_configs = (flexmock(), flexmock())
mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT)
for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs):
mock_module.should_receive('parse_section_options').with_args(
parser, section_format,
).and_return(section_config).once()
section_configs = module.parse_configuration('filename')
parsed_config = module.parse_configuration('filename')
assert section_configs == mock_section_configs
assert parsed_config == module.Parsed_config(*mock_section_configs)

10
sample/config
View file

@ -6,8 +6,8 @@ source_directories: /home /etc
repository: user@backupserver:sourcehostname.attic
[retention]
# Retention policy for how many backups to keep in each category.
# See https://attic-backup.org/usage.html#attic-prune for details.
# Retention policy for how many backups to keep in each category. See
# https://attic-backup.org/usage.html#attic-prune for details.
#keep_within: 3h
#keep_hourly: 24
keep_daily: 7
@ -15,3 +15,9 @@ keep_weekly: 4
keep_monthly: 6
keep_yearly: 1
#prefix: sourcehostname
[consistency]
# Space-separated list of consistency checks to run: "repository", "archives",
# or both. Defaults to both. Set to "disabled" to disable all consistency
# checks. See https://attic-backup.org/usage.html#attic-check for details.
checks: repository archives

2
setup.py
View file

@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='atticmatic',
version='0.0.5',
version='0.0.6',
description='A wrapper script for Attic backup software that creates and prunes backups',
author='Dan Helfman',
author_email='witten@torsion.org',