Config generation support for sequences of maps, needed for database dump hooks (#225).

This commit is contained in:
Dan Helfman 2019-10-21 15:17:47 -07:00
parent 17586d49ac
commit f8bc67be8d
7 changed files with 155 additions and 29 deletions

1
.gitignore vendored
View file

@ -5,6 +5,7 @@
.coverage .coverage
.pytest_cache .pytest_cache
.tox .tox
__pycache__
build/ build/
dist/ dist/
pip-wheel-metadata/ pip-wheel-metadata/

3
NEWS
View file

@ -1,3 +1,6 @@
1.3.27.dev0
* #225: Database dump/restore hooks for PostgreSQL (incomplete as of right now).
1.3.26 1.3.26
* #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful * #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful
(non-checkpoint) archives. (non-checkpoint) archives.

View file

@ -54,10 +54,10 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
# Add comments to each section, and then add comments to the fields in each section. # Add comments to each section, and then add comments to the fields in each section.
generate.add_comments_to_configuration(destination_config, schema) generate.add_comments_to_configuration_map(destination_config, schema)
for section_name, section_config in destination_config.items(): for section_name, section_config in destination_config.items():
generate.add_comments_to_configuration( generate.add_comments_to_configuration_map(
section_config, schema['map'][section_name], indent=generate.INDENT section_config, schema['map'][section_name], indent=generate.INDENT
) )

View file

@ -1,8 +1,11 @@
import io
import os import os
import re
from ruamel import yaml from ruamel import yaml
INDENT = 4 INDENT = 4
SEQUENCE_INDENT = 2
def _insert_newline_before_comment(config, field_name): def _insert_newline_before_comment(config, field_name):
@ -15,7 +18,7 @@ def _insert_newline_before_comment(config, field_name):
) )
def _schema_to_sample_configuration(schema, level=0): def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
''' '''
Given a loaded configuration schema, generate and return sample config for it. Include comments Given a loaded configuration schema, generate and return sample config for it. Include comments
for each section based on the schema "desc" description. for each section based on the schema "desc" description.
@ -24,14 +27,29 @@ def _schema_to_sample_configuration(schema, level=0):
if example is not None: if example is not None:
return example return example
if 'seq' in schema:
config = yaml.comments.CommentedSeq(
[
_schema_to_sample_configuration(item_schema, level, parent_is_sequence=True)
for item_schema in schema['seq']
]
)
add_comments_to_configuration_sequence(
config, schema, indent=(level * INDENT) + SEQUENCE_INDENT
)
elif 'map' in schema:
config = yaml.comments.CommentedMap( config = yaml.comments.CommentedMap(
[ [
(section_name, _schema_to_sample_configuration(section_schema, level + 1)) (section_name, _schema_to_sample_configuration(section_schema, level + 1))
for section_name, section_schema in schema['map'].items() for section_name, section_schema in schema['map'].items()
] ]
) )
indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
add_comments_to_configuration(config, schema, indent=(level * INDENT)) add_comments_to_configuration_map(
config, schema, indent=indent, skip_first=parent_is_sequence
)
else:
raise ValueError('Schema at level {} is unsupported: {}'.format(level, schema))
return config return config
@ -42,13 +60,12 @@ def _comment_out_line(line):
if not stripped_line or stripped_line.startswith('#'): if not stripped_line or stripped_line.startswith('#'):
return line return line
# Comment out the names of optional sections. # Comment out the names of optional sections, inserting the '#' after any indent for aesthetics.
one_indent = ' ' * INDENT matches = re.match(r'(\s*)', line)
if not line.startswith(one_indent): indent_spaces = matches.group(0) if matches else ''
return '# ' + line count_indent_spaces = len(indent_spaces)
# Otherwise, comment out the line, but insert the "#" after the first indent for aesthetics. return '# '.join((indent_spaces, line[count_indent_spaces:]))
return '# '.join((one_indent, line[INDENT:]))
REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'} REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
@ -90,7 +107,12 @@ def _render_configuration(config):
''' '''
Given a config data structure of nested OrderedDicts, render the config as YAML and return it. Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
''' '''
return yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT) dumper = yaml.YAML()
dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
rendered = io.StringIO()
dumper.dump(config, rendered)
return rendered.getvalue()
def write_configuration(config_filename, rendered_config, mode=0o600): def write_configuration(config_filename, rendered_config, mode=0o600):
@ -112,13 +134,49 @@ def write_configuration(config_filename, rendered_config, mode=0o600):
os.chmod(config_filename, mode) os.chmod(config_filename, mode)
def add_comments_to_configuration(config, schema, indent=0): def add_comments_to_configuration_sequence(config, schema, indent=0):
'''
If the given config sequence's items are maps, then mine the schema for the description of the
map's first item, and slap that atop the sequence. Indent the comment the given number of
characters.
Doing this for sequences of maps results in nice comments that look like:
```
things:
# First key description. Added by this function.
- key: foo
# Second key description. Added by add_comments_to_configuration_map().
other: bar
```
'''
if 'map' not in schema['seq'][0]:
return
for field_name in config[0].keys():
field_schema = schema['seq'][0]['map'].get(field_name, {})
description = field_schema.get('desc')
# No description to use? Skip it.
if not field_schema or not description:
return
config[0].yaml_set_start_comment(description, indent=indent)
# We only want the first key's description here, as the rest of the keys get commented by
# add_comments_to_configuration_map().
return
def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False):
''' '''
Using descriptions from a schema as a source, add those descriptions as comments to the given Using descriptions from a schema as a source, add those descriptions as comments to the given
config before each field. This function only adds comments for the top-most config map level. config mapping, before each field. Indent the comment the given number of characters.
Indent the comment the given number of characters.
''' '''
for index, field_name in enumerate(config.keys()): for index, field_name in enumerate(config.keys()):
if skip_first and index == 0:
continue
field_schema = schema['map'].get(field_name, {}) field_schema = schema['map'].get(field_name, {})
description = field_schema.get('desc') description = field_schema.get('desc')
@ -127,6 +185,7 @@ def add_comments_to_configuration(config, schema, indent=0):
continue continue
config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent) config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
if index > 0: if index > 0:
_insert_newline_before_comment(config, field_name) _insert_newline_before_comment(config, field_name)

View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup from setuptools import find_packages, setup
VERSION = '1.3.26' VERSION = '1.3.27.dev0'
setup( setup(

View file

@ -40,6 +40,12 @@ def test_comment_out_line_comments_indented_option():
assert module._comment_out_line(line) == ' # enabled: true' assert module._comment_out_line(line) == ' # enabled: true'
def test_comment_out_line_comments_twice_indented_option():
line = ' - item'
assert module._comment_out_line(line) == ' # - item'
def test_comment_out_optional_configuration_comments_optional_config_only(): def test_comment_out_optional_configuration_comments_optional_config_only():
flexmock(module)._comment_out_line = lambda line: '# ' + line flexmock(module)._comment_out_line = lambda line: '# ' + line
config = ''' config = '''
@ -74,10 +80,10 @@ location:
assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip() assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip()
def test_render_configuration_does_not_raise(): def test_render_configuration_converts_configuration_to_yaml_string():
flexmock(module.yaml).should_receive('round_trip_dump') yaml_string = module._render_configuration({'foo': 'bar'})
module._render_configuration({}) assert yaml_string == 'foo: bar\n'
def test_write_configuration_does_not_raise(): def test_write_configuration_does_not_raise():
@ -107,12 +113,33 @@ def test_write_configuration_with_already_existing_directory_does_not_raise():
module.write_configuration('config.yaml', 'config: yaml') module.write_configuration('config.yaml', 'config: yaml')
def test_add_comments_to_configuration_does_not_raise(): def test_add_comments_to_configuration_sequence_of_strings_does_not_raise():
config = module.yaml.comments.CommentedSeq(['foo', 'bar'])
schema = {'seq': [{'type': 'str'}]}
module.add_comments_to_configuration_sequence(config, schema)
def test_add_comments_to_configuration_sequence_of_maps_does_not_raise():
config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
schema = {'seq': [{'map': {'foo': {'desc': 'yo'}}}]}
module.add_comments_to_configuration_sequence(config, schema)
def test_add_comments_to_configuration_sequence_of_maps_without_description_does_not_raise():
config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
schema = {'seq': [{'map': {'foo': {}}}]}
module.add_comments_to_configuration_sequence(config, schema)
def test_add_comments_to_configuration_map_does_not_raise():
# Ensure that it can deal with fields both in the schema and missing from the schema. # Ensure that it can deal with fields both in the schema and missing from the schema.
config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)]) config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}} schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}}
module.add_comments_to_configuration(config, schema) module.add_comments_to_configuration_map(config, schema)
def test_generate_sample_configuration_does_not_raise(): def test_generate_sample_configuration_does_not_raise():

View file

@ -1,13 +1,14 @@
from collections import OrderedDict from collections import OrderedDict
import pytest
from flexmock import flexmock from flexmock import flexmock
from borgmatic.config import generate as module from borgmatic.config import generate as module
def test_schema_to_sample_configuration_generates_config_with_examples(): def test_schema_to_sample_configuration_generates_config_map_with_examples():
flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
flexmock(module).should_receive('add_comments_to_configuration') flexmock(module).should_receive('add_comments_to_configuration_map')
schema = { schema = {
'map': OrderedDict( 'map': OrderedDict(
[ [
@ -35,3 +36,38 @@ def test_schema_to_sample_configuration_generates_config_with_examples():
('section2', OrderedDict([('field2', 'Example 2'), ('field3', 'Example 3')])), ('section2', OrderedDict([('field2', 'Example 2'), ('field3', 'Example 3')])),
] ]
) )
def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example():
flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
flexmock(module).should_receive('add_comments_to_configuration_sequence')
schema = {'seq': [{'type': 'str'}], 'example': ['hi']}
config = module._schema_to_sample_configuration(schema)
assert config == ['hi']
def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_examples():
flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
flexmock(module).should_receive('add_comments_to_configuration_sequence')
schema = {
'seq': [
{
'map': OrderedDict(
[('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})]
)
}
]
}
config = module._schema_to_sample_configuration(schema)
assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])]
def test_schema_to_sample_configuration_with_unsupported_schema_raises():
schema = {'gobbledygook': [{'type': 'not-your'}]}
with pytest.raises(ValueError):
module._schema_to_sample_configuration(schema)