Config generation support for sequences of maps, needed for database dump hooks (#225).
This commit is contained in:
parent
17586d49ac
commit
f8bc67be8d
7 changed files with 155 additions and 29 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -5,6 +5,7 @@
|
||||||
.coverage
|
.coverage
|
||||||
.pytest_cache
|
.pytest_cache
|
||||||
.tox
|
.tox
|
||||||
|
__pycache__
|
||||||
build/
|
build/
|
||||||
dist/
|
dist/
|
||||||
pip-wheel-metadata/
|
pip-wheel-metadata/
|
||||||
|
|
3
NEWS
3
NEWS
|
@ -1,3 +1,6 @@
|
||||||
|
1.3.27.dev0
|
||||||
|
* #225: Database dump/restore hooks for PostgreSQL (incomplete as of right now).
|
||||||
|
|
||||||
1.3.26
|
1.3.26
|
||||||
* #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful
|
* #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful
|
||||||
(non-checkpoint) archives.
|
(non-checkpoint) archives.
|
||||||
|
|
|
@ -54,10 +54,10 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
|
||||||
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
|
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
|
||||||
|
|
||||||
# Add comments to each section, and then add comments to the fields in each section.
|
# Add comments to each section, and then add comments to the fields in each section.
|
||||||
generate.add_comments_to_configuration(destination_config, schema)
|
generate.add_comments_to_configuration_map(destination_config, schema)
|
||||||
|
|
||||||
for section_name, section_config in destination_config.items():
|
for section_name, section_config in destination_config.items():
|
||||||
generate.add_comments_to_configuration(
|
generate.add_comments_to_configuration_map(
|
||||||
section_config, schema['map'][section_name], indent=generate.INDENT
|
section_config, schema['map'][section_name], indent=generate.INDENT
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -1,8 +1,11 @@
|
||||||
|
import io
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
|
|
||||||
from ruamel import yaml
|
from ruamel import yaml
|
||||||
|
|
||||||
INDENT = 4
|
INDENT = 4
|
||||||
|
SEQUENCE_INDENT = 2
|
||||||
|
|
||||||
|
|
||||||
def _insert_newline_before_comment(config, field_name):
|
def _insert_newline_before_comment(config, field_name):
|
||||||
|
@ -15,7 +18,7 @@ def _insert_newline_before_comment(config, field_name):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _schema_to_sample_configuration(schema, level=0):
|
def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
|
||||||
'''
|
'''
|
||||||
Given a loaded configuration schema, generate and return sample config for it. Include comments
|
Given a loaded configuration schema, generate and return sample config for it. Include comments
|
||||||
for each section based on the schema "desc" description.
|
for each section based on the schema "desc" description.
|
||||||
|
@ -24,14 +27,29 @@ def _schema_to_sample_configuration(schema, level=0):
|
||||||
if example is not None:
|
if example is not None:
|
||||||
return example
|
return example
|
||||||
|
|
||||||
|
if 'seq' in schema:
|
||||||
|
config = yaml.comments.CommentedSeq(
|
||||||
|
[
|
||||||
|
_schema_to_sample_configuration(item_schema, level, parent_is_sequence=True)
|
||||||
|
for item_schema in schema['seq']
|
||||||
|
]
|
||||||
|
)
|
||||||
|
add_comments_to_configuration_sequence(
|
||||||
|
config, schema, indent=(level * INDENT) + SEQUENCE_INDENT
|
||||||
|
)
|
||||||
|
elif 'map' in schema:
|
||||||
config = yaml.comments.CommentedMap(
|
config = yaml.comments.CommentedMap(
|
||||||
[
|
[
|
||||||
(section_name, _schema_to_sample_configuration(section_schema, level + 1))
|
(section_name, _schema_to_sample_configuration(section_schema, level + 1))
|
||||||
for section_name, section_schema in schema['map'].items()
|
for section_name, section_schema in schema['map'].items()
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
|
||||||
add_comments_to_configuration(config, schema, indent=(level * INDENT))
|
add_comments_to_configuration_map(
|
||||||
|
config, schema, indent=indent, skip_first=parent_is_sequence
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError('Schema at level {} is unsupported: {}'.format(level, schema))
|
||||||
|
|
||||||
return config
|
return config
|
||||||
|
|
||||||
|
@ -42,13 +60,12 @@ def _comment_out_line(line):
|
||||||
if not stripped_line or stripped_line.startswith('#'):
|
if not stripped_line or stripped_line.startswith('#'):
|
||||||
return line
|
return line
|
||||||
|
|
||||||
# Comment out the names of optional sections.
|
# Comment out the names of optional sections, inserting the '#' after any indent for aesthetics.
|
||||||
one_indent = ' ' * INDENT
|
matches = re.match(r'(\s*)', line)
|
||||||
if not line.startswith(one_indent):
|
indent_spaces = matches.group(0) if matches else ''
|
||||||
return '# ' + line
|
count_indent_spaces = len(indent_spaces)
|
||||||
|
|
||||||
# Otherwise, comment out the line, but insert the "#" after the first indent for aesthetics.
|
return '# '.join((indent_spaces, line[count_indent_spaces:]))
|
||||||
return '# '.join((one_indent, line[INDENT:]))
|
|
||||||
|
|
||||||
|
|
||||||
REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
|
REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
|
||||||
|
@ -90,7 +107,12 @@ def _render_configuration(config):
|
||||||
'''
|
'''
|
||||||
Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
|
Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
|
||||||
'''
|
'''
|
||||||
return yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)
|
dumper = yaml.YAML()
|
||||||
|
dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
|
||||||
|
rendered = io.StringIO()
|
||||||
|
dumper.dump(config, rendered)
|
||||||
|
|
||||||
|
return rendered.getvalue()
|
||||||
|
|
||||||
|
|
||||||
def write_configuration(config_filename, rendered_config, mode=0o600):
|
def write_configuration(config_filename, rendered_config, mode=0o600):
|
||||||
|
@ -112,13 +134,49 @@ def write_configuration(config_filename, rendered_config, mode=0o600):
|
||||||
os.chmod(config_filename, mode)
|
os.chmod(config_filename, mode)
|
||||||
|
|
||||||
|
|
||||||
def add_comments_to_configuration(config, schema, indent=0):
|
def add_comments_to_configuration_sequence(config, schema, indent=0):
|
||||||
|
'''
|
||||||
|
If the given config sequence's items are maps, then mine the schema for the description of the
|
||||||
|
map's first item, and slap that atop the sequence. Indent the comment the given number of
|
||||||
|
characters.
|
||||||
|
|
||||||
|
Doing this for sequences of maps results in nice comments that look like:
|
||||||
|
|
||||||
|
```
|
||||||
|
things:
|
||||||
|
# First key description. Added by this function.
|
||||||
|
- key: foo
|
||||||
|
# Second key description. Added by add_comments_to_configuration_map().
|
||||||
|
other: bar
|
||||||
|
```
|
||||||
|
'''
|
||||||
|
if 'map' not in schema['seq'][0]:
|
||||||
|
return
|
||||||
|
|
||||||
|
for field_name in config[0].keys():
|
||||||
|
field_schema = schema['seq'][0]['map'].get(field_name, {})
|
||||||
|
description = field_schema.get('desc')
|
||||||
|
|
||||||
|
# No description to use? Skip it.
|
||||||
|
if not field_schema or not description:
|
||||||
|
return
|
||||||
|
|
||||||
|
config[0].yaml_set_start_comment(description, indent=indent)
|
||||||
|
|
||||||
|
# We only want the first key's description here, as the rest of the keys get commented by
|
||||||
|
# add_comments_to_configuration_map().
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False):
|
||||||
'''
|
'''
|
||||||
Using descriptions from a schema as a source, add those descriptions as comments to the given
|
Using descriptions from a schema as a source, add those descriptions as comments to the given
|
||||||
config before each field. This function only adds comments for the top-most config map level.
|
config mapping, before each field. Indent the comment the given number of characters.
|
||||||
Indent the comment the given number of characters.
|
|
||||||
'''
|
'''
|
||||||
for index, field_name in enumerate(config.keys()):
|
for index, field_name in enumerate(config.keys()):
|
||||||
|
if skip_first and index == 0:
|
||||||
|
continue
|
||||||
|
|
||||||
field_schema = schema['map'].get(field_name, {})
|
field_schema = schema['map'].get(field_name, {})
|
||||||
description = field_schema.get('desc')
|
description = field_schema.get('desc')
|
||||||
|
|
||||||
|
@ -127,6 +185,7 @@ def add_comments_to_configuration(config, schema, indent=0):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
|
config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
|
||||||
|
|
||||||
if index > 0:
|
if index > 0:
|
||||||
_insert_newline_before_comment(config, field_name)
|
_insert_newline_before_comment(config, field_name)
|
||||||
|
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -1,6 +1,6 @@
|
||||||
from setuptools import find_packages, setup
|
from setuptools import find_packages, setup
|
||||||
|
|
||||||
VERSION = '1.3.26'
|
VERSION = '1.3.27.dev0'
|
||||||
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
|
|
|
@ -40,6 +40,12 @@ def test_comment_out_line_comments_indented_option():
|
||||||
assert module._comment_out_line(line) == ' # enabled: true'
|
assert module._comment_out_line(line) == ' # enabled: true'
|
||||||
|
|
||||||
|
|
||||||
|
def test_comment_out_line_comments_twice_indented_option():
|
||||||
|
line = ' - item'
|
||||||
|
|
||||||
|
assert module._comment_out_line(line) == ' # - item'
|
||||||
|
|
||||||
|
|
||||||
def test_comment_out_optional_configuration_comments_optional_config_only():
|
def test_comment_out_optional_configuration_comments_optional_config_only():
|
||||||
flexmock(module)._comment_out_line = lambda line: '# ' + line
|
flexmock(module)._comment_out_line = lambda line: '# ' + line
|
||||||
config = '''
|
config = '''
|
||||||
|
@ -74,10 +80,10 @@ location:
|
||||||
assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip()
|
assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip()
|
||||||
|
|
||||||
|
|
||||||
def test_render_configuration_does_not_raise():
|
def test_render_configuration_converts_configuration_to_yaml_string():
|
||||||
flexmock(module.yaml).should_receive('round_trip_dump')
|
yaml_string = module._render_configuration({'foo': 'bar'})
|
||||||
|
|
||||||
module._render_configuration({})
|
assert yaml_string == 'foo: bar\n'
|
||||||
|
|
||||||
|
|
||||||
def test_write_configuration_does_not_raise():
|
def test_write_configuration_does_not_raise():
|
||||||
|
@ -107,12 +113,33 @@ def test_write_configuration_with_already_existing_directory_does_not_raise():
|
||||||
module.write_configuration('config.yaml', 'config: yaml')
|
module.write_configuration('config.yaml', 'config: yaml')
|
||||||
|
|
||||||
|
|
||||||
def test_add_comments_to_configuration_does_not_raise():
|
def test_add_comments_to_configuration_sequence_of_strings_does_not_raise():
|
||||||
|
config = module.yaml.comments.CommentedSeq(['foo', 'bar'])
|
||||||
|
schema = {'seq': [{'type': 'str'}]}
|
||||||
|
|
||||||
|
module.add_comments_to_configuration_sequence(config, schema)
|
||||||
|
|
||||||
|
|
||||||
|
def test_add_comments_to_configuration_sequence_of_maps_does_not_raise():
|
||||||
|
config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
|
||||||
|
schema = {'seq': [{'map': {'foo': {'desc': 'yo'}}}]}
|
||||||
|
|
||||||
|
module.add_comments_to_configuration_sequence(config, schema)
|
||||||
|
|
||||||
|
|
||||||
|
def test_add_comments_to_configuration_sequence_of_maps_without_description_does_not_raise():
|
||||||
|
config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
|
||||||
|
schema = {'seq': [{'map': {'foo': {}}}]}
|
||||||
|
|
||||||
|
module.add_comments_to_configuration_sequence(config, schema)
|
||||||
|
|
||||||
|
|
||||||
|
def test_add_comments_to_configuration_map_does_not_raise():
|
||||||
# Ensure that it can deal with fields both in the schema and missing from the schema.
|
# Ensure that it can deal with fields both in the schema and missing from the schema.
|
||||||
config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
|
config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
|
||||||
schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}}
|
schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}}
|
||||||
|
|
||||||
module.add_comments_to_configuration(config, schema)
|
module.add_comments_to_configuration_map(config, schema)
|
||||||
|
|
||||||
|
|
||||||
def test_generate_sample_configuration_does_not_raise():
|
def test_generate_sample_configuration_does_not_raise():
|
||||||
|
|
|
@ -1,13 +1,14 @@
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
|
||||||
|
import pytest
|
||||||
from flexmock import flexmock
|
from flexmock import flexmock
|
||||||
|
|
||||||
from borgmatic.config import generate as module
|
from borgmatic.config import generate as module
|
||||||
|
|
||||||
|
|
||||||
def test_schema_to_sample_configuration_generates_config_with_examples():
|
def test_schema_to_sample_configuration_generates_config_map_with_examples():
|
||||||
flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
|
flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
|
||||||
flexmock(module).should_receive('add_comments_to_configuration')
|
flexmock(module).should_receive('add_comments_to_configuration_map')
|
||||||
schema = {
|
schema = {
|
||||||
'map': OrderedDict(
|
'map': OrderedDict(
|
||||||
[
|
[
|
||||||
|
@ -35,3 +36,38 @@ def test_schema_to_sample_configuration_generates_config_with_examples():
|
||||||
('section2', OrderedDict([('field2', 'Example 2'), ('field3', 'Example 3')])),
|
('section2', OrderedDict([('field2', 'Example 2'), ('field3', 'Example 3')])),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example():
|
||||||
|
flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
|
||||||
|
flexmock(module).should_receive('add_comments_to_configuration_sequence')
|
||||||
|
schema = {'seq': [{'type': 'str'}], 'example': ['hi']}
|
||||||
|
|
||||||
|
config = module._schema_to_sample_configuration(schema)
|
||||||
|
|
||||||
|
assert config == ['hi']
|
||||||
|
|
||||||
|
|
||||||
|
def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_examples():
|
||||||
|
flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
|
||||||
|
flexmock(module).should_receive('add_comments_to_configuration_sequence')
|
||||||
|
schema = {
|
||||||
|
'seq': [
|
||||||
|
{
|
||||||
|
'map': OrderedDict(
|
||||||
|
[('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})]
|
||||||
|
)
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
config = module._schema_to_sample_configuration(schema)
|
||||||
|
|
||||||
|
assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])]
|
||||||
|
|
||||||
|
|
||||||
|
def test_schema_to_sample_configuration_with_unsupported_schema_raises():
|
||||||
|
schema = {'gobbledygook': [{'type': 'not-your'}]}
|
||||||
|
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
module._schema_to_sample_configuration(schema)
|
||||||
|
|
Loading…
Reference in a new issue