psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Selectively omit list values when including configuration files (#672).

Browse source
This commit is contained in:
Dan Helfman 2023-04-13 14:39:36 -07:00
parent 08843d51d9
commit 4a94c2c9bf
4 changed files with 362 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
NEWS
View file

@ -10,6 +10,9 @@
* #672: Selectively shallow merge certain mappings or sequences when including configuration files.
See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge
* #672: Selectively omit list values when including configuration files. See the documentation for
more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#list-merge
* #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag.
See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes

32
borgmatic/config/load.py
View file

@ -38,9 +38,9 @@ def include_configuration(loader, filename_node, include_directory):
return load_configuration(include_filename)
def retain_node_error(loader, node):
def raise_retain_node_error(loader, node):
'''
Given a ruamel.yaml.loader.Loader and a YAML node, raise an error.
Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!retain" usage.
Raise ValueError if a mapping or sequence node is given, as that indicates that "!retain" was
used in a configuration file without a merge. In configuration files with a merge, mapping and
@ -56,6 +56,19 @@ def retain_node_error(loader, node):
raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.')
def raise_omit_node_error(loader, node):
'''
Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!omit" usage.
Raise ValueError unconditionally, as an "!omit" node here indicates it was used in a
configuration file without a merge. In configuration files with a merge, nodes with "!omit"
tags are handled by deep_merge_nodes() below.
'''
raise ValueError(
'The !omit tag may only be used on a scalar (e.g., string) list element within a configuration file containing a merged !include tag.'
)
class Include_constructor(ruamel.yaml.SafeConstructor):
'''
A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
@ -68,7 +81,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
'!include',
functools.partial(include_configuration, include_directory=include_directory),
)
self.add_constructor('!retain', retain_node_error)
self.add_constructor('!retain', raise_retain_node_error)
self.add_constructor('!omit', raise_omit_node_error)
def flatten_mapping(self, node):
'''
@ -134,6 +148,16 @@ def load_configuration(filename):
return config
def filter_omitted_nodes(nodes):
'''
Given a list of nodes, return a filtered list omitting any nodes with an "!omit" tag or with a
value matching such nodes.
'''
omitted_values = tuple(node.value for node in nodes if node.tag == '!omit')
return [node for node in nodes if node.value not in omitted_values]
DELETED_NODE = object()
@ -247,7 +271,7 @@ def deep_merge_nodes(nodes):
b_key,
ruamel.yaml.nodes.SequenceNode(
tag=b_value.tag,
value=a_value.value + b_value.value,
value=filter_omitted_nodes(a_value.value + b_value.value),
start_mark=b_value.start_mark,
end_mark=b_value.end_mark,
flow_style=b_value.flow_style,

60
docs/how-to/make-per-application-backups.md
View file

@ -272,9 +272,65 @@ Once this include gets merged in, the resulting configuration would have a
When there's an option collision between the local file and the merged
include, the local file's option takes precedence.
#### List merge
<span class="minilink minilink-addedin">New in version 1.6.1</span> Colliding
list values are appended together.
<span class="minilink minilink-addedin">New in version 1.7.12</span> If there
is a list value from an include that you *don't* want in your local
configuration file, you can omit it with an `!omit` tag. For instance:
```yaml
<<: !include /etc/borgmatic/common.yaml
location:
source_directories:
- !omit /home
- /var
```
And `common.yaml` like this:
```yaml
location:
source_directories:
- /home
- /etc
```
Once this include gets merged in, the resulting configuration will have a
`source_directories` value of `/etc` and `/var`—with `/home` omitted.
This feature currently only works on scalar (e.g. string or number) list items
and will not work elsewhere in a configuration file. Be sure to put the
`!omit` tag *before* the list item (after the dash). Putting `!omit` after the
list item will not work, as it gets interpreted as part of the string. Here's
an example of some things not to do:
```yaml
<<: !include /etc/borgmatic/common.yaml
location:
source_directories:
# Do not do this! It will not work. "!omit" belongs before "/home".
- /home !omit
# Do not do this either! "!omit" only works on scalar list items.
repositories: !omit
# Also do not do this for the same reason! This is a list item, but it's
# not a scalar.
- !omit path: repo.borg
```
Additionally, the `!omit` tag only works in a configuration file that also
performs a merge include with `<<: !include`. It doesn't make sense within,
for instance, an included configuration file itself (unless it in turn
performs its own merge include). That's because `!omit` only applies to the
file doing the include; it doesn't work in reverse or propagate through
includes.
### Shallow merge
@ -296,7 +352,7 @@ on the `retention` mapping:
location:
repositories:
- repo.borg
- path: repo.borg
retention: !retain
keep_daily: 5
@ -307,7 +363,7 @@ And `common.yaml` like this:
```yaml
location:
repositories:
- common.borg
- path: common.borg
retention:
keep_hourly: 24

285
tests/integration/config/test_load.py
View file

@ -211,7 +211,7 @@ def test_load_configuration_with_retain_tag_but_without_merge_include_raises():
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
with pytest.raises(ValueError):
assert module.load_configuration('config.yaml')
module.load_configuration('config.yaml')
def test_load_configuration_with_retain_tag_on_scalar_raises():
@ -239,7 +239,156 @@ def test_load_configuration_with_retain_tag_on_scalar_raises():
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
with pytest.raises(ValueError):
assert module.load_configuration('config.yaml')
module.load_configuration('config.yaml')
def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_values():
builtins = flexmock(sys.modules['builtins'])
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
flexmock(module.os.path).should_receive('isabs').and_return(False)
flexmock(module.os.path).should_receive('exists').and_return(True)
include_file = io.StringIO(
'''
stuff:
- a
- b
- c
'''
)
include_file.name = 'include.yaml'
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
config_file = io.StringIO(
'''
stuff:
- x
- !omit b
- y
<<: !include include.yaml
'''
)
config_file.name = 'config.yaml'
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
assert module.load_configuration('config.yaml') == {'stuff': ['a', 'c', 'x', 'y']}
def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_does_not_raise():
builtins = flexmock(sys.modules['builtins'])
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
flexmock(module.os.path).should_receive('isabs').and_return(False)
flexmock(module.os.path).should_receive('exists').and_return(True)
include_file = io.StringIO(
'''
stuff:
- a
- b
- c
'''
)
include_file.name = 'include.yaml'
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
config_file = io.StringIO(
'''
stuff:
- x
- !omit q
- y
<<: !include include.yaml
'''
)
config_file.name = 'config.yaml'
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
assert module.load_configuration('config.yaml') == {'stuff': ['a', 'b', 'c', 'x', 'y']}
def test_load_configuration_with_omit_tag_on_non_list_item_raises():
builtins = flexmock(sys.modules['builtins'])
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
flexmock(module.os.path).should_receive('isabs').and_return(False)
flexmock(module.os.path).should_receive('exists').and_return(True)
include_file = io.StringIO(
'''
stuff:
- a
- b
- c
'''
)
include_file.name = 'include.yaml'
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
config_file = io.StringIO(
'''
stuff: !omit
- x
- y
<<: !include include.yaml
'''
)
config_file.name = 'config.yaml'
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
with pytest.raises(ValueError):
module.load_configuration('config.yaml')
def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises():
builtins = flexmock(sys.modules['builtins'])
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
flexmock(module.os.path).should_receive('isabs').and_return(False)
flexmock(module.os.path).should_receive('exists').and_return(True)
include_file = io.StringIO(
'''
stuff:
- foo: bar
baz: quux
'''
)
include_file.name = 'include.yaml'
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
config_file = io.StringIO(
'''
stuff:
- !omit foo: bar
baz: quux
<<: !include include.yaml
'''
)
config_file.name = 'config.yaml'
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
with pytest.raises(ValueError):
module.load_configuration('config.yaml')
def test_load_configuration_with_omit_tag_but_without_merge_raises():
builtins = flexmock(sys.modules['builtins'])
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
flexmock(module.os.path).should_receive('isabs').and_return(False)
flexmock(module.os.path).should_receive('exists').and_return(True)
include_file = io.StringIO(
'''
stuff:
- a
- !omit b
- c
'''
)
include_file.name = 'include.yaml'
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
config_file = io.StringIO(
'''
stuff:
- x
- y
<<: !include include.yaml
'''
)
config_file.name = 'config.yaml'
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
with pytest.raises(ValueError):
module.load_configuration('config.yaml')
def test_load_configuration_does_not_merge_include_list():
@ -277,13 +426,33 @@ def test_load_configuration_does_not_merge_include_list():
module.ruamel.yaml.nodes.ScalarNode,
),
)
def test_retain_node_error_raises(node_class):
def test_raise_retain_node_error_raises(node_class):
with pytest.raises(ValueError):
module.retain_node_error(
module.raise_retain_node_error(
loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock())
)
def test_raise_omit_node_error_raises():
with pytest.raises(ValueError):
module.raise_omit_node_error(loader=flexmock(), node=flexmock())
def test_filter_omitted_nodes():
nodes = [
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'),
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
module.ruamel.yaml.nodes.ScalarNode(tag='!omit', value='b'),
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
]
result = module.filter_omitted_nodes(nodes)
assert [item.value for item in result] == ['a', 'c', 'a', 'c']
def test_deep_merge_nodes_replaces_colliding_scalar_values():
node_values = [
(
@ -483,7 +652,15 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
tag='tag:yaml.org,2002:str', value='before_backup'
),
module.ruamel.yaml.nodes.SequenceNode(
tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2']
tag='tag:yaml.org,2002:seq',
value=[
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 1'
),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 2'
),
],
),
),
],
@ -499,7 +676,15 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
tag='tag:yaml.org,2002:str', value='before_backup'
),
module.ruamel.yaml.nodes.SequenceNode(
tag='tag:yaml.org,2002:seq', value=['echo 3', 'echo 4']
tag='tag:yaml.org,2002:seq',
value=[
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 3'
),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 4'
),
],
),
),
],
@ -514,10 +699,10 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
options = section_value.value
assert len(options) == 1
assert options[0][0].value == 'before_backup'
assert options[0][1].value == ['echo 1', 'echo 2', 'echo 3', 'echo 4']
assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 2', 'echo 3', 'echo 4']
def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain():
def test_deep_merge_nodes_only_keeps_mapping_values_tagged_with_retain():
node_values = [
(
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
@ -568,7 +753,7 @@ def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain():
assert options[0][1].value == '5'
def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
def test_deep_merge_nodes_only_keeps_sequence_values_tagged_with_retain():
node_values = [
(
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
@ -580,7 +765,15 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
tag='tag:yaml.org,2002:str', value='before_backup'
),
module.ruamel.yaml.nodes.SequenceNode(
tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2']
tag='tag:yaml.org,2002:seq',
value=[
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 1'
),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 2'
),
],
),
),
],
@ -596,7 +789,15 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
tag='tag:yaml.org,2002:str', value='before_backup'
),
module.ruamel.yaml.nodes.SequenceNode(
tag='!retain', value=['echo 3', 'echo 4']
tag='!retain',
value=[
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 3'
),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 4'
),
],
),
),
],
@ -612,4 +813,64 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
assert len(options) == 1
assert options[0][0].value == 'before_backup'
assert options[0][1].tag == 'tag:yaml.org,2002:seq'
assert options[0][1].value == ['echo 3', 'echo 4']
assert [item.value for item in options[0][1].value] == ['echo 3', 'echo 4']
def test_deep_merge_nodes_skips_sequence_values_tagged_with_omit():
node_values = [
(
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
module.ruamel.yaml.nodes.MappingNode(
tag='tag:yaml.org,2002:map',
value=[
(
module.ruamel.yaml.nodes.ScalarNode(
tag='tag:yaml.org,2002:str', value='before_backup'
),
module.ruamel.yaml.nodes.SequenceNode(
tag='tag:yaml.org,2002:seq',
value=[
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 1'
),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 2'
),
],
),
),
],
),
),
(
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
module.ruamel.yaml.nodes.MappingNode(
tag='tag:yaml.org,2002:map',
value=[
(
module.ruamel.yaml.nodes.ScalarNode(
tag='tag:yaml.org,2002:str', value='before_backup'
),
module.ruamel.yaml.nodes.SequenceNode(
tag='tag:yaml.org,2002:seq',
value=[
module.ruamel.yaml.ScalarNode(tag='!omit', value='echo 2'),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 3'
),
],
),
),
],
),
),
]
result = module.deep_merge_nodes(node_values)
assert len(result) == 1
(section_key, section_value) = result[0]
assert section_key.value == 'hooks'
options = section_value.value
assert len(options) == 1
assert options[0][0].value == 'before_backup'
assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 3']