Selectively omit list values when including configuration files (#672).

This commit is contained in:
Dan Helfman 2023-04-13 14:39:36 -07:00
parent 08843d51d9
commit 4a94c2c9bf
4 changed files with 362 additions and 18 deletions

3
NEWS
View file

@ -10,6 +10,9 @@
* #672: Selectively shallow merge certain mappings or sequences when including configuration files. * #672: Selectively shallow merge certain mappings or sequences when including configuration files.
See the documentation for more information: See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge
* #672: Selectively omit list values when including configuration files. See the documentation for
more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#list-merge
* #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag. * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag.
See the documentation for more information: See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes

View file

@ -38,9 +38,9 @@ def include_configuration(loader, filename_node, include_directory):
return load_configuration(include_filename) return load_configuration(include_filename)
def retain_node_error(loader, node): def raise_retain_node_error(loader, node):
''' '''
Given a ruamel.yaml.loader.Loader and a YAML node, raise an error. Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!retain" usage.
Raise ValueError if a mapping or sequence node is given, as that indicates that "!retain" was Raise ValueError if a mapping or sequence node is given, as that indicates that "!retain" was
used in a configuration file without a merge. In configuration files with a merge, mapping and used in a configuration file without a merge. In configuration files with a merge, mapping and
@ -56,6 +56,19 @@ def retain_node_error(loader, node):
raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.') raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.')
def raise_omit_node_error(loader, node):
'''
Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!omit" usage.
Raise ValueError unconditionally, as an "!omit" node here indicates it was used in a
configuration file without a merge. In configuration files with a merge, nodes with "!omit"
tags are handled by deep_merge_nodes() below.
'''
raise ValueError(
'The !omit tag may only be used on a scalar (e.g., string) list element within a configuration file containing a merged !include tag.'
)
class Include_constructor(ruamel.yaml.SafeConstructor): class Include_constructor(ruamel.yaml.SafeConstructor):
''' '''
A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
@ -68,7 +81,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
'!include', '!include',
functools.partial(include_configuration, include_directory=include_directory), functools.partial(include_configuration, include_directory=include_directory),
) )
self.add_constructor('!retain', retain_node_error) self.add_constructor('!retain', raise_retain_node_error)
self.add_constructor('!omit', raise_omit_node_error)
def flatten_mapping(self, node): def flatten_mapping(self, node):
''' '''
@ -134,6 +148,16 @@ def load_configuration(filename):
return config return config
def filter_omitted_nodes(nodes):
'''
Given a list of nodes, return a filtered list omitting any nodes with an "!omit" tag or with a
value matching such nodes.
'''
omitted_values = tuple(node.value for node in nodes if node.tag == '!omit')
return [node for node in nodes if node.value not in omitted_values]
DELETED_NODE = object() DELETED_NODE = object()
@ -247,7 +271,7 @@ def deep_merge_nodes(nodes):
b_key, b_key,
ruamel.yaml.nodes.SequenceNode( ruamel.yaml.nodes.SequenceNode(
tag=b_value.tag, tag=b_value.tag,
value=a_value.value + b_value.value, value=filter_omitted_nodes(a_value.value + b_value.value),
start_mark=b_value.start_mark, start_mark=b_value.start_mark,
end_mark=b_value.end_mark, end_mark=b_value.end_mark,
flow_style=b_value.flow_style, flow_style=b_value.flow_style,

View file

@ -272,9 +272,65 @@ Once this include gets merged in, the resulting configuration would have a
When there's an option collision between the local file and the merged When there's an option collision between the local file and the merged
include, the local file's option takes precedence. include, the local file's option takes precedence.
#### List merge
<span class="minilink minilink-addedin">New in version 1.6.1</span> Colliding <span class="minilink minilink-addedin">New in version 1.6.1</span> Colliding
list values are appended together. list values are appended together.
<span class="minilink minilink-addedin">New in version 1.7.12</span> If there
is a list value from an include that you *don't* want in your local
configuration file, you can omit it with an `!omit` tag. For instance:
```yaml
<<: !include /etc/borgmatic/common.yaml
location:
source_directories:
- !omit /home
- /var
```
And `common.yaml` like this:
```yaml
location:
source_directories:
- /home
- /etc
```
Once this include gets merged in, the resulting configuration will have a
`source_directories` value of `/etc` and `/var`—with `/home` omitted.
This feature currently only works on scalar (e.g. string or number) list items
and will not work elsewhere in a configuration file. Be sure to put the
`!omit` tag *before* the list item (after the dash). Putting `!omit` after the
list item will not work, as it gets interpreted as part of the string. Here's
an example of some things not to do:
```yaml
<<: !include /etc/borgmatic/common.yaml
location:
source_directories:
# Do not do this! It will not work. "!omit" belongs before "/home".
- /home !omit
# Do not do this either! "!omit" only works on scalar list items.
repositories: !omit
# Also do not do this for the same reason! This is a list item, but it's
# not a scalar.
- !omit path: repo.borg
```
Additionally, the `!omit` tag only works in a configuration file that also
performs a merge include with `<<: !include`. It doesn't make sense within,
for instance, an included configuration file itself (unless it in turn
performs its own merge include). That's because `!omit` only applies to the
file doing the include; it doesn't work in reverse or propagate through
includes.
### Shallow merge ### Shallow merge
@ -296,7 +352,7 @@ on the `retention` mapping:
location: location:
repositories: repositories:
- repo.borg - path: repo.borg
retention: !retain retention: !retain
keep_daily: 5 keep_daily: 5
@ -307,7 +363,7 @@ And `common.yaml` like this:
```yaml ```yaml
location: location:
repositories: repositories:
- common.borg - path: common.borg
retention: retention:
keep_hourly: 24 keep_hourly: 24

View file

@ -211,7 +211,7 @@ def test_load_configuration_with_retain_tag_but_without_merge_include_raises():
builtins.should_receive('open').with_args('config.yaml').and_return(config_file) builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
with pytest.raises(ValueError): with pytest.raises(ValueError):
assert module.load_configuration('config.yaml') module.load_configuration('config.yaml')
def test_load_configuration_with_retain_tag_on_scalar_raises(): def test_load_configuration_with_retain_tag_on_scalar_raises():
@ -239,7 +239,156 @@ def test_load_configuration_with_retain_tag_on_scalar_raises():
builtins.should_receive('open').with_args('config.yaml').and_return(config_file) builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
with pytest.raises(ValueError): with pytest.raises(ValueError):
assert module.load_configuration('config.yaml') module.load_configuration('config.yaml')
def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_values():
builtins = flexmock(sys.modules['builtins'])
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
flexmock(module.os.path).should_receive('isabs').and_return(False)
flexmock(module.os.path).should_receive('exists').and_return(True)
include_file = io.StringIO(
'''
stuff:
- a
- b
- c
'''
)
include_file.name = 'include.yaml'
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
config_file = io.StringIO(
'''
stuff:
- x
- !omit b
- y
<<: !include include.yaml
'''
)
config_file.name = 'config.yaml'
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
assert module.load_configuration('config.yaml') == {'stuff': ['a', 'c', 'x', 'y']}
def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_does_not_raise():
builtins = flexmock(sys.modules['builtins'])
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
flexmock(module.os.path).should_receive('isabs').and_return(False)
flexmock(module.os.path).should_receive('exists').and_return(True)
include_file = io.StringIO(
'''
stuff:
- a
- b
- c
'''
)
include_file.name = 'include.yaml'
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
config_file = io.StringIO(
'''
stuff:
- x
- !omit q
- y
<<: !include include.yaml
'''
)
config_file.name = 'config.yaml'
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
assert module.load_configuration('config.yaml') == {'stuff': ['a', 'b', 'c', 'x', 'y']}
def test_load_configuration_with_omit_tag_on_non_list_item_raises():
builtins = flexmock(sys.modules['builtins'])
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
flexmock(module.os.path).should_receive('isabs').and_return(False)
flexmock(module.os.path).should_receive('exists').and_return(True)
include_file = io.StringIO(
'''
stuff:
- a
- b
- c
'''
)
include_file.name = 'include.yaml'
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
config_file = io.StringIO(
'''
stuff: !omit
- x
- y
<<: !include include.yaml
'''
)
config_file.name = 'config.yaml'
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
with pytest.raises(ValueError):
module.load_configuration('config.yaml')
def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises():
builtins = flexmock(sys.modules['builtins'])
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
flexmock(module.os.path).should_receive('isabs').and_return(False)
flexmock(module.os.path).should_receive('exists').and_return(True)
include_file = io.StringIO(
'''
stuff:
- foo: bar
baz: quux
'''
)
include_file.name = 'include.yaml'
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
config_file = io.StringIO(
'''
stuff:
- !omit foo: bar
baz: quux
<<: !include include.yaml
'''
)
config_file.name = 'config.yaml'
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
with pytest.raises(ValueError):
module.load_configuration('config.yaml')
def test_load_configuration_with_omit_tag_but_without_merge_raises():
builtins = flexmock(sys.modules['builtins'])
flexmock(module.os).should_receive('getcwd').and_return('/tmp')
flexmock(module.os.path).should_receive('isabs').and_return(False)
flexmock(module.os.path).should_receive('exists').and_return(True)
include_file = io.StringIO(
'''
stuff:
- a
- !omit b
- c
'''
)
include_file.name = 'include.yaml'
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
config_file = io.StringIO(
'''
stuff:
- x
- y
<<: !include include.yaml
'''
)
config_file.name = 'config.yaml'
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
with pytest.raises(ValueError):
module.load_configuration('config.yaml')
def test_load_configuration_does_not_merge_include_list(): def test_load_configuration_does_not_merge_include_list():
@ -277,13 +426,33 @@ def test_load_configuration_does_not_merge_include_list():
module.ruamel.yaml.nodes.ScalarNode, module.ruamel.yaml.nodes.ScalarNode,
), ),
) )
def test_retain_node_error_raises(node_class): def test_raise_retain_node_error_raises(node_class):
with pytest.raises(ValueError): with pytest.raises(ValueError):
module.retain_node_error( module.raise_retain_node_error(
loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock()) loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock())
) )
def test_raise_omit_node_error_raises():
with pytest.raises(ValueError):
module.raise_omit_node_error(loader=flexmock(), node=flexmock())
def test_filter_omitted_nodes():
nodes = [
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'),
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
module.ruamel.yaml.nodes.ScalarNode(tag='!omit', value='b'),
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
]
result = module.filter_omitted_nodes(nodes)
assert [item.value for item in result] == ['a', 'c', 'a', 'c']
def test_deep_merge_nodes_replaces_colliding_scalar_values(): def test_deep_merge_nodes_replaces_colliding_scalar_values():
node_values = [ node_values = [
( (
@ -483,7 +652,15 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
tag='tag:yaml.org,2002:str', value='before_backup' tag='tag:yaml.org,2002:str', value='before_backup'
), ),
module.ruamel.yaml.nodes.SequenceNode( module.ruamel.yaml.nodes.SequenceNode(
tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2'] tag='tag:yaml.org,2002:seq',
value=[
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 1'
),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 2'
),
],
), ),
), ),
], ],
@ -499,7 +676,15 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
tag='tag:yaml.org,2002:str', value='before_backup' tag='tag:yaml.org,2002:str', value='before_backup'
), ),
module.ruamel.yaml.nodes.SequenceNode( module.ruamel.yaml.nodes.SequenceNode(
tag='tag:yaml.org,2002:seq', value=['echo 3', 'echo 4'] tag='tag:yaml.org,2002:seq',
value=[
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 3'
),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 4'
),
],
), ),
), ),
], ],
@ -514,10 +699,10 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
options = section_value.value options = section_value.value
assert len(options) == 1 assert len(options) == 1
assert options[0][0].value == 'before_backup' assert options[0][0].value == 'before_backup'
assert options[0][1].value == ['echo 1', 'echo 2', 'echo 3', 'echo 4'] assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 2', 'echo 3', 'echo 4']
def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain(): def test_deep_merge_nodes_only_keeps_mapping_values_tagged_with_retain():
node_values = [ node_values = [
( (
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
@ -568,7 +753,7 @@ def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain():
assert options[0][1].value == '5' assert options[0][1].value == '5'
def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain(): def test_deep_merge_nodes_only_keeps_sequence_values_tagged_with_retain():
node_values = [ node_values = [
( (
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
@ -580,7 +765,15 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
tag='tag:yaml.org,2002:str', value='before_backup' tag='tag:yaml.org,2002:str', value='before_backup'
), ),
module.ruamel.yaml.nodes.SequenceNode( module.ruamel.yaml.nodes.SequenceNode(
tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2'] tag='tag:yaml.org,2002:seq',
value=[
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 1'
),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 2'
),
],
), ),
), ),
], ],
@ -596,7 +789,15 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
tag='tag:yaml.org,2002:str', value='before_backup' tag='tag:yaml.org,2002:str', value='before_backup'
), ),
module.ruamel.yaml.nodes.SequenceNode( module.ruamel.yaml.nodes.SequenceNode(
tag='!retain', value=['echo 3', 'echo 4'] tag='!retain',
value=[
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 3'
),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 4'
),
],
), ),
), ),
], ],
@ -612,4 +813,64 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain():
assert len(options) == 1 assert len(options) == 1
assert options[0][0].value == 'before_backup' assert options[0][0].value == 'before_backup'
assert options[0][1].tag == 'tag:yaml.org,2002:seq' assert options[0][1].tag == 'tag:yaml.org,2002:seq'
assert options[0][1].value == ['echo 3', 'echo 4'] assert [item.value for item in options[0][1].value] == ['echo 3', 'echo 4']
def test_deep_merge_nodes_skips_sequence_values_tagged_with_omit():
node_values = [
(
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
module.ruamel.yaml.nodes.MappingNode(
tag='tag:yaml.org,2002:map',
value=[
(
module.ruamel.yaml.nodes.ScalarNode(
tag='tag:yaml.org,2002:str', value='before_backup'
),
module.ruamel.yaml.nodes.SequenceNode(
tag='tag:yaml.org,2002:seq',
value=[
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 1'
),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 2'
),
],
),
),
],
),
),
(
module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
module.ruamel.yaml.nodes.MappingNode(
tag='tag:yaml.org,2002:map',
value=[
(
module.ruamel.yaml.nodes.ScalarNode(
tag='tag:yaml.org,2002:str', value='before_backup'
),
module.ruamel.yaml.nodes.SequenceNode(
tag='tag:yaml.org,2002:seq',
value=[
module.ruamel.yaml.ScalarNode(tag='!omit', value='echo 2'),
module.ruamel.yaml.ScalarNode(
tag='tag:yaml.org,2002:str', value='echo 3'
),
],
),
),
],
),
),
]
result = module.deep_merge_nodes(node_values)
assert len(result) == 1
(section_key, section_value) = result[0]
assert section_key.value == 'hooks'
options = section_value.value
assert len(options) == 1
assert options[0][0].value == 'before_backup'
assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 3']