Override particular configuration options from the command-line via "--override" flag (#268).

This commit is contained in:
Dan Helfman 2019-12-17 11:46:27 -08:00
parent afaabd14a8
commit f787dfe809
10 changed files with 278 additions and 9 deletions

5
NEWS
View file

@ -1,3 +1,8 @@
1.4.21.dev0
* #268: Override particular configuration options from the command-line via "--override" flag. See
the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides
1.4.20 1.4.20
* Fix repository probing during "borgmatic init" to respect verbosity flag and remote_path option. * Fix repository probing during "borgmatic init" to respect verbosity flag and remote_path option.
* #249: Update Healthchecks/Cronitor/Cronhub monitoring integrations to fire for "check" and * #249: Update Healthchecks/Cronitor/Cronhub monitoring integrations to fire for "check" and

View file

@ -164,6 +164,13 @@ def parse_arguments(*unparsed_arguments):
default=None, default=None,
help='Write log messages to this file instead of syslog', help='Write log messages to this file instead of syslog',
) )
global_group.add_argument(
'--override',
metavar='SECTION.OPTION=VALUE',
nargs='+',
dest='overrides',
help='One or more configuration file options to override with specified values',
)
global_group.add_argument( global_group.add_argument(
'--version', '--version',
dest='version', dest='version',

View file

@ -372,7 +372,7 @@ def run_actions(
yield json.loads(json_output) yield json.loads(json_output)
def load_configurations(config_filenames): def load_configurations(config_filenames, overrides=None):
''' '''
Given a sequence of configuration filenames, load and validate each configuration file. Return Given a sequence of configuration filenames, load and validate each configuration file. Return
the results as a tuple of: dict of configuration filename to corresponding parsed configuration, the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
@ -386,7 +386,7 @@ def load_configurations(config_filenames):
for config_filename in config_filenames: for config_filename in config_filenames:
try: try:
configs[config_filename] = validate.parse_configuration( configs[config_filename] = validate.parse_configuration(
config_filename, validate.schema_filename() config_filename, validate.schema_filename(), overrides
) )
except (ValueError, OSError, validate.Validation_error) as error: except (ValueError, OSError, validate.Validation_error) as error:
logs.extend( logs.extend(
@ -584,7 +584,7 @@ def main(): # pragma: no cover
sys.exit(0) sys.exit(0)
config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
configs, parse_logs = load_configurations(config_filenames) configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides)
colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs)) colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
try: try:

View file

@ -0,0 +1,71 @@
import io
import ruamel.yaml
def set_values(config, keys, value):
'''
Given a hierarchy of configuration dicts, a sequence of parsed key strings, and a string value,
descend into the hierarchy based on the keys to set the value into the right place.
'''
if not keys:
return
first_key = keys[0]
if len(keys) == 1:
config[first_key] = value
return
if first_key not in config:
config[first_key] = {}
set_values(config[first_key], keys[1:], value)
def convert_value_type(value):
'''
Given a string value, determine its logical type (string, boolean, integer, etc.), and return it
converted to that type.
'''
return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
def parse_overrides(raw_overrides):
'''
Given a sequence of configuration file override strings in the form of "section.option=value",
parse and return a sequence of tuples (keys, values), where keys is a sequence of strings. For
instance, given the following raw overrides:
['section.my_option=value1', 'section.other_option=value2']
... return this:
(
(('section', 'my_option'), 'value1'),
(('section', 'other_option'), 'value2'),
)
Raise ValueError if an override can't be parsed.
'''
if not raw_overrides:
return ()
try:
return tuple(
(tuple(raw_keys.split('.')), convert_value_type(value))
for raw_override in raw_overrides
for raw_keys, value in (raw_override.split('=', 1),)
)
except ValueError:
raise ValueError('Invalid override. Make sure you use the form: SECTION.OPTION=VALUE')
def apply_overrides(config, raw_overrides):
'''
Given a sequence of configuration file override strings in the form of "section.option=value"
and a configuration dict, parse each override and set it the configuration dict.
'''
overrides = parse_overrides(raw_overrides)
for (keys, value) in overrides:
set_values(config, keys, value)

View file

@ -6,7 +6,7 @@ import pykwalify.core
import pykwalify.errors import pykwalify.errors
import ruamel.yaml import ruamel.yaml
from borgmatic.config import load from borgmatic.config import load, override
def schema_filename(): def schema_filename():
@ -82,11 +82,12 @@ def remove_examples(schema):
return schema return schema
def parse_configuration(config_filename, schema_filename): def parse_configuration(config_filename, schema_filename, overrides=None):
''' '''
Given the path to a config filename in YAML format and the path to a schema filename in Given the path to a config filename in YAML format, the path to a schema filename in pykwalify
pykwalify YAML schema format, return the parsed configuration as a data structure of nested YAML schema format, a sequence of configuration file override strings in the form of
dicts and lists corresponding to the schema. Example return value: "section.option=value", return the parsed configuration as a data structure of nested dicts and
lists corresponding to the schema. Example return value:
{'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'}, {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}} 'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
@ -102,6 +103,8 @@ def parse_configuration(config_filename, schema_filename):
except (ruamel.yaml.error.YAMLError, RecursionError) as error: except (ruamel.yaml.error.YAMLError, RecursionError) as error:
raise Validation_error(config_filename, (str(error),)) raise Validation_error(config_filename, (str(error),))
override.apply_overrides(config, overrides)
validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema)) validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
parsed_result = validator.validate(raise_exception=False) parsed_result = validator.validate(raise_exception=False)

View file

@ -115,6 +115,40 @@ Note that this `<<` include merging syntax is only for merging in mappings
directly, please see the section above about standard includes. directly, please see the section above about standard includes.
## Configuration overrides
In more complex multi-application setups, you may want to override particular
borgmatic configuration file options at the time you run borgmatic. For
instance, you could reuse a common configuration file for multiple
applications, but then set the repository for each application at runtime. Or
you might want to try a variant of an option for testing purposes without
actually touching your configuration file.
Whatever the reason, you can override borgmatic configuration options at the
command-line via the `--override` flag. Here's an example:
```bash
borgmatic create --override location.remote_path=borg1
```
What this does is load your configuration files, and for each one, disregard
the configured value for the `remote_path` option in the `location` section,
and use the value of `borg1` instead.
Note that the value is parsed as an actual YAML string, so you can even set
list values by using brackets. For instance:
```bash
borgmatic create --override location.repositories=[test1.borg,test2.borg]
```
There is not currently a way to override a single element of a list without
replacing the whole list.
Be sure to quote your overrides if they contain spaces or other characters
that your shell may interpret.
## Related documentation ## Related documentation
* [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/) * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)

View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup from setuptools import find_packages, setup
VERSION = '1.4.20' VERSION = '1.4.21.dev0'
setup( setup(

View file

@ -0,0 +1,40 @@
import pytest
from borgmatic.config import override as module
@pytest.mark.parametrize(
'value,expected_result',
(
('thing', 'thing'),
('33', 33),
('33b', '33b'),
('true', True),
('false', False),
('[foo]', ['foo']),
('[foo, bar]', ['foo', 'bar']),
),
)
def test_convert_value_type_coerces_values(value, expected_result):
assert module.convert_value_type(value) == expected_result
def test_apply_overrides_updates_config():
raw_overrides = [
'section.key=value1',
'other_section.thing=value2',
'section.nested.key=value3',
'new.foo=bar',
]
config = {
'section': {'key': 'value', 'other': 'other_value'},
'other_section': {'thing': 'thing_value'},
}
module.apply_overrides(config, raw_overrides)
assert config == {
'section': {'key': 'value1', 'other': 'other_value', 'nested': {'key': 'value3'}},
'other_section': {'thing': 'value2'},
'new': {'foo': 'bar'},
}

View file

@ -212,3 +212,30 @@ def test_parse_configuration_raises_for_validation_error():
with pytest.raises(module.Validation_error): with pytest.raises(module.Validation_error):
module.parse_configuration('config.yaml', 'schema.yaml') module.parse_configuration('config.yaml', 'schema.yaml')
def test_parse_configuration_applies_overrides():
mock_config_and_schema(
'''
location:
source_directories:
- /home
repositories:
- hostname.borg
local_path: borg1
'''
)
result = module.parse_configuration(
'config.yaml', 'schema.yaml', overrides=['location.local_path=borg2']
)
assert result == {
'location': {
'source_directories': ['/home'],
'repositories': ['hostname.borg'],
'local_path': 'borg2',
}
}

View file

@ -0,0 +1,82 @@
import pytest
from flexmock import flexmock
from borgmatic.config import override as module
def test_set_values_with_empty_keys_bails():
config = {}
module.set_values(config, keys=(), value='value')
assert config == {}
def test_set_values_with_one_key_sets_it_into_config():
config = {}
module.set_values(config, keys=('key',), value='value')
assert config == {'key': 'value'}
def test_set_values_with_one_key_overwrites_existing_key():
config = {'key': 'old_value', 'other': 'other_value'}
module.set_values(config, keys=('key',), value='value')
assert config == {'key': 'value', 'other': 'other_value'}
def test_set_values_with_multiple_keys_creates_hierarchy():
config = {}
module.set_values(config, ('section', 'key'), 'value')
assert config == {'section': {'key': 'value'}}
def test_set_values_with_multiple_keys_updates_hierarchy():
config = {'section': {'other': 'other_value'}}
module.set_values(config, ('section', 'key'), 'value')
assert config == {'section': {'key': 'value', 'other': 'other_value'}}
def test_parse_overrides_splits_keys_and_values():
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
raw_overrides = ['section.my_option=value1', 'section.other_option=value2']
expected_result = (
(('section', 'my_option'), 'value1'),
(('section', 'other_option'), 'value2'),
)
module.parse_overrides(raw_overrides) == expected_result
def test_parse_overrides_allows_value_with_equal_sign():
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
raw_overrides = ['section.option=this===value']
expected_result = ((('section', 'option'), 'this===value'),)
module.parse_overrides(raw_overrides) == expected_result
def test_parse_overrides_raises_on_missing_equal_sign():
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
raw_overrides = ['section.option']
with pytest.raises(ValueError):
module.parse_overrides(raw_overrides)
def test_parse_overrides_allows_value_with_single_key():
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
raw_overrides = ['option=value']
expected_result = ((('option',), 'value'),)
module.parse_overrides(raw_overrides) == expected_result
def test_parse_overrides_handles_empty_overrides():
module.parse_overrides(raw_overrides=None) == ()