Override particular configuration options from the command-line via "--override" flag (#268).
This commit is contained in:
parent
afaabd14a8
commit
f787dfe809
10 changed files with 278 additions and 9 deletions
5
NEWS
5
NEWS
|
@ -1,3 +1,8 @@
|
||||||
|
1.4.21.dev0
|
||||||
|
* #268: Override particular configuration options from the command-line via "--override" flag. See
|
||||||
|
the documentation for more information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides
|
||||||
|
|
||||||
1.4.20
|
1.4.20
|
||||||
* Fix repository probing during "borgmatic init" to respect verbosity flag and remote_path option.
|
* Fix repository probing during "borgmatic init" to respect verbosity flag and remote_path option.
|
||||||
* #249: Update Healthchecks/Cronitor/Cronhub monitoring integrations to fire for "check" and
|
* #249: Update Healthchecks/Cronitor/Cronhub monitoring integrations to fire for "check" and
|
||||||
|
|
|
@ -164,6 +164,13 @@ def parse_arguments(*unparsed_arguments):
|
||||||
default=None,
|
default=None,
|
||||||
help='Write log messages to this file instead of syslog',
|
help='Write log messages to this file instead of syslog',
|
||||||
)
|
)
|
||||||
|
global_group.add_argument(
|
||||||
|
'--override',
|
||||||
|
metavar='SECTION.OPTION=VALUE',
|
||||||
|
nargs='+',
|
||||||
|
dest='overrides',
|
||||||
|
help='One or more configuration file options to override with specified values',
|
||||||
|
)
|
||||||
global_group.add_argument(
|
global_group.add_argument(
|
||||||
'--version',
|
'--version',
|
||||||
dest='version',
|
dest='version',
|
||||||
|
|
|
@ -372,7 +372,7 @@ def run_actions(
|
||||||
yield json.loads(json_output)
|
yield json.loads(json_output)
|
||||||
|
|
||||||
|
|
||||||
def load_configurations(config_filenames):
|
def load_configurations(config_filenames, overrides=None):
|
||||||
'''
|
'''
|
||||||
Given a sequence of configuration filenames, load and validate each configuration file. Return
|
Given a sequence of configuration filenames, load and validate each configuration file. Return
|
||||||
the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
|
the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
|
||||||
|
@ -386,7 +386,7 @@ def load_configurations(config_filenames):
|
||||||
for config_filename in config_filenames:
|
for config_filename in config_filenames:
|
||||||
try:
|
try:
|
||||||
configs[config_filename] = validate.parse_configuration(
|
configs[config_filename] = validate.parse_configuration(
|
||||||
config_filename, validate.schema_filename()
|
config_filename, validate.schema_filename(), overrides
|
||||||
)
|
)
|
||||||
except (ValueError, OSError, validate.Validation_error) as error:
|
except (ValueError, OSError, validate.Validation_error) as error:
|
||||||
logs.extend(
|
logs.extend(
|
||||||
|
@ -584,7 +584,7 @@ def main(): # pragma: no cover
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
|
config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
|
||||||
configs, parse_logs = load_configurations(config_filenames)
|
configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides)
|
||||||
|
|
||||||
colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
|
colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
|
||||||
try:
|
try:
|
||||||
|
|
71
borgmatic/config/override.py
Normal file
71
borgmatic/config/override.py
Normal file
|
@ -0,0 +1,71 @@
|
||||||
|
import io
|
||||||
|
|
||||||
|
import ruamel.yaml
|
||||||
|
|
||||||
|
|
||||||
|
def set_values(config, keys, value):
|
||||||
|
'''
|
||||||
|
Given a hierarchy of configuration dicts, a sequence of parsed key strings, and a string value,
|
||||||
|
descend into the hierarchy based on the keys to set the value into the right place.
|
||||||
|
'''
|
||||||
|
if not keys:
|
||||||
|
return
|
||||||
|
|
||||||
|
first_key = keys[0]
|
||||||
|
if len(keys) == 1:
|
||||||
|
config[first_key] = value
|
||||||
|
return
|
||||||
|
|
||||||
|
if first_key not in config:
|
||||||
|
config[first_key] = {}
|
||||||
|
|
||||||
|
set_values(config[first_key], keys[1:], value)
|
||||||
|
|
||||||
|
|
||||||
|
def convert_value_type(value):
|
||||||
|
'''
|
||||||
|
Given a string value, determine its logical type (string, boolean, integer, etc.), and return it
|
||||||
|
converted to that type.
|
||||||
|
'''
|
||||||
|
return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
|
||||||
|
|
||||||
|
|
||||||
|
def parse_overrides(raw_overrides):
|
||||||
|
'''
|
||||||
|
Given a sequence of configuration file override strings in the form of "section.option=value",
|
||||||
|
parse and return a sequence of tuples (keys, values), where keys is a sequence of strings. For
|
||||||
|
instance, given the following raw overrides:
|
||||||
|
|
||||||
|
['section.my_option=value1', 'section.other_option=value2']
|
||||||
|
|
||||||
|
... return this:
|
||||||
|
|
||||||
|
(
|
||||||
|
(('section', 'my_option'), 'value1'),
|
||||||
|
(('section', 'other_option'), 'value2'),
|
||||||
|
)
|
||||||
|
|
||||||
|
Raise ValueError if an override can't be parsed.
|
||||||
|
'''
|
||||||
|
if not raw_overrides:
|
||||||
|
return ()
|
||||||
|
|
||||||
|
try:
|
||||||
|
return tuple(
|
||||||
|
(tuple(raw_keys.split('.')), convert_value_type(value))
|
||||||
|
for raw_override in raw_overrides
|
||||||
|
for raw_keys, value in (raw_override.split('=', 1),)
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError('Invalid override. Make sure you use the form: SECTION.OPTION=VALUE')
|
||||||
|
|
||||||
|
|
||||||
|
def apply_overrides(config, raw_overrides):
|
||||||
|
'''
|
||||||
|
Given a sequence of configuration file override strings in the form of "section.option=value"
|
||||||
|
and a configuration dict, parse each override and set it the configuration dict.
|
||||||
|
'''
|
||||||
|
overrides = parse_overrides(raw_overrides)
|
||||||
|
|
||||||
|
for (keys, value) in overrides:
|
||||||
|
set_values(config, keys, value)
|
|
@ -6,7 +6,7 @@ import pykwalify.core
|
||||||
import pykwalify.errors
|
import pykwalify.errors
|
||||||
import ruamel.yaml
|
import ruamel.yaml
|
||||||
|
|
||||||
from borgmatic.config import load
|
from borgmatic.config import load, override
|
||||||
|
|
||||||
|
|
||||||
def schema_filename():
|
def schema_filename():
|
||||||
|
@ -82,11 +82,12 @@ def remove_examples(schema):
|
||||||
return schema
|
return schema
|
||||||
|
|
||||||
|
|
||||||
def parse_configuration(config_filename, schema_filename):
|
def parse_configuration(config_filename, schema_filename, overrides=None):
|
||||||
'''
|
'''
|
||||||
Given the path to a config filename in YAML format and the path to a schema filename in
|
Given the path to a config filename in YAML format, the path to a schema filename in pykwalify
|
||||||
pykwalify YAML schema format, return the parsed configuration as a data structure of nested
|
YAML schema format, a sequence of configuration file override strings in the form of
|
||||||
dicts and lists corresponding to the schema. Example return value:
|
"section.option=value", return the parsed configuration as a data structure of nested dicts and
|
||||||
|
lists corresponding to the schema. Example return value:
|
||||||
|
|
||||||
{'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
|
{'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
|
||||||
'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
|
'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
|
||||||
|
@ -102,6 +103,8 @@ def parse_configuration(config_filename, schema_filename):
|
||||||
except (ruamel.yaml.error.YAMLError, RecursionError) as error:
|
except (ruamel.yaml.error.YAMLError, RecursionError) as error:
|
||||||
raise Validation_error(config_filename, (str(error),))
|
raise Validation_error(config_filename, (str(error),))
|
||||||
|
|
||||||
|
override.apply_overrides(config, overrides)
|
||||||
|
|
||||||
validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
|
validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
|
||||||
parsed_result = validator.validate(raise_exception=False)
|
parsed_result = validator.validate(raise_exception=False)
|
||||||
|
|
||||||
|
|
|
@ -115,6 +115,40 @@ Note that this `<<` include merging syntax is only for merging in mappings
|
||||||
directly, please see the section above about standard includes.
|
directly, please see the section above about standard includes.
|
||||||
|
|
||||||
|
|
||||||
|
## Configuration overrides
|
||||||
|
|
||||||
|
In more complex multi-application setups, you may want to override particular
|
||||||
|
borgmatic configuration file options at the time you run borgmatic. For
|
||||||
|
instance, you could reuse a common configuration file for multiple
|
||||||
|
applications, but then set the repository for each application at runtime. Or
|
||||||
|
you might want to try a variant of an option for testing purposes without
|
||||||
|
actually touching your configuration file.
|
||||||
|
|
||||||
|
Whatever the reason, you can override borgmatic configuration options at the
|
||||||
|
command-line via the `--override` flag. Here's an example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
borgmatic create --override location.remote_path=borg1
|
||||||
|
```
|
||||||
|
|
||||||
|
What this does is load your configuration files, and for each one, disregard
|
||||||
|
the configured value for the `remote_path` option in the `location` section,
|
||||||
|
and use the value of `borg1` instead.
|
||||||
|
|
||||||
|
Note that the value is parsed as an actual YAML string, so you can even set
|
||||||
|
list values by using brackets. For instance:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
borgmatic create --override location.repositories=[test1.borg,test2.borg]
|
||||||
|
```
|
||||||
|
|
||||||
|
There is not currently a way to override a single element of a list without
|
||||||
|
replacing the whole list.
|
||||||
|
|
||||||
|
Be sure to quote your overrides if they contain spaces or other characters
|
||||||
|
that your shell may interpret.
|
||||||
|
|
||||||
|
|
||||||
## Related documentation
|
## Related documentation
|
||||||
|
|
||||||
* [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
|
* [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -1,6 +1,6 @@
|
||||||
from setuptools import find_packages, setup
|
from setuptools import find_packages, setup
|
||||||
|
|
||||||
VERSION = '1.4.20'
|
VERSION = '1.4.21.dev0'
|
||||||
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
|
|
40
tests/integration/config/test_override.py
Normal file
40
tests/integration/config/test_override.py
Normal file
|
@ -0,0 +1,40 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from borgmatic.config import override as module
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'value,expected_result',
|
||||||
|
(
|
||||||
|
('thing', 'thing'),
|
||||||
|
('33', 33),
|
||||||
|
('33b', '33b'),
|
||||||
|
('true', True),
|
||||||
|
('false', False),
|
||||||
|
('[foo]', ['foo']),
|
||||||
|
('[foo, bar]', ['foo', 'bar']),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
def test_convert_value_type_coerces_values(value, expected_result):
|
||||||
|
assert module.convert_value_type(value) == expected_result
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_overrides_updates_config():
|
||||||
|
raw_overrides = [
|
||||||
|
'section.key=value1',
|
||||||
|
'other_section.thing=value2',
|
||||||
|
'section.nested.key=value3',
|
||||||
|
'new.foo=bar',
|
||||||
|
]
|
||||||
|
config = {
|
||||||
|
'section': {'key': 'value', 'other': 'other_value'},
|
||||||
|
'other_section': {'thing': 'thing_value'},
|
||||||
|
}
|
||||||
|
|
||||||
|
module.apply_overrides(config, raw_overrides)
|
||||||
|
|
||||||
|
assert config == {
|
||||||
|
'section': {'key': 'value1', 'other': 'other_value', 'nested': {'key': 'value3'}},
|
||||||
|
'other_section': {'thing': 'value2'},
|
||||||
|
'new': {'foo': 'bar'},
|
||||||
|
}
|
|
@ -212,3 +212,30 @@ def test_parse_configuration_raises_for_validation_error():
|
||||||
|
|
||||||
with pytest.raises(module.Validation_error):
|
with pytest.raises(module.Validation_error):
|
||||||
module.parse_configuration('config.yaml', 'schema.yaml')
|
module.parse_configuration('config.yaml', 'schema.yaml')
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_configuration_applies_overrides():
|
||||||
|
mock_config_and_schema(
|
||||||
|
'''
|
||||||
|
location:
|
||||||
|
source_directories:
|
||||||
|
- /home
|
||||||
|
|
||||||
|
repositories:
|
||||||
|
- hostname.borg
|
||||||
|
|
||||||
|
local_path: borg1
|
||||||
|
'''
|
||||||
|
)
|
||||||
|
|
||||||
|
result = module.parse_configuration(
|
||||||
|
'config.yaml', 'schema.yaml', overrides=['location.local_path=borg2']
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == {
|
||||||
|
'location': {
|
||||||
|
'source_directories': ['/home'],
|
||||||
|
'repositories': ['hostname.borg'],
|
||||||
|
'local_path': 'borg2',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
82
tests/unit/config/test_override.py
Normal file
82
tests/unit/config/test_override.py
Normal file
|
@ -0,0 +1,82 @@
|
||||||
|
import pytest
|
||||||
|
from flexmock import flexmock
|
||||||
|
|
||||||
|
from borgmatic.config import override as module
|
||||||
|
|
||||||
|
|
||||||
|
def test_set_values_with_empty_keys_bails():
|
||||||
|
config = {}
|
||||||
|
|
||||||
|
module.set_values(config, keys=(), value='value')
|
||||||
|
|
||||||
|
assert config == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_set_values_with_one_key_sets_it_into_config():
|
||||||
|
config = {}
|
||||||
|
|
||||||
|
module.set_values(config, keys=('key',), value='value')
|
||||||
|
|
||||||
|
assert config == {'key': 'value'}
|
||||||
|
|
||||||
|
|
||||||
|
def test_set_values_with_one_key_overwrites_existing_key():
|
||||||
|
config = {'key': 'old_value', 'other': 'other_value'}
|
||||||
|
|
||||||
|
module.set_values(config, keys=('key',), value='value')
|
||||||
|
|
||||||
|
assert config == {'key': 'value', 'other': 'other_value'}
|
||||||
|
|
||||||
|
|
||||||
|
def test_set_values_with_multiple_keys_creates_hierarchy():
|
||||||
|
config = {}
|
||||||
|
|
||||||
|
module.set_values(config, ('section', 'key'), 'value')
|
||||||
|
|
||||||
|
assert config == {'section': {'key': 'value'}}
|
||||||
|
|
||||||
|
|
||||||
|
def test_set_values_with_multiple_keys_updates_hierarchy():
|
||||||
|
config = {'section': {'other': 'other_value'}}
|
||||||
|
module.set_values(config, ('section', 'key'), 'value')
|
||||||
|
|
||||||
|
assert config == {'section': {'key': 'value', 'other': 'other_value'}}
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_overrides_splits_keys_and_values():
|
||||||
|
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
|
||||||
|
raw_overrides = ['section.my_option=value1', 'section.other_option=value2']
|
||||||
|
expected_result = (
|
||||||
|
(('section', 'my_option'), 'value1'),
|
||||||
|
(('section', 'other_option'), 'value2'),
|
||||||
|
)
|
||||||
|
|
||||||
|
module.parse_overrides(raw_overrides) == expected_result
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_overrides_allows_value_with_equal_sign():
|
||||||
|
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
|
||||||
|
raw_overrides = ['section.option=this===value']
|
||||||
|
expected_result = ((('section', 'option'), 'this===value'),)
|
||||||
|
|
||||||
|
module.parse_overrides(raw_overrides) == expected_result
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_overrides_raises_on_missing_equal_sign():
|
||||||
|
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
|
||||||
|
raw_overrides = ['section.option']
|
||||||
|
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
module.parse_overrides(raw_overrides)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_overrides_allows_value_with_single_key():
|
||||||
|
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
|
||||||
|
raw_overrides = ['option=value']
|
||||||
|
expected_result = ((('option',), 'value'),)
|
||||||
|
|
||||||
|
module.parse_overrides(raw_overrides) == expected_result
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_overrides_handles_empty_overrides():
|
||||||
|
module.parse_overrides(raw_overrides=None) == ()
|
Loading…
Reference in a new issue