Allow environment variable resolution in configuration file

- all string fields containing an environment variable like ${FOO} will
  be resolved
- supported format ${FOO}, ${FOO:-bar} and ${FOO-bar} to allow default
  values if variable is not present in environment
- add --no-env argument for CLI to disable the feature which is enabled
  by default

Resolves: #546
This commit is contained in:
Sébastien MB 2022-06-16 18:52:54 +02:00
parent f2c2f3139e
commit 97b5cd089d
5 changed files with 119 additions and 4 deletions

View file

@ -188,6 +188,12 @@ def make_parsers():
action='extend', action='extend',
help='One or more configuration file options to override with specified values', help='One or more configuration file options to override with specified values',
) )
global_group.add_argument(
'--no-env',
dest='resolve_env',
action='store_false',
help='Do not resolve environment variables in configuration file',
)
global_group.add_argument( global_group.add_argument(
'--bash-completion', '--bash-completion',
default=False, default=False,

View file

@ -650,7 +650,7 @@ def run_actions(
) )
def load_configurations(config_filenames, overrides=None): def load_configurations(config_filenames, overrides=None, resolve_env=True):
''' '''
Given a sequence of configuration filenames, load and validate each configuration file. Return Given a sequence of configuration filenames, load and validate each configuration file. Return
the results as a tuple of: dict of configuration filename to corresponding parsed configuration, the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
@ -664,7 +664,7 @@ def load_configurations(config_filenames, overrides=None):
for config_filename in config_filenames: for config_filename in config_filenames:
try: try:
configs[config_filename] = validate.parse_configuration( configs[config_filename] = validate.parse_configuration(
config_filename, validate.schema_filename(), overrides config_filename, validate.schema_filename(), overrides, resolve_env
) )
except PermissionError: except PermissionError:
logs.extend( logs.extend(
@ -892,7 +892,9 @@ def main(): # pragma: no cover
sys.exit(0) sys.exit(0)
config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides) configs, parse_logs = load_configurations(
config_filenames, global_arguments.overrides, global_arguments.resolve_env
)
any_json_flags = any( any_json_flags = any(
getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values() getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()

View file

@ -1,7 +1,11 @@
import io import io
import os
import re
import ruamel.yaml import ruamel.yaml
_VARIABLE_PATTERN = re.compile(r'(?<!\\)\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\}')
def set_values(config, keys, value): def set_values(config, keys, value):
''' '''
@ -77,3 +81,35 @@ def apply_overrides(config, raw_overrides):
for (keys, value) in overrides: for (keys, value) in overrides:
set_values(config, keys, value) set_values(config, keys, value)
def _resolve_string(matcher):
'''
Get the value from environment given a matcher containing a name and an optional default value.
If the variable is not defined in environment and no default value is provided, an Error is raised.
'''
name, default = matcher.group("name"), matcher.group("default")
out = os.getenv(name, default=default)
if out is None:
raise ValueError("Cannot find variable ${name} in envivonment".format(name=name))
return out
def resolve_env_variables(item):
'''
Resolves variables like or ${FOO} from given configuration with values from process environment
Supported formats:
- ${FOO} will return FOO env variable
- ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
If any variable is missing in environment and no default value is provided, an Error is raised.
'''
if isinstance(item, str):
return _VARIABLE_PATTERN.sub(_resolve_string, item)
if isinstance(item, list):
for i, subitem in enumerate(item):
item[i] = resolve_env_variables(subitem)
if isinstance(item, dict):
for key, value in item.items():
item[key] = resolve_env_variables(value)
return item

View file

@ -79,7 +79,7 @@ def apply_logical_validation(config_filename, parsed_configuration):
) )
def parse_configuration(config_filename, schema_filename, overrides=None): def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True):
''' '''
Given the path to a config filename in YAML format, the path to a schema filename in a YAML Given the path to a config filename in YAML format, the path to a schema filename in a YAML
rendition of JSON Schema format, a sequence of configuration file override strings in the form rendition of JSON Schema format, a sequence of configuration file override strings in the form
@ -99,6 +99,8 @@ def parse_configuration(config_filename, schema_filename, overrides=None):
raise Validation_error(config_filename, (str(error),)) raise Validation_error(config_filename, (str(error),))
override.apply_overrides(config, overrides) override.apply_overrides(config, overrides)
if resolve_env:
override.resolve_env_variables(config)
normalize.normalize(config) normalize.normalize(config)
try: try:

View file

@ -0,0 +1,69 @@
import pytest
from borgmatic.config import override as module
def test_env(monkeypatch):
monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
config = {'key': 'Hello $MY_CUSTOM_VALUE'}
module.resolve_env_variables(config)
assert config == {'key': 'Hello $MY_CUSTOM_VALUE'}
def test_env_braces(monkeypatch):
monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
module.resolve_env_variables(config)
assert config == {'key': 'Hello foo'}
def test_env_default_value(monkeypatch):
monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False)
config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'}
module.resolve_env_variables(config)
assert config == {'key': 'Hello bar'}
def test_env_unknown(monkeypatch):
monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False)
config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
with pytest.raises(ValueError):
module.resolve_env_variables(config)
def test_env_full(monkeypatch):
monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
monkeypatch.delenv("MY_CUSTOM_VALUE2", raising=False)
config = {
'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
'dict': {
'key': 'value',
'anotherdict': {
'key': 'My ${MY_CUSTOM_VALUE} here',
'other': '${MY_CUSTOM_VALUE}',
'list': [
'/home/${MY_CUSTOM_VALUE}/.local',
'/var/log/',
'/home/${MY_CUSTOM_VALUE2:-bar}/.config',
],
},
},
'list': [
'/home/${MY_CUSTOM_VALUE}/.local',
'/var/log/',
'/home/${MY_CUSTOM_VALUE2-bar}/.config',
],
}
module.resolve_env_variables(config)
assert config == {
'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
'dict': {
'key': 'value',
'anotherdict': {
'key': 'My foo here',
'other': 'foo',
'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',],
},
},
'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',],
}