Code style, rename command-line flag, and move new code into its own file (#546)

NEWS

@@ -1,3 +1,7 @@
+1.6.4.dev0
+ * #546: Substitute an environment variable anywhere in a borgmatic configuration option value with
+   new "${MY_ENV_VAR}" syntax.
+
 1.6.3
  * #541: Add "borgmatic list --find" flag for searching for files across multiple archives, useful
    for hunting down that file you accidentally deleted so you can extract it. See the documentation

borgmatic/commands/arguments.py

@@ -189,7 +189,7 @@ def make_parsers():
         help='One or more configuration file options to override with specified values',
     )
     global_group.add_argument(
-        '--no-env',
+        '--no-environment-interpolation',
         dest='resolve_env',
         action='store_false',
         help='Do not resolve environment variables in configuration file',

borgmatic/config/environment.py

@@ -0,0 +1,37 @@
+import os
+import re
+
+
+_VARIABLE_PATTERN = re.compile(r'(?<!\\)\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\}')
+
+
+def _resolve_string(matcher):
+    '''
+    Get the value from environment given a matcher containing a name and an optional default value.
+    If the variable is not defined in environment and no default value is provided, an Error is raised.
+    '''
+    name, default = matcher.group("name"), matcher.group("default")
+    out = os.getenv(name, default=default)
+    if out is None:
+        raise ValueError("Cannot find variable ${name} in environment".format(name=name))
+    return out
+
+
+def resolve_env_variables(item):
+    '''
+    Resolves variables like or ${FOO} from given configuration with values from process environment
+    Supported formats:
+     - ${FOO} will return FOO env variable
+     - ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
+
+    If any variable is missing in environment and no default value is provided, an Error is raised.
+    '''
+    if isinstance(item, str):
+        return _VARIABLE_PATTERN.sub(_resolve_string, item)
+    if isinstance(item, list):
+        for i, subitem in enumerate(item):
+            item[i] = resolve_env_variables(subitem)
+    if isinstance(item, dict):
+        for key, value in item.items():
+            item[key] = resolve_env_variables(value)
+    return item

borgmatic/config/override.py

@@ -1,11 +1,7 @@
 import io
-import os
-import re
 
 import ruamel.yaml
 
-_VARIABLE_PATTERN = re.compile(r'(?<!\\)\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\}')
-
 
 def set_values(config, keys, value):
     '''
@@ -81,35 +77,3 @@ def apply_overrides(config, raw_overrides):
 
     for (keys, value) in overrides:
         set_values(config, keys, value)
-
-
-def _resolve_string(matcher):
-    '''
-    Get the value from environment given a matcher containing a name and an optional default value.
-    If the variable is not defined in environment and no default value is provided, an Error is raised.
-    '''
-    name, default = matcher.group("name"), matcher.group("default")
-    out = os.getenv(name, default=default)
-    if out is None:
-        raise ValueError("Cannot find variable ${name} in envivonment".format(name=name))
-    return out
-
-
-def resolve_env_variables(item):
-    '''
-    Resolves variables like or ${FOO} from given configuration with values from process environment
-    Supported formats:
-     - ${FOO} will return FOO env variable
-     - ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
-
-    If any variable is missing in environment and no default value is provided, an Error is raised.
-    '''
-    if isinstance(item, str):
-        return _VARIABLE_PATTERN.sub(_resolve_string, item)
-    if isinstance(item, list):
-        for i, subitem in enumerate(item):
-            item[i] = resolve_env_variables(subitem)
-    if isinstance(item, dict):
-        for key, value in item.items():
-            item[key] = resolve_env_variables(value)
-    return item

borgmatic/config/validate.py

@@ -4,7 +4,7 @@ import jsonschema
 import pkg_resources
 import ruamel.yaml
 
-from borgmatic.config import load, normalize, override
+from borgmatic.config import environment, load, normalize, override
 
 
 def schema_filename():
@@ -98,10 +98,10 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv
     except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
+    normalize.normalize(config)
     override.apply_overrides(config, overrides)
     if resolve_env:
-        override.resolve_env_variables(config)
+        environment.resolve_env_variables(config)
-    normalize.normalize(config)
 
     try:
         validator = jsonschema.Draft7Validator(schema)

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.6.3'
+VERSION = '1.6.4.dev0'
 
 
 setup(

tests/unit/config/test_environment.py

@@ -1,39 +1,39 @@
 import pytest
 
-from borgmatic.config import override as module
+from borgmatic.config import environment as module
 
 
 def test_env(monkeypatch):
-    monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
     config = {'key': 'Hello $MY_CUSTOM_VALUE'}
     module.resolve_env_variables(config)
     assert config == {'key': 'Hello $MY_CUSTOM_VALUE'}
 
 
 def test_env_braces(monkeypatch):
-    monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
     config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
     module.resolve_env_variables(config)
     assert config == {'key': 'Hello foo'}
 
 
 def test_env_default_value(monkeypatch):
-    monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False)
+    monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False)
     config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'}
     module.resolve_env_variables(config)
     assert config == {'key': 'Hello bar'}
 
 
 def test_env_unknown(monkeypatch):
-    monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False)
+    monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False)
     config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
     with pytest.raises(ValueError):
         module.resolve_env_variables(config)
 
 
 def test_env_full(monkeypatch):
-    monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
-    monkeypatch.delenv("MY_CUSTOM_VALUE2", raising=False)
+    monkeypatch.delenv('MY_CUSTOM_VALUE2', raising=False)
     config = {
         'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
         'dict': {
@@ -62,8 +62,8 @@ def test_env_full(monkeypatch):
             'anotherdict': {
                 'key': 'My foo here',
                 'other': 'foo',
-                'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',],
+                'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'],
             },
         },
-        'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',],
+        'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'],
     }