Basic YAML configuration file parsing.

.hgignore

@@ -2,6 +2,7 @@ syntax: glob
 *.egg-info
 *.pyc
 *.swp
+.cache
 .tox
 build
 dist

MANIFEST.in

@@ -0,0 +1 @@
+include borgmatic/config/schema.yaml

NEWS

@@ -1,4 +1,4 @@
-1.0.3-dev
+1.1.0
 
  * #18: Fix for README mention of sample files not included in package.
  * #22: Sample files for triggering borgmatic from a systemd timer.

borgmatic/command.py

@@ -5,7 +5,7 @@ from subprocess import CalledProcessError
 import sys
 
 from borgmatic import borg
-from borgmatic.config import parse_configuration, CONFIG_FORMAT
+from borgmatic.config.legacy import parse_configuration, CONFIG_FORMAT
 
 
 DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'

borgmatic/config/schema.yaml

@@ -0,0 +1,48 @@
+map:
+    location:
+        required: True
+        map:
+            source_directories:
+                required: True
+                seq:
+                    - type: scalar
+            one_file_system:
+                type: bool
+            remote_path:
+                type: scalar
+            repository:
+                required: True
+                type: scalar
+    storage:
+        map:
+            encryption_passphrase:
+                type: scalar
+            compression:
+                type: scalar
+            umask:
+                type: scalar
+    retention:
+        map:
+            keep_within:
+                type: scalar
+            keep_hourly:
+                type: int
+            keep_daily:
+                type: int
+            keep_weekly:
+                type: int
+            keep_monthly:
+                type: int
+            keep_yearly:
+                type: int
+            prefix:
+                type: scalar
+    consistency:
+        map:
+            checks:
+                seq:
+                    - type: str
+                      enum: ['repository', 'archives', 'disabled']
+                      unique: True
+            check_last:
+                type: int

borgmatic/config/yaml.py

@@ -0,0 +1,84 @@
+import logging
+import sys
+import warnings
+
+import pkg_resources
+import pykwalify.core
+import pykwalify.errors
+import ruamel.yaml.error
+
+
+def schema_filename():
+    '''
+    Path to the installed YAML configuration schema file, used to validate and parse the
+    configuration.
+    '''
+    return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
+
+
+class Validation_error(ValueError):
+    '''
+    A collection of error message strings generated when attempting to validate a particular
+    configurartion file.
+    '''
+    def __init__(self, config_filename, error_messages):
+        self.config_filename = config_filename
+        self.error_messages = error_messages
+
+
+def parse_configuration(config_filename, schema_filename):
+    '''
+    Given the path to a config filename in YAML format and the path to a schema filename in
+    pykwalify YAML schema format, return the parsed configuration as a data structure of nested
+    dicts and lists corresponding to the schema. Example return value:
+
+       {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
+       'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
+
+    Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
+    have permissions to read the file, or Validation_error if the config does not match the schema.
+    '''
+    warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
+    logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
+
+    try:
+        validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename])
+    except pykwalify.errors.CoreError as error:
+        if 'do not exists on disk' in str(error):
+            raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
+        if 'Unable to load any data' in str(error):
+            # If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
+            # So reach back to the originating exception from ruamel.yaml for something more useful.
+            raise Validation_error(config_filename, (error.__context__,))
+        raise
+
+    parsed_result = validator.validate(raise_exception=False)
+
+    if validator.validation_errors:
+        raise Validation_error(config_filename, validator.validation_errors)
+
+    return parsed_result
+
+
+def display_validation_error(validation_error):
+    '''
+    Given a Validation_error, display its error messages to stderr.
+    '''
+    print(
+        'An error occurred while parsing a configuration file at {}:'.format(
+            validation_error.config_filename
+        ),
+        file=sys.stderr,
+    )
+
+    for error in validation_error.error_messages:
+        print(error, file=sys.stderr)
+
+
+# FOR TESTING
+if __name__ == '__main__':
+    try:
+        configuration = parse_configuration('sample/config.yaml', schema_filename())
+        print(configuration)
+    except Validation_error as error:
+        display_validation_error(error)

borgmatic/tests/builtins.py

@@ -1,6 +0,0 @@
-from flexmock import flexmock
-import sys
-
-
-def builtins_mock():
-    return flexmock(sys.modules['builtins'])

borgmatic/tests/integration/config/test_legacy.py

@@ -3,7 +3,7 @@ from io import StringIO
 from collections import OrderedDict
 import string
 
-from borgmatic import config as module
+from borgmatic.config import legacy as module
 
 
 def test_parse_section_options_with_punctuation_should_return_section_options():

borgmatic/tests/unit/config/test_legacy.py

@@ -3,7 +3,7 @@ from collections import OrderedDict
 from flexmock import flexmock
 import pytest
 
-from borgmatic import config as module
+from borgmatic.config import legacy as module
 
 
 def test_option_should_create_config_option():

borgmatic/tests/unit/test_borg.py

@@ -1,11 +1,11 @@
 from collections import OrderedDict
 from subprocess import STDOUT
+import sys
 import os
 
 from flexmock import flexmock
 
 from borgmatic import borg as module
-from borgmatic.tests.builtins import builtins_mock
 from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
@@ -389,7 +389,7 @@ def test_check_archives_should_call_borg_with_parameters():
     )
     insert_platform_mock()
     insert_datetime_mock()
-    builtins_mock().should_receive('open').and_return(stdout)
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
     flexmock(module.os).should_receive('devnull')
 
     module.check_archives(
@@ -464,7 +464,7 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param
     )
     insert_platform_mock()
     insert_datetime_mock()
-    builtins_mock().should_receive('open').and_return(stdout)
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
     flexmock(module.os).should_receive('devnull')
 
     module.check_archives(

sample/config.yaml

@@ -0,0 +1,54 @@
+location:
+    # List of source directories to backup. Globs are expanded.
+    source_directories:
+        - /home
+        - /etc
+        - /var/log/syslog*
+
+    # Stay in same file system (do not cross mount points).
+    #one_file_system: yes
+
+    # Alternate Borg remote executable (defaults to "borg"):
+    #remote_path: borg1
+
+    # Path to local or remote repository.
+    repository: user@backupserver:sourcehostname.borg
+
+#storage:
+    # Passphrase to unlock the encryption key with. Only use on repositories
+    # that were initialized with passphrase/repokey encryption.
+    #encryption_passphrase: foo
+
+    # Type of compression to use when creating archives. See
+    # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
+    # for details. Defaults to no compression.
+    #compression: lz4
+
+    # Umask to be used for borg create.
+    #umask: 0077
+
+retention:
+    # Retention policy for how many backups to keep in each category. See
+    # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for
+    # details.
+    #keep_within: 3H
+    #keep_hourly: 24
+    keep_daily: 7
+    keep_weekly: 4
+    keep_monthly: 6
+    keep_yearly: 1
+
+    # When pruning, only consider archive names starting with this prefix.
+    #prefix: sourcehostname
+
+consistency:
+    # List of consistency checks to run: "repository", "archives", or both.
+    # Defaults to both. Set to "disabled" to disable all consistency checks.
+    # See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
+    # for details.
+    checks:
+        - repository
+        - archives
+
+    # Restrict the number of checked archives to the last n.
+    #check_last: 3

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.0.3-dev'
+VERSION = '1.1.0'
 
 
 setup(
@@ -30,8 +30,14 @@ setup(
     obsoletes=[
         'atticmatic',
     ],
+    install_requires=(
+        'pykwalify',
+        'ruamel.yaml<=0.15',
+        'setuptools',
+    ),
     tests_require=(
         'flexmock',
         'pytest',
-    )
+    ),
+    include_package_data=True,
 )