Basic YAML generating / validating / converting to.

NEWS

@@ -1,8 +1,9 @@
-1.1.0
+1.1.0.dev0
 
+ * Switched config file format to YAML. Run convert-borgmatic-config to upgrade.
+ * Dropped Python 2 support. Now Python 3 only.
  * #18: Fix for README mention of sample files not included in package.
  * #22: Sample files for triggering borgmatic from a systemd timer.
- * Dropped Python 2 support. Now Python 3 only.
  * Added logo.
 
 1.0.3

borgmatic/commands/borgmatic.py

@@ -5,7 +5,7 @@ from subprocess import CalledProcessError
 import sys
 
 from borgmatic import borg
-from borgmatic.config.yaml import parse_configuration, schema_filename
+from borgmatic.config.validate import parse_configuration, schema_filename
 
 
 DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
@@ -14,9 +14,8 @@ DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 
 def parse_arguments(*arguments):
     '''
-    Given the name of the command with which this script was invoked and command-line arguments,
-    parse the arguments and return them as an ArgumentParser instance. Use the command name to
-    determine the default configuration and excludes paths.
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
     '''
     parser = ArgumentParser()
     parser.add_argument(

borgmatic/commands/convert_config.py

@@ -0,0 +1,54 @@
+from __future__ import print_function
+from argparse import ArgumentParser
+import os
+from subprocess import CalledProcessError
+import sys
+
+from ruamel import yaml
+
+from borgmatic import borg
+from borgmatic.config import convert, generate, legacy, validate
+
+
+DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
+DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
+DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
+
+
+def parse_arguments(*arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
+    '''
+    parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.')
+    parser.add_argument(
+        '-s', '--source',
+        dest='source_filename',
+        default=DEFAULT_SOURCE_CONFIG_FILENAME,
+        help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
+    )
+    parser.add_argument(
+        '-d', '--destination',
+        dest='destination_filename',
+        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
+        help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
+    )
+
+    return parser.parse_args(arguments)
+
+
+def main():
+    try:
+        args = parse_arguments(*sys.argv[1:])
+        source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT)
+        schema = yaml.round_trip_load(open(validate.schema_filename()).read())
+
+        destination_config = convert.convert_legacy_parsed_config(source_config, schema)
+
+        generate.write_configuration(args.destination_filename, destination_config)
+
+        # TODO: As a backstop, check that the written config can actually be read and parsed, and
+        # that it matches the destination config data structure that was written.
+    except (ValueError, OSError) as error:
+        print(error, file=sys.stderr)
+        sys.exit(1)

borgmatic/config/convert.py

@@ -0,0 +1,41 @@
+from ruamel import yaml
+
+from borgmatic.config import generate
+
+
+def _convert_section(source_section_config, section_schema):
+    '''
+    Given a legacy Parsed_config instance for a single section, convert it to its corresponding
+    yaml.comments.CommentedMap representation in preparation for actual serialization to YAML.
+
+    Additionally, use the section schema as a source of helpful comments to include within the
+    returned CommentedMap.
+    '''
+    destination_section_config = yaml.comments.CommentedMap(source_section_config)
+    generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT)
+
+    return destination_section_config
+
+
+def convert_legacy_parsed_config(source_config, schema):
+    '''
+    Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its
+    corresponding yaml.comments.CommentedMap representation in preparation for actual serialization
+    to YAML.
+
+    Additionally, use the given schema as a source of helpful comments to include within the
+    returned CommentedMap.
+    '''
+    destination_config = yaml.comments.CommentedMap([
+        (section_name, _convert_section(section_config, schema['map'][section_name]))
+        for section_name, section_config in source_config._asdict().items()
+    ])
+
+    destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
+
+    if source_config.consistency['checks']:
+        destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
+
+    generate.add_comments_to_configuration(destination_config, schema)
+
+    return destination_config

borgmatic/config/generate.py

@@ -0,0 +1,90 @@
+from collections import OrderedDict
+
+from ruamel import yaml
+
+
+INDENT = 4
+
+
+def write_configuration(config_filename, config):
+    '''
+    Given a target config filename and a config data structure of nested OrderedDicts, write out the
+    config to file as YAML.
+    '''
+    with open(config_filename, 'w') as config_file:
+        config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))
+
+
+def _insert_newline_before_comment(config, field_name):
+    '''
+    Using some ruamel.yaml black magic, insert a blank line in the config right befor the given
+    field and its comments.
+    '''
+    config.ca.items[field_name][1].insert(
+        0,
+        yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None),
+    )
+
+
+def add_comments_to_configuration(config, schema, indent=0):
+    '''
+    Using descriptions from a schema as a source, add those descriptions as comments to the given
+    config before each field. This function only adds comments for the top-most config map level.
+    Indent the comment the given number of characters.
+    '''
+    for index, field_name in enumerate(config.keys()):
+        field_schema = schema['map'].get(field_name, {})
+        description = field_schema.get('desc')
+
+        # No description to use? Skip it.
+        if not schema or not description:
+            continue
+
+        config.yaml_set_comment_before_after_key(
+            key=field_name,
+            before=description,
+            indent=indent,
+        )
+        if index > 0:
+            _insert_newline_before_comment(config, field_name)
+
+
+def _section_schema_to_sample_configuration(section_schema):
+    '''
+    Given the schema for a particular config section, generate and return sample config for that
+    section. Include comments for each field based on the schema "desc" description.
+    '''
+    section_config = yaml.comments.CommentedMap([
+        (field_name, field_schema['example'])
+        for field_name, field_schema in section_schema['map'].items()
+    ])
+
+    add_comments_to_configuration(section_config, section_schema, indent=INDENT)
+
+    return section_config
+
+
+def _schema_to_sample_configuration(schema):
+    '''
+    Given a loaded configuration schema, generate and return sample config for it. Include comments
+    for each section based on the schema "desc" description.
+    '''
+    config = yaml.comments.CommentedMap([
+        (section_name, _section_schema_to_sample_configuration(section_schema))
+        for section_name, section_schema in schema['map'].items()
+    ])
+
+    add_comments_to_configuration(config, schema)
+
+    return config
+
+
+def generate_sample_configuration(config_filename, schema_filename):
+    '''
+    Given a target config filename and the path to a schema filename in pykwalify YAML schema
+    format, write out a sample configuration file based on that schema.
+    '''
+    schema = yaml.round_trip_load(open(schema_filename))
+    config = _schema_to_sample_configuration(schema)
+
+    write_configuration(config_filename, config)

borgmatic/config/schema.yaml

@@ -1,48 +1,110 @@
+name: Borgmatic configuration file schema
 map:
     location:
+        desc: |
+            Where to look for files to backup, and where to store those backups. See
+            https://borgbackup.readthedocs.io/en/stable/quickstart.html and
+            https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details.
         required: True
         map:
             source_directories:
                 required: True
                 seq:
                     - type: scalar
+                desc: List of source directories to backup. Globs are expanded.
+                example:
+                    - /home
+                    - /etc
+                    - /var/log/syslog*
             one_file_system:
                 type: bool
+                desc: Stay in same file system (do not cross mount points).
+                example: yes
             remote_path:
                 type: scalar
+                desc: Alternate Borg remote executable. Defaults to "borg".
+                example: borg1
             repository:
                 required: True
                 type: scalar
+                desc: Path to local or remote repository.
+                example: user@backupserver:sourcehostname.borg
     storage:
+        desc: |
+            Repository storage options. See
+            https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and
+            https://borgbackup.readthedocs.io/en/stable/usage.html#environment-variables for details.
         map:
             encryption_passphrase:
                 type: scalar
+                desc: |
+                    Passphrase to unlock the encryption key with. Only use on repositories that were
+                    initialized with passphrase/repokey encryption. Quote the value if it contains
+                    punctuation, so it parses correctly. And backslash any quote or backslash
+                    literals as well.
+                example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
             compression:
                 type: scalar
+                desc: |
+                    Type of compression to use when creating archives. See
+                    https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
+                    Defaults to no compression.
+                example: lz4
             umask:
                 type: scalar
+                desc: Umask to be used for borg create.
+                example: 0077
     retention:
+        desc: |
+            Retention policy for how many backups to keep in each category. See
+            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
         map:
             keep_within:
                 type: scalar
+                desc: Keep all archives within this time interval.
+                example: 3H
             keep_hourly:
                 type: int
+                desc: Number of hourly archives to keep.
+                example: 24
             keep_daily:
                 type: int
+                desc: Number of daily archives to keep.
+                example: 7
             keep_weekly:
                 type: int
+                desc: Number of weekly archives to keep.
+                example: 4
             keep_monthly:
                 type: int
+                desc: Number of monthly archives to keep.
+                example: 6
             keep_yearly:
                 type: int
+                desc: Number of yearly archives to keep.
+                example: 1
             prefix:
                 type: scalar
+                desc: When pruning, only consider archive names starting with this prefix.
+                example: sourcehostname
     consistency:
+        desc: |
+            Consistency checks to run after backups. See
+            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
         map:
             checks:
                 seq:
                     - type: str
                       enum: ['repository', 'archives', 'disabled']
                       unique: True
+                desc: |
+                    List of consistency checks to run: "repository", "archives", or both. Defaults
+                    to both. Set to "disabled" to disable all consistency checks. See
+                    https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
+                example:
+                    - repository
+                    - archives
             check_last:
                 type: int
+                desc: Restrict the number of checked archives to the last n.
+                example: 3

borgmatic/config/validate.py

@@ -5,7 +5,7 @@ import warnings
 import pkg_resources
 import pykwalify.core
 import pykwalify.errors
-import ruamel.yaml.error
+from ruamel import yaml
 
 
 def schema_filename():
@@ -38,20 +38,18 @@ def parse_configuration(config_filename, schema_filename):
     Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
     have permissions to read the file, or Validation_error if the config does not match the schema.
     '''
-    warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
-    logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
-
     try:
-        validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename])
-    except pykwalify.errors.CoreError as error:
-        if 'do not exists on disk' in str(error):
-            raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
-        if 'Unable to load any data' in str(error):
-            # If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
-            # So reach back to the originating exception from ruamel.yaml for something more useful.
-            raise Validation_error(config_filename, (error.__context__,))
-        raise
+        schema = yaml.round_trip_load(open(schema_filename))
+    except yaml.error.YAMLError as error:
+        raise Validation_error(config_filename, (str(error),))
 
+    # pykwalify gets angry if the example field is not a string. So rather than bend to its will,
+    # simply remove all examples before passing the schema to pykwalify.
+    for section_name, section_schema in schema['map'].items():
+        for field_name, field_schema in section_schema['map'].items():
+            field_schema.pop('example')
+
+    validator = pykwalify.core.Core(source_file=config_filename, schema_data=schema)
     parsed_result = validator.validate(raise_exception=False)
 
     if validator.validation_errors:
@@ -73,12 +71,3 @@ def display_validation_error(validation_error):
 
     for error in validation_error.error_messages:
         print(error, file=sys.stderr)
-
-
-# FOR TESTING
-if __name__ == '__main__':
-    try:
-        configuration = parse_configuration('sample/config.yaml', schema_filename())
-        print(configuration)
-    except Validation_error as error:
-        display_validation_error(error)

borgmatic/tests/integration/commands/test_borgmatic.py

@@ -4,7 +4,7 @@ import sys
 from flexmock import flexmock
 import pytest
 
-from borgmatic import command as module
+from borgmatic.commands import borgmatic as module
 
 
 def test_parse_arguments_with_no_arguments_uses_defaults():

borgmatic/tests/integration/config/test_validate.py

@@ -6,7 +6,7 @@ import os
 from flexmock import flexmock
 import pytest
 
-from borgmatic.config import yaml as module
+from borgmatic.config import validate as module
 
 
 def test_schema_filename_returns_plausable_path():
@@ -18,13 +18,13 @@ def test_schema_filename_returns_plausable_path():
 def mock_config_and_schema(config_yaml):
     '''
     Set up mocks for the config config YAML string and the default schema so that pykwalify consumes
-    them when parsing the configuration. This is a little brittle in that it's relying on pykwalify
-    to open() the respective files in a particular order.
+    them when parsing the configuration. This is a little brittle in that it's relying on the code
+    under test to open() the respective files in a particular order.
     '''
-    config_stream = io.StringIO(config_yaml)
     schema_stream = open(module.schema_filename())
+    config_stream = io.StringIO(config_yaml)
     builtins = flexmock(sys.modules['builtins']).should_call('open').mock
-    builtins.should_receive('open').and_return(config_stream).and_return(schema_stream)
+    builtins.should_receive('open').and_return(schema_stream).and_return(config_stream)
     flexmock(os.path).should_receive('exists').and_return(True)
 
 
@@ -87,7 +87,8 @@ def test_parse_configuration_raises_for_missing_config_file():
 
 def test_parse_configuration_raises_for_missing_schema_file():
     mock_config_and_schema('')
-    flexmock(os.path).should_receive('exists').with_args('schema.yaml').and_return(False)
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('schema.yaml').and_raise(FileNotFoundError)
 
     with pytest.raises(FileNotFoundError):
         module.parse_configuration('config.yaml', 'schema.yaml')

borgmatic/tests/unit/config/test_convert.py

@@ -0,0 +1,44 @@
+from collections import defaultdict, OrderedDict, namedtuple
+
+from borgmatic.config import convert as module
+
+
+Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency'))
+
+
+def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
+    source_config = Parsed_config(
+        location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]),
+        storage=OrderedDict([('encryption_passphrase', 'supersecret')]),
+        retention=OrderedDict([('keep_daily', 7)]),
+        consistency=OrderedDict([('checks', 'repository')]),
+    )
+    schema = {'map': defaultdict(lambda: {'map': {}})}
+
+    destination_config = module.convert_legacy_parsed_config(source_config, schema)
+
+    assert destination_config == OrderedDict([
+        ('location', OrderedDict([('source_directories', ['/home']), ('repository', 'hostname.borg')])),
+        ('storage', OrderedDict([('encryption_passphrase', 'supersecret')])),
+        ('retention', OrderedDict([('keep_daily', 7)])),
+        ('consistency', OrderedDict([('checks', ['repository'])])),
+    ])
+
+
+def test_convert_legacy_parsed_config_splits_space_separated_values():
+    source_config = Parsed_config(
+        location=OrderedDict([('source_directories', '/home /etc')]),
+        storage=OrderedDict(),
+        retention=OrderedDict(),
+        consistency=OrderedDict([('checks', 'repository archives')]),
+    )
+    schema = {'map': defaultdict(lambda: {'map': {}})}
+
+    destination_config = module.convert_legacy_parsed_config(source_config, schema) 
+
+    assert destination_config == OrderedDict([
+        ('location', OrderedDict([('source_directories', ['/home', '/etc'])])),
+        ('storage', OrderedDict()),
+        ('retention', OrderedDict()),
+        ('consistency', OrderedDict([('checks', ['repository', 'archives'])])),
+    ])

sample/config.yaml

@@ -16,8 +16,10 @@ location:
 
 #storage:
     # Passphrase to unlock the encryption key with. Only use on repositories
-    # that were initialized with passphrase/repokey encryption.
-    #encryption_passphrase: foo
+    # that were initialized with passphrase/repokey encryption. Quote the value
+    # if it contains punctuation so it parses correctly. And backslash any
+    # quote or backslash literals as well.
+    #encryption_passphrase: "foo"
 
     # Type of compression to use when creating archives. See
     # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.0'
+VERSION = '1.1.0.dev0'
 
 
 setup(
@@ -24,7 +24,8 @@ setup(
     packages=find_packages(),
     entry_points={
         'console_scripts': [
-            'borgmatic = borgmatic.command:main',
+            'borgmatic = borgmatic.commands.borgmatic:main',
+            'convert-borgmatic-config = borgmatic.commands.convert_config:main',
         ]
     },
     obsoletes=[

test_requirements.txt

@@ -1,2 +1,5 @@
 flexmock==0.10.2
+pykwalify==1.6.0
 pytest==2.9.1
+pytest-cov==2.5.1
+ruamel.yaml==0.15.18

tox.ini

@@ -5,4 +5,4 @@ skipsdist=True
 [testenv]
 usedevelop=True
 deps=-rtest_requirements.txt
-commands = py.test borgmatic []
+commands = py.test --cov=borgmatic borgmatic []