Basic YAML configuration file parsing.
This commit is contained in:
parent
e00f74ddf7
commit
6e85940d63
16 changed files with 203 additions and 15 deletions
|
@ -2,6 +2,7 @@ syntax: glob
|
||||||
*.egg-info
|
*.egg-info
|
||||||
*.pyc
|
*.pyc
|
||||||
*.swp
|
*.swp
|
||||||
|
.cache
|
||||||
.tox
|
.tox
|
||||||
build
|
build
|
||||||
dist
|
dist
|
||||||
|
|
1
MANIFEST.in
Normal file
1
MANIFEST.in
Normal file
|
@ -0,0 +1 @@
|
||||||
|
include borgmatic/config/schema.yaml
|
2
NEWS
2
NEWS
|
@ -1,4 +1,4 @@
|
||||||
1.0.3-dev
|
1.1.0
|
||||||
|
|
||||||
* #18: Fix for README mention of sample files not included in package.
|
* #18: Fix for README mention of sample files not included in package.
|
||||||
* #22: Sample files for triggering borgmatic from a systemd timer.
|
* #22: Sample files for triggering borgmatic from a systemd timer.
|
||||||
|
|
|
@ -5,7 +5,7 @@ from subprocess import CalledProcessError
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from borgmatic import borg
|
from borgmatic import borg
|
||||||
from borgmatic.config import parse_configuration, CONFIG_FORMAT
|
from borgmatic.config.legacy import parse_configuration, CONFIG_FORMAT
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'
|
DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'
|
||||||
|
|
0
borgmatic/config/__init__.py
Normal file
0
borgmatic/config/__init__.py
Normal file
48
borgmatic/config/schema.yaml
Normal file
48
borgmatic/config/schema.yaml
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
map:
|
||||||
|
location:
|
||||||
|
required: True
|
||||||
|
map:
|
||||||
|
source_directories:
|
||||||
|
required: True
|
||||||
|
seq:
|
||||||
|
- type: scalar
|
||||||
|
one_file_system:
|
||||||
|
type: bool
|
||||||
|
remote_path:
|
||||||
|
type: scalar
|
||||||
|
repository:
|
||||||
|
required: True
|
||||||
|
type: scalar
|
||||||
|
storage:
|
||||||
|
map:
|
||||||
|
encryption_passphrase:
|
||||||
|
type: scalar
|
||||||
|
compression:
|
||||||
|
type: scalar
|
||||||
|
umask:
|
||||||
|
type: scalar
|
||||||
|
retention:
|
||||||
|
map:
|
||||||
|
keep_within:
|
||||||
|
type: scalar
|
||||||
|
keep_hourly:
|
||||||
|
type: int
|
||||||
|
keep_daily:
|
||||||
|
type: int
|
||||||
|
keep_weekly:
|
||||||
|
type: int
|
||||||
|
keep_monthly:
|
||||||
|
type: int
|
||||||
|
keep_yearly:
|
||||||
|
type: int
|
||||||
|
prefix:
|
||||||
|
type: scalar
|
||||||
|
consistency:
|
||||||
|
map:
|
||||||
|
checks:
|
||||||
|
seq:
|
||||||
|
- type: str
|
||||||
|
enum: ['repository', 'archives', 'disabled']
|
||||||
|
unique: True
|
||||||
|
check_last:
|
||||||
|
type: int
|
84
borgmatic/config/yaml.py
Normal file
84
borgmatic/config/yaml.py
Normal file
|
@ -0,0 +1,84 @@
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
import pkg_resources
|
||||||
|
import pykwalify.core
|
||||||
|
import pykwalify.errors
|
||||||
|
import ruamel.yaml.error
|
||||||
|
|
||||||
|
|
||||||
|
def schema_filename():
|
||||||
|
'''
|
||||||
|
Path to the installed YAML configuration schema file, used to validate and parse the
|
||||||
|
configuration.
|
||||||
|
'''
|
||||||
|
return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
|
||||||
|
|
||||||
|
|
||||||
|
class Validation_error(ValueError):
|
||||||
|
'''
|
||||||
|
A collection of error message strings generated when attempting to validate a particular
|
||||||
|
configurartion file.
|
||||||
|
'''
|
||||||
|
def __init__(self, config_filename, error_messages):
|
||||||
|
self.config_filename = config_filename
|
||||||
|
self.error_messages = error_messages
|
||||||
|
|
||||||
|
|
||||||
|
def parse_configuration(config_filename, schema_filename):
|
||||||
|
'''
|
||||||
|
Given the path to a config filename in YAML format and the path to a schema filename in
|
||||||
|
pykwalify YAML schema format, return the parsed configuration as a data structure of nested
|
||||||
|
dicts and lists corresponding to the schema. Example return value:
|
||||||
|
|
||||||
|
{'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
|
||||||
|
'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
|
||||||
|
|
||||||
|
Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
|
||||||
|
have permissions to read the file, or Validation_error if the config does not match the schema.
|
||||||
|
'''
|
||||||
|
warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
|
||||||
|
logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
|
||||||
|
|
||||||
|
try:
|
||||||
|
validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename])
|
||||||
|
except pykwalify.errors.CoreError as error:
|
||||||
|
if 'do not exists on disk' in str(error):
|
||||||
|
raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
|
||||||
|
if 'Unable to load any data' in str(error):
|
||||||
|
# If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
|
||||||
|
# So reach back to the originating exception from ruamel.yaml for something more useful.
|
||||||
|
raise Validation_error(config_filename, (error.__context__,))
|
||||||
|
raise
|
||||||
|
|
||||||
|
parsed_result = validator.validate(raise_exception=False)
|
||||||
|
|
||||||
|
if validator.validation_errors:
|
||||||
|
raise Validation_error(config_filename, validator.validation_errors)
|
||||||
|
|
||||||
|
return parsed_result
|
||||||
|
|
||||||
|
|
||||||
|
def display_validation_error(validation_error):
|
||||||
|
'''
|
||||||
|
Given a Validation_error, display its error messages to stderr.
|
||||||
|
'''
|
||||||
|
print(
|
||||||
|
'An error occurred while parsing a configuration file at {}:'.format(
|
||||||
|
validation_error.config_filename
|
||||||
|
),
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
|
||||||
|
for error in validation_error.error_messages:
|
||||||
|
print(error, file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
# FOR TESTING
|
||||||
|
if __name__ == '__main__':
|
||||||
|
try:
|
||||||
|
configuration = parse_configuration('sample/config.yaml', schema_filename())
|
||||||
|
print(configuration)
|
||||||
|
except Validation_error as error:
|
||||||
|
display_validation_error(error)
|
|
@ -1,6 +0,0 @@
|
||||||
from flexmock import flexmock
|
|
||||||
import sys
|
|
||||||
|
|
||||||
|
|
||||||
def builtins_mock():
|
|
||||||
return flexmock(sys.modules['builtins'])
|
|
0
borgmatic/tests/integration/config/__init__.py
Normal file
0
borgmatic/tests/integration/config/__init__.py
Normal file
|
@ -3,7 +3,7 @@ from io import StringIO
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
import string
|
import string
|
||||||
|
|
||||||
from borgmatic import config as module
|
from borgmatic.config import legacy as module
|
||||||
|
|
||||||
|
|
||||||
def test_parse_section_options_with_punctuation_should_return_section_options():
|
def test_parse_section_options_with_punctuation_should_return_section_options():
|
0
borgmatic/tests/unit/config/__init__.py
Normal file
0
borgmatic/tests/unit/config/__init__.py
Normal file
|
@ -3,7 +3,7 @@ from collections import OrderedDict
|
||||||
from flexmock import flexmock
|
from flexmock import flexmock
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from borgmatic import config as module
|
from borgmatic.config import legacy as module
|
||||||
|
|
||||||
|
|
||||||
def test_option_should_create_config_option():
|
def test_option_should_create_config_option():
|
|
@ -1,11 +1,11 @@
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from subprocess import STDOUT
|
from subprocess import STDOUT
|
||||||
|
import sys
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from flexmock import flexmock
|
from flexmock import flexmock
|
||||||
|
|
||||||
from borgmatic import borg as module
|
from borgmatic import borg as module
|
||||||
from borgmatic.tests.builtins import builtins_mock
|
|
||||||
from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
|
from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
|
||||||
|
|
||||||
|
|
||||||
|
@ -389,7 +389,7 @@ def test_check_archives_should_call_borg_with_parameters():
|
||||||
)
|
)
|
||||||
insert_platform_mock()
|
insert_platform_mock()
|
||||||
insert_datetime_mock()
|
insert_datetime_mock()
|
||||||
builtins_mock().should_receive('open').and_return(stdout)
|
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
|
||||||
flexmock(module.os).should_receive('devnull')
|
flexmock(module.os).should_receive('devnull')
|
||||||
|
|
||||||
module.check_archives(
|
module.check_archives(
|
||||||
|
@ -464,7 +464,7 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param
|
||||||
)
|
)
|
||||||
insert_platform_mock()
|
insert_platform_mock()
|
||||||
insert_datetime_mock()
|
insert_datetime_mock()
|
||||||
builtins_mock().should_receive('open').and_return(stdout)
|
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
|
||||||
flexmock(module.os).should_receive('devnull')
|
flexmock(module.os).should_receive('devnull')
|
||||||
|
|
||||||
module.check_archives(
|
module.check_archives(
|
||||||
|
|
54
sample/config.yaml
Normal file
54
sample/config.yaml
Normal file
|
@ -0,0 +1,54 @@
|
||||||
|
location:
|
||||||
|
# List of source directories to backup. Globs are expanded.
|
||||||
|
source_directories:
|
||||||
|
- /home
|
||||||
|
- /etc
|
||||||
|
- /var/log/syslog*
|
||||||
|
|
||||||
|
# Stay in same file system (do not cross mount points).
|
||||||
|
#one_file_system: yes
|
||||||
|
|
||||||
|
# Alternate Borg remote executable (defaults to "borg"):
|
||||||
|
#remote_path: borg1
|
||||||
|
|
||||||
|
# Path to local or remote repository.
|
||||||
|
repository: user@backupserver:sourcehostname.borg
|
||||||
|
|
||||||
|
#storage:
|
||||||
|
# Passphrase to unlock the encryption key with. Only use on repositories
|
||||||
|
# that were initialized with passphrase/repokey encryption.
|
||||||
|
#encryption_passphrase: foo
|
||||||
|
|
||||||
|
# Type of compression to use when creating archives. See
|
||||||
|
# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
|
||||||
|
# for details. Defaults to no compression.
|
||||||
|
#compression: lz4
|
||||||
|
|
||||||
|
# Umask to be used for borg create.
|
||||||
|
#umask: 0077
|
||||||
|
|
||||||
|
retention:
|
||||||
|
# Retention policy for how many backups to keep in each category. See
|
||||||
|
# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for
|
||||||
|
# details.
|
||||||
|
#keep_within: 3H
|
||||||
|
#keep_hourly: 24
|
||||||
|
keep_daily: 7
|
||||||
|
keep_weekly: 4
|
||||||
|
keep_monthly: 6
|
||||||
|
keep_yearly: 1
|
||||||
|
|
||||||
|
# When pruning, only consider archive names starting with this prefix.
|
||||||
|
#prefix: sourcehostname
|
||||||
|
|
||||||
|
consistency:
|
||||||
|
# List of consistency checks to run: "repository", "archives", or both.
|
||||||
|
# Defaults to both. Set to "disabled" to disable all consistency checks.
|
||||||
|
# See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
|
||||||
|
# for details.
|
||||||
|
checks:
|
||||||
|
- repository
|
||||||
|
- archives
|
||||||
|
|
||||||
|
# Restrict the number of checked archives to the last n.
|
||||||
|
#check_last: 3
|
10
setup.py
10
setup.py
|
@ -1,7 +1,7 @@
|
||||||
from setuptools import setup, find_packages
|
from setuptools import setup, find_packages
|
||||||
|
|
||||||
|
|
||||||
VERSION = '1.0.3-dev'
|
VERSION = '1.1.0'
|
||||||
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
|
@ -30,8 +30,14 @@ setup(
|
||||||
obsoletes=[
|
obsoletes=[
|
||||||
'atticmatic',
|
'atticmatic',
|
||||||
],
|
],
|
||||||
|
install_requires=(
|
||||||
|
'pykwalify',
|
||||||
|
'ruamel.yaml<=0.15',
|
||||||
|
'setuptools',
|
||||||
|
),
|
||||||
tests_require=(
|
tests_require=(
|
||||||
'flexmock',
|
'flexmock',
|
||||||
'pytest',
|
'pytest',
|
||||||
)
|
),
|
||||||
|
include_package_data=True,
|
||||||
)
|
)
|
||||||
|
|
Loading…
Reference in a new issue