Support for backing up to multiple repositories.
This commit is contained in:
parent
548212274f
commit
499f8aa0a4
9 changed files with 115 additions and 63 deletions
1
NEWS
1
NEWS
|
@ -5,6 +5,7 @@
|
||||||
* Dropped Python 2 support. Now Python 3 only.
|
* Dropped Python 2 support. Now Python 3 only.
|
||||||
* #18: Fix for README mention of sample files not included in package.
|
* #18: Fix for README mention of sample files not included in package.
|
||||||
* #22: Sample files for triggering borgmatic from a systemd timer.
|
* #22: Sample files for triggering borgmatic from a systemd timer.
|
||||||
|
* Support for backing up to multiple repositories.
|
||||||
* To free up space, now pruning backups prior to creating a new backup.
|
* To free up space, now pruning backups prior to creating a new backup.
|
||||||
* Enabled test coverage output during tox runs.
|
* Enabled test coverage output during tox runs.
|
||||||
* Added logo.
|
* Added logo.
|
||||||
|
|
|
@ -21,8 +21,9 @@ location:
|
||||||
- /etc
|
- /etc
|
||||||
- /var/log/syslog*
|
- /var/log/syslog*
|
||||||
|
|
||||||
# Path to local or remote repository.
|
# Paths to local or remote repositories.
|
||||||
repository: user@backupserver:sourcehostname.borg
|
repositories:
|
||||||
|
- user@backupserver:sourcehostname.borg
|
||||||
|
|
||||||
# Any paths matching these patterns are excluded from backups.
|
# Any paths matching these patterns are excluded from backups.
|
||||||
exclude_patterns:
|
exclude_patterns:
|
||||||
|
|
|
@ -39,8 +39,7 @@ def _write_exclude_file(exclude_patterns=None):
|
||||||
|
|
||||||
|
|
||||||
def create_archive(
|
def create_archive(
|
||||||
verbosity, storage_config, source_directories, repository, exclude_patterns=None,
|
verbosity, repository, location_config, storage_config, command=COMMAND,
|
||||||
command=COMMAND, one_file_system=None, remote_path=None,
|
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
|
Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
|
||||||
|
@ -49,17 +48,18 @@ def create_archive(
|
||||||
sources = tuple(
|
sources = tuple(
|
||||||
itertools.chain.from_iterable(
|
itertools.chain.from_iterable(
|
||||||
glob.glob(directory) or [directory]
|
glob.glob(directory) or [directory]
|
||||||
for directory in source_directories
|
for directory in location_config['source_directories']
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
exclude_file = _write_exclude_file(exclude_patterns)
|
exclude_file = _write_exclude_file(location_config.get('exclude_patterns'))
|
||||||
exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
|
exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
|
||||||
compression = storage_config.get('compression', None)
|
compression = storage_config.get('compression', None)
|
||||||
compression_flags = ('--compression', compression) if compression else ()
|
compression_flags = ('--compression', compression) if compression else ()
|
||||||
umask = storage_config.get('umask', None)
|
umask = storage_config.get('umask', None)
|
||||||
umask_flags = ('--umask', str(umask)) if umask else ()
|
umask_flags = ('--umask', str(umask)) if umask else ()
|
||||||
one_file_system_flags = ('--one-file-system',) if one_file_system else ()
|
one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
|
||||||
|
remote_path = location_config.get('remote_path')
|
||||||
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
||||||
verbosity_flags = {
|
verbosity_flags = {
|
||||||
VERBOSITY_SOME: ('--info', '--stats',),
|
VERBOSITY_SOME: ('--info', '--stats',),
|
||||||
|
|
|
@ -44,16 +44,22 @@ def main(): # pragma: no cover
|
||||||
args = parse_arguments(*sys.argv[1:])
|
args = parse_arguments(*sys.argv[1:])
|
||||||
convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename)
|
convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename)
|
||||||
config = validate.parse_configuration(args.config_filename, validate.schema_filename())
|
config = validate.parse_configuration(args.config_filename, validate.schema_filename())
|
||||||
repository = config['location']['repository']
|
(location, storage, retention, consistency) = (
|
||||||
remote_path = config['location']['remote_path']
|
|
||||||
(storage, retention, consistency) = (
|
|
||||||
config.get(section_name, {})
|
config.get(section_name, {})
|
||||||
for section_name in ('storage', 'retention', 'consistency')
|
for section_name in ('location', 'storage', 'retention', 'consistency')
|
||||||
)
|
)
|
||||||
|
remote_path = location.get('remote_path')
|
||||||
|
|
||||||
borg.initialize(storage)
|
borg.initialize(storage)
|
||||||
|
|
||||||
|
for repository in location['repositories']:
|
||||||
borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
|
borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
|
||||||
borg.create_archive(args.verbosity, storage, **config['location'])
|
borg.create_archive(
|
||||||
|
args.verbosity,
|
||||||
|
repository,
|
||||||
|
location,
|
||||||
|
storage,
|
||||||
|
)
|
||||||
borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
|
borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
|
||||||
except (ValueError, OSError, CalledProcessError) as error:
|
except (ValueError, OSError, CalledProcessError) as error:
|
||||||
print(error, file=sys.stderr)
|
print(error, file=sys.stderr)
|
||||||
|
|
|
@ -32,9 +32,12 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
|
||||||
for section_name, section_config in source_config._asdict().items()
|
for section_name, section_config in source_config._asdict().items()
|
||||||
])
|
])
|
||||||
|
|
||||||
# Split space-seperated values into actual lists, and merge in excludes.
|
# Split space-seperated values into actual lists, make "repository" into a list, and merge in
|
||||||
destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
|
# excludes.
|
||||||
destination_config['location']['exclude_patterns'] = source_excludes
|
location = destination_config['location']
|
||||||
|
location['source_directories'] = source_config.location['source_directories'].split(' ')
|
||||||
|
location['repositories'] = [location.pop('repository')]
|
||||||
|
location['exclude_patterns'] = source_excludes
|
||||||
|
|
||||||
if source_config.consistency['checks']:
|
if source_config.consistency['checks']:
|
||||||
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
|
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
|
||||||
|
|
|
@ -25,11 +25,15 @@ map:
|
||||||
type: scalar
|
type: scalar
|
||||||
desc: Alternate Borg remote executable. Defaults to "borg".
|
desc: Alternate Borg remote executable. Defaults to "borg".
|
||||||
example: borg1
|
example: borg1
|
||||||
repository:
|
repositories:
|
||||||
required: True
|
required: True
|
||||||
type: scalar
|
seq:
|
||||||
desc: Path to local or remote repository (required).
|
- type: scalar
|
||||||
example: user@backupserver:sourcehostname.borg
|
desc: |
|
||||||
|
Paths to local or remote repositories (required). Multiple repositories are
|
||||||
|
backed up to in sequence.
|
||||||
|
example:
|
||||||
|
- user@backupserver:sourcehostname.borg
|
||||||
exclude_patterns:
|
exclude_patterns:
|
||||||
seq:
|
seq:
|
||||||
- type: scalar
|
- type: scalar
|
||||||
|
|
|
@ -35,7 +35,8 @@ def test_parse_configuration_transforms_file_into_mapping():
|
||||||
- /home
|
- /home
|
||||||
- /etc
|
- /etc
|
||||||
|
|
||||||
repository: hostname.borg
|
repositories:
|
||||||
|
- hostname.borg
|
||||||
|
|
||||||
retention:
|
retention:
|
||||||
keep_daily: 7
|
keep_daily: 7
|
||||||
|
@ -50,7 +51,7 @@ def test_parse_configuration_transforms_file_into_mapping():
|
||||||
result = module.parse_configuration('config.yaml', 'schema.yaml')
|
result = module.parse_configuration('config.yaml', 'schema.yaml')
|
||||||
|
|
||||||
assert result == {
|
assert result == {
|
||||||
'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
|
'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']},
|
||||||
'retention': {'keep_daily': 7},
|
'retention': {'keep_daily': 7},
|
||||||
'consistency': {'checks': ['repository', 'archives']},
|
'consistency': {'checks': ['repository', 'archives']},
|
||||||
}
|
}
|
||||||
|
@ -65,7 +66,8 @@ def test_parse_configuration_passes_through_quoted_punctuation():
|
||||||
source_directories:
|
source_directories:
|
||||||
- /home
|
- /home
|
||||||
|
|
||||||
repository: "{}.borg"
|
repositories:
|
||||||
|
- "{}.borg"
|
||||||
'''.format(escaped_punctuation)
|
'''.format(escaped_punctuation)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -74,7 +76,7 @@ def test_parse_configuration_passes_through_quoted_punctuation():
|
||||||
assert result == {
|
assert result == {
|
||||||
'location': {
|
'location': {
|
||||||
'source_directories': ['/home'],
|
'source_directories': ['/home'],
|
||||||
'repository': '{}.borg'.format(string.punctuation),
|
'repositories': ['{}.borg'.format(string.punctuation)],
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -105,7 +107,8 @@ def test_parse_configuration_raises_for_validation_error():
|
||||||
'''
|
'''
|
||||||
location:
|
location:
|
||||||
source_directories: yes
|
source_directories: yes
|
||||||
repository: hostname.borg
|
repositories:
|
||||||
|
- hostname.borg
|
||||||
'''
|
'''
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -28,7 +28,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
|
||||||
'location',
|
'location',
|
||||||
OrderedDict([
|
OrderedDict([
|
||||||
('source_directories', ['/home']),
|
('source_directories', ['/home']),
|
||||||
('repository', 'hostname.borg'),
|
('repositories', ['hostname.borg']),
|
||||||
('exclude_patterns', ['/var']),
|
('exclude_patterns', ['/var']),
|
||||||
]),
|
]),
|
||||||
),
|
),
|
||||||
|
@ -41,7 +41,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
|
||||||
def test_convert_legacy_parsed_config_splits_space_separated_values():
|
def test_convert_legacy_parsed_config_splits_space_separated_values():
|
||||||
flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
|
flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
|
||||||
source_config = Parsed_config(
|
source_config = Parsed_config(
|
||||||
location=OrderedDict([('source_directories', '/home /etc')]),
|
location=OrderedDict([('source_directories', '/home /etc'), ('repository', 'hostname.borg')]),
|
||||||
storage=OrderedDict(),
|
storage=OrderedDict(),
|
||||||
retention=OrderedDict(),
|
retention=OrderedDict(),
|
||||||
consistency=OrderedDict([('checks', 'repository archives')]),
|
consistency=OrderedDict([('checks', 'repository archives')]),
|
||||||
|
@ -56,6 +56,7 @@ def test_convert_legacy_parsed_config_splits_space_separated_values():
|
||||||
'location',
|
'location',
|
||||||
OrderedDict([
|
OrderedDict([
|
||||||
('source_directories', ['/home', '/etc']),
|
('source_directories', ['/home', '/etc']),
|
||||||
|
('repositories', ['hostname.borg']),
|
||||||
('exclude_patterns', ['/var']),
|
('exclude_patterns', ['/var']),
|
||||||
]),
|
]),
|
||||||
),
|
),
|
||||||
|
|
|
@ -77,11 +77,14 @@ def test_create_archive_should_call_borg_with_parameters():
|
||||||
insert_datetime_mock()
|
insert_datetime_mock()
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
exclude_patterns=None,
|
|
||||||
verbosity=None,
|
verbosity=None,
|
||||||
storage_config={},
|
|
||||||
source_directories=['foo', 'bar'],
|
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={},
|
||||||
command='borg',
|
command='borg',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -93,11 +96,14 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
|
||||||
insert_datetime_mock()
|
insert_datetime_mock()
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
exclude_patterns=['exclude'],
|
|
||||||
verbosity=None,
|
verbosity=None,
|
||||||
storage_config={},
|
|
||||||
source_directories=['foo', 'bar'],
|
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': ['exclude'],
|
||||||
|
},
|
||||||
|
storage_config={},
|
||||||
command='borg',
|
command='borg',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -109,11 +115,14 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter
|
||||||
insert_datetime_mock()
|
insert_datetime_mock()
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
exclude_patterns=None,
|
|
||||||
verbosity=VERBOSITY_SOME,
|
verbosity=VERBOSITY_SOME,
|
||||||
storage_config={},
|
|
||||||
source_directories=['foo', 'bar'],
|
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={},
|
||||||
command='borg',
|
command='borg',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -125,11 +134,14 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete
|
||||||
insert_datetime_mock()
|
insert_datetime_mock()
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
exclude_patterns=None,
|
|
||||||
verbosity=VERBOSITY_LOTS,
|
verbosity=VERBOSITY_LOTS,
|
||||||
storage_config={},
|
|
||||||
source_directories=['foo', 'bar'],
|
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={},
|
||||||
command='borg',
|
command='borg',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -141,11 +153,14 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param
|
||||||
insert_datetime_mock()
|
insert_datetime_mock()
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
exclude_patterns=None,
|
|
||||||
verbosity=None,
|
verbosity=None,
|
||||||
storage_config={'compression': 'rle'},
|
|
||||||
source_directories=['foo', 'bar'],
|
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={'compression': 'rle'},
|
||||||
command='borg',
|
command='borg',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -157,13 +172,16 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst
|
||||||
insert_datetime_mock()
|
insert_datetime_mock()
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
exclude_patterns=None,
|
|
||||||
verbosity=None,
|
verbosity=None,
|
||||||
storage_config={},
|
|
||||||
source_directories=['foo', 'bar'],
|
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'one_file_system': True,
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={},
|
||||||
command='borg',
|
command='borg',
|
||||||
one_file_system=True,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -174,13 +192,16 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param
|
||||||
insert_datetime_mock()
|
insert_datetime_mock()
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
exclude_patterns=None,
|
|
||||||
verbosity=None,
|
verbosity=None,
|
||||||
storage_config={},
|
|
||||||
source_directories=['foo', 'bar'],
|
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'remote_path': 'borg1',
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={},
|
||||||
command='borg',
|
command='borg',
|
||||||
remote_path='borg1',
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -191,11 +212,14 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
|
||||||
insert_datetime_mock()
|
insert_datetime_mock()
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
exclude_patterns=None,
|
|
||||||
verbosity=None,
|
verbosity=None,
|
||||||
storage_config={'umask': 740},
|
|
||||||
source_directories=['foo', 'bar'],
|
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={'umask': 740},
|
||||||
command='borg',
|
command='borg',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -208,11 +232,14 @@ def test_create_archive_with_source_directories_glob_expands():
|
||||||
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
|
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
exclude_patterns=None,
|
|
||||||
verbosity=None,
|
verbosity=None,
|
||||||
storage_config={},
|
|
||||||
source_directories=['foo*'],
|
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo*'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={},
|
||||||
command='borg',
|
command='borg',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -225,11 +252,14 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
|
||||||
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
|
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
exclude_patterns=None,
|
|
||||||
verbosity=None,
|
verbosity=None,
|
||||||
storage_config={},
|
|
||||||
source_directories=['foo*'],
|
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo*'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={},
|
||||||
command='borg',
|
command='borg',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -242,11 +272,14 @@ def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
|
||||||
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
|
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
exclude_patterns=None,
|
|
||||||
verbosity=None,
|
verbosity=None,
|
||||||
storage_config={},
|
|
||||||
source_directories=['foo*'],
|
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo*'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={},
|
||||||
command='borg',
|
command='borg',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue