Pass extra options directly to particular Borg commands, handy for Borg options that borgmatic does not yet support natively (#235).

This commit is contained in:
Dan Helfman 2019-12-04 15:48:10 -08:00
parent 00f62ca023
commit 0c6c61a272
12 changed files with 130 additions and 14 deletions

5
NEWS
View file

@ -1,3 +1,8 @@
1.4.17
* #235: Pass extra options directly to particular Borg commands, handy for Borg options that
borgmatic does not yet support natively. Use "extra_borg_options" in the storage configuration
section.
1.4.16 1.4.16
* #256: Fix for "before_backup" hook not triggering an error when the command contains "borg" and * #256: Fix for "before_backup" hook not triggering an error when the command contains "borg" and
has an exit code of 1. has an exit code of 1.

View file

@ -103,6 +103,7 @@ def check_archives(
checks = _parse_checks(consistency_config, only_checks) checks = _parse_checks(consistency_config, only_checks)
check_last = consistency_config.get('check_last', None) check_last = consistency_config.get('check_last', None)
lock_wait = None lock_wait = None
extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))): if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))):
remote_path_flags = ('--remote-path', remote_path) if remote_path else () remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
@ -123,6 +124,7 @@ def check_archives(
+ remote_path_flags + remote_path_flags
+ lock_wait_flags + lock_wait_flags
+ verbosity_flags + verbosity_flags
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+ (repository,) + (repository,)
) )

View file

@ -150,6 +150,7 @@ def create_archive(
files_cache = location_config.get('files_cache') files_cache = location_config.get('files_cache')
default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
archive_name_format = storage_config.get('archive_name_format', default_archive_name_format) archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
full_command = ( full_command = (
(local_path, 'create') (local_path, 'create')
@ -185,6 +186,7 @@ def create_archive(
+ (('--dry-run',) if dry_run else ()) + (('--dry-run',) if dry_run else ())
+ (('--progress',) if progress else ()) + (('--progress',) if progress else ())
+ (('--json',) if json else ()) + (('--json',) if json else ())
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+ ( + (
'{repository}::{archive_name_format}'.format( '{repository}::{archive_name_format}'.format(
repository=repository, archive_name_format=archive_name_format repository=repository, archive_name_format=archive_name_format

View file

@ -11,6 +11,7 @@ INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
def initialize_repository( def initialize_repository(
repository, repository,
storage_config,
encryption_mode, encryption_mode,
append_only=None, append_only=None,
storage_quota=None, storage_quota=None,
@ -18,9 +19,9 @@ def initialize_repository(
remote_path=None, remote_path=None,
): ):
''' '''
Given a local or remote repository path, a Borg encryption mode, whether the repository should Given a local or remote repository path, a storage configuration dict, a Borg encryption mode,
be append-only, and the storage quota to use, initialize the repository. If the repository whether the repository should be append-only, and the storage quota to use, initialize the
already exists, then log and skip initialization. repository. If the repository already exists, then log and skip initialization.
''' '''
info_command = (local_path, 'info', repository) info_command = (local_path, 'info', repository)
logger.debug(' '.join(info_command)) logger.debug(' '.join(info_command))
@ -33,6 +34,8 @@ def initialize_repository(
if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE: if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
raise raise
extra_borg_options = storage_config.get('extra_borg_options', {}).get('init', '')
init_command = ( init_command = (
(local_path, 'init') (local_path, 'init')
+ (('--encryption', encryption_mode) if encryption_mode else ()) + (('--encryption', encryption_mode) if encryption_mode else ())
@ -41,6 +44,7 @@ def initialize_repository(
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+ (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
+ (('--remote-path', remote_path) if remote_path else ()) + (('--remote-path', remote_path) if remote_path else ())
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+ (repository,) + (repository,)
) )

View file

@ -49,6 +49,7 @@ def prune_archives(
''' '''
umask = storage_config.get('umask', None) umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None) lock_wait = storage_config.get('lock_wait', None)
extra_borg_options = storage_config.get('extra_borg_options', {}).get('prune', '')
full_command = ( full_command = (
(local_path, 'prune') (local_path, 'prune')
@ -61,6 +62,7 @@ def prune_archives(
+ (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+ (('--dry-run',) if dry_run else ()) + (('--dry-run',) if dry_run else ())
+ (('--stats',) if stats else ()) + (('--stats',) if stats else ())
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+ (repository,) + (repository,)
) )

View file

@ -189,6 +189,7 @@ def run_actions(
logger.info('{}: Initializing repository'.format(repository)) logger.info('{}: Initializing repository'.format(repository))
borg_init.initialize_repository( borg_init.initialize_repository(
repository, repository,
storage,
arguments['init'].encryption_mode, arguments['init'].encryption_mode,
arguments['init'].append_only, arguments['init'].append_only,
arguments['init'].storage_quota, arguments['init'].storage_quota,

View file

@ -245,6 +245,29 @@ map:
Bypass Borg error about a previously unknown unencrypted repository. Defaults to Bypass Borg error about a previously unknown unencrypted repository. Defaults to
false. false.
example: true example: true
extra_borg_options:
map:
init:
type: str
desc: Extra command-line options to pass to "borg init".
example: "--make-parent-dirs"
prune:
type: str
desc: Extra command-line options to pass to "borg prune".
example: "--save-space"
create:
type: str
desc: Extra command-line options to pass to "borg create".
example: "--no-files-cache"
check:
type: str
desc: Extra command-line options to pass to "borg check".
example: "--save-space"
desc: |
Additional options to pass directly to particular Borg commands, handy for Borg
options that borgmatic does not yet support natively. Note that borgmatic does
not perform any validation on these options. Running borgmatic with
"--verbosity 2" shows the exact Borg command-line invocation.
retention: retention:
desc: | desc: |
Retention policy for how many backups to keep in each category. See Retention policy for how many backups to keep in each category. See

View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup from setuptools import find_packages, setup
VERSION = '1.4.16' VERSION = '1.4.17'
setup( setup(

View file

@ -296,3 +296,17 @@ def test_check_archives_with_retention_prefix():
module.check_archives( module.check_archives(
repository='repo', storage_config={}, consistency_config=consistency_config repository='repo', storage_config={}, consistency_config=consistency_config
) )
def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options():
checks = ('repository',)
consistency_config = {'check_last': None}
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').and_return(())
insert_execute_command_mock(('borg', 'check', '--extra', '--options', 'repo'))
module.check_archives(
repository='repo',
storage_config={'extra_borg_options': {'check': '--extra --options'}},
consistency_config=consistency_config,
)

View file

@ -1092,3 +1092,28 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
}, },
storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'},
) )
def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
flexmock(module).should_receive('execute_command').with_args(
('borg', 'create', '--extra', '--options') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
error_on_warnings=False,
)
module.create_archive(
dry_run=False,
repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={'extra_borg_options': {'create': '--extra --options'}},
)

View file

@ -32,7 +32,7 @@ def test_initialize_repository_calls_borg_with_parameters():
insert_info_command_not_found_mock() insert_info_command_not_found_mock()
insert_init_command_mock(INIT_COMMAND + ('repo',)) insert_init_command_mock(INIT_COMMAND + ('repo',))
module.initialize_repository(repository='repo', encryption_mode='repokey') module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
def test_initialize_repository_raises_for_borg_init_error(): def test_initialize_repository_raises_for_borg_init_error():
@ -42,14 +42,16 @@ def test_initialize_repository_raises_for_borg_init_error():
) )
with pytest.raises(subprocess.CalledProcessError): with pytest.raises(subprocess.CalledProcessError):
module.initialize_repository(repository='repo', encryption_mode='repokey') module.initialize_repository(
repository='repo', storage_config={}, encryption_mode='repokey'
)
def test_initialize_repository_skips_initialization_when_repository_already_exists(): def test_initialize_repository_skips_initialization_when_repository_already_exists():
insert_info_command_found_mock() insert_info_command_found_mock()
flexmock(module).should_receive('execute_command_without_capture').never() flexmock(module).should_receive('execute_command_without_capture').never()
module.initialize_repository(repository='repo', encryption_mode='repokey') module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
def test_initialize_repository_raises_for_unknown_info_command_error(): def test_initialize_repository_raises_for_unknown_info_command_error():
@ -58,21 +60,27 @@ def test_initialize_repository_raises_for_unknown_info_command_error():
) )
with pytest.raises(subprocess.CalledProcessError): with pytest.raises(subprocess.CalledProcessError):
module.initialize_repository(repository='repo', encryption_mode='repokey') module.initialize_repository(
repository='repo', storage_config={}, encryption_mode='repokey'
)
def test_initialize_repository_with_append_only_calls_borg_with_append_only_parameter(): def test_initialize_repository_with_append_only_calls_borg_with_append_only_parameter():
insert_info_command_not_found_mock() insert_info_command_not_found_mock()
insert_init_command_mock(INIT_COMMAND + ('--append-only', 'repo')) insert_init_command_mock(INIT_COMMAND + ('--append-only', 'repo'))
module.initialize_repository(repository='repo', encryption_mode='repokey', append_only=True) module.initialize_repository(
repository='repo', storage_config={}, encryption_mode='repokey', append_only=True
)
def test_initialize_repository_with_storage_quota_calls_borg_with_storage_quota_parameter(): def test_initialize_repository_with_storage_quota_calls_borg_with_storage_quota_parameter():
insert_info_command_not_found_mock() insert_info_command_not_found_mock()
insert_init_command_mock(INIT_COMMAND + ('--storage-quota', '5G', 'repo')) insert_init_command_mock(INIT_COMMAND + ('--storage-quota', '5G', 'repo'))
module.initialize_repository(repository='repo', encryption_mode='repokey', storage_quota='5G') module.initialize_repository(
repository='repo', storage_config={}, encryption_mode='repokey', storage_quota='5G'
)
def test_initialize_repository_with_log_info_calls_borg_with_info_parameter(): def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
@ -80,7 +88,7 @@ def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
insert_init_command_mock(INIT_COMMAND + ('--info', 'repo')) insert_init_command_mock(INIT_COMMAND + ('--info', 'repo'))
insert_logging_mock(logging.INFO) insert_logging_mock(logging.INFO)
module.initialize_repository(repository='repo', encryption_mode='repokey') module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter(): def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
@ -88,18 +96,33 @@ def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
insert_init_command_mock(INIT_COMMAND + ('--debug', 'repo')) insert_init_command_mock(INIT_COMMAND + ('--debug', 'repo'))
insert_logging_mock(logging.DEBUG) insert_logging_mock(logging.DEBUG)
module.initialize_repository(repository='repo', encryption_mode='repokey') module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
def test_initialize_repository_with_local_path_calls_borg_via_local_path(): def test_initialize_repository_with_local_path_calls_borg_via_local_path():
insert_info_command_not_found_mock() insert_info_command_not_found_mock()
insert_init_command_mock(('borg1',) + INIT_COMMAND[1:] + ('repo',)) insert_init_command_mock(('borg1',) + INIT_COMMAND[1:] + ('repo',))
module.initialize_repository(repository='repo', encryption_mode='repokey', local_path='borg1') module.initialize_repository(
repository='repo', storage_config={}, encryption_mode='repokey', local_path='borg1'
)
def test_initialize_repository_with_remote_path_calls_borg_with_remote_path_parameter(): def test_initialize_repository_with_remote_path_calls_borg_with_remote_path_parameter():
insert_info_command_not_found_mock() insert_info_command_not_found_mock()
insert_init_command_mock(INIT_COMMAND + ('--remote-path', 'borg1', 'repo')) insert_init_command_mock(INIT_COMMAND + ('--remote-path', 'borg1', 'repo'))
module.initialize_repository(repository='repo', encryption_mode='repokey', remote_path='borg1') module.initialize_repository(
repository='repo', storage_config={}, encryption_mode='repokey', remote_path='borg1'
)
def test_initialize_repository_with_extra_borg_options_calls_borg_with_extra_options():
insert_info_command_not_found_mock()
insert_init_command_mock(INIT_COMMAND + ('--extra', '--options', 'repo'))
module.initialize_repository(
repository='repo',
storage_config={'extra_borg_options': {'init': '--extra --options'}},
encryption_mode='repokey',
)

View file

@ -188,3 +188,18 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
storage_config=storage_config, storage_config=storage_config,
retention_config=retention_config, retention_config=retention_config,
) )
def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options():
retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS
)
insert_execute_command_mock(PRUNE_COMMAND + ('--extra', '--options', 'repo'), logging.INFO)
module.prune_archives(
dry_run=False,
repository='repo',
storage_config={'extra_borg_options': {'prune': '--extra --options'}},
retention_config=retention_config,
)