From 331adca23e49fa40e71efa2602b319334aef9307 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 10 Jun 2016 13:31:37 -0700 Subject: [PATCH] #19: Support for Borg's --remote-path option to use an alternate Borg executable. --- NEWS | 5 ++ borgmatic/borg.py | 15 ++-- borgmatic/command.py | 5 +- borgmatic/config.py | 1 + borgmatic/tests/unit/test_borg.py | 144 +++++++++++++++++++++--------- sample/config | 10 ++- setup.py | 2 +- 7 files changed, 127 insertions(+), 55 deletions(-) diff --git a/NEWS b/NEWS index 7b7dccb..af155a9 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.0.1 + + * #19: Support for Borg's --remote-path option to use an alternate Borg + executable. See sample/config. + 1.0.0 * Attic is no longer supported, as there hasn't been any recent development on diff --git a/borgmatic/borg.py b/borgmatic/borg.py index adde7cf..27c570a 100644 --- a/borgmatic/borg.py +++ b/borgmatic/borg.py @@ -24,7 +24,7 @@ def initialize(storage_config, command=COMMAND): def create_archive( excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND, - one_file_system=None + one_file_system=None, remote_path=None, ): ''' Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated @@ -39,6 +39,7 @@ def create_archive( umask = storage_config.get('umask', None) umask_flags = ('--umask', str(umask)) if umask else () one_file_system_flags = ('--one-file-system',) if one_file_system else () + remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { VERBOSITY_SOME: ('--stats',), VERBOSITY_LOTS: ('--verbose', '--stats'), @@ -52,7 +53,7 @@ def create_archive( timestamp=datetime.now().isoformat(), ), ) + sources + exclude_flags + compression_flags + one_file_system_flags + \ - umask_flags + verbosity_flags + remote_path_flags + umask_flags + verbosity_flags subprocess.check_call(full_command) @@ -79,12 +80,13 @@ def _make_prune_flags(retention_config): ) -def prune_archives(verbosity, repository, retention_config, command=COMMAND): +def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None): ''' Given a verbosity flag, a local or remote repository path, a retention config dict, and a command to run, prune attic archives according the the retention policy specified in that configuration. ''' + remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { VERBOSITY_SOME: ('--stats',), VERBOSITY_LOTS: ('--verbose', '--stats'), @@ -97,7 +99,7 @@ def prune_archives(verbosity, repository, retention_config, command=COMMAND): element for pair in _make_prune_flags(retention_config) for element in pair - ) + verbosity_flags + ) + remote_path_flags + verbosity_flags subprocess.check_call(full_command) @@ -155,7 +157,7 @@ def _make_check_flags(checks, check_last=None): ) + last_flag -def check_archives(verbosity, repository, consistency_config, command=COMMAND): +def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None): ''' Given a verbosity flag, a local or remote repository path, a consistency config dict, and a command to run, check the contained attic archives for consistency. @@ -167,6 +169,7 @@ def check_archives(verbosity, repository, consistency_config, command=COMMAND): if not checks: return + remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { VERBOSITY_SOME: ('--verbose',), VERBOSITY_LOTS: ('--verbose',), @@ -175,7 +178,7 @@ def check_archives(verbosity, repository, consistency_config, command=COMMAND): full_command = ( command, 'check', repository, - ) + _make_check_flags(checks, check_last) + verbosity_flags + ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags # The check command spews to stdout/stderr even without the verbose flag. Suppress it. stdout = None if verbosity_flags else open(os.devnull, 'w') diff --git a/borgmatic/command.py b/borgmatic/command.py index 337fc3f..bf6713d 100644 --- a/borgmatic/command.py +++ b/borgmatic/command.py @@ -45,13 +45,14 @@ def main(): args = parse_arguments(*sys.argv[1:]) config = parse_configuration(args.config_filename, CONFIG_FORMAT) repository = config.location['repository'] + remote_path = config.location['remote_path'] borg.initialize(config.storage) borg.create_archive( args.excludes_filename, args.verbosity, config.storage, **config.location ) - borg.prune_archives(args.verbosity, repository, config.retention) - borg.check_archives(args.verbosity, repository, config.consistency) + borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path) + borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path) except (ValueError, IOError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/borgmatic/config.py b/borgmatic/config.py index d7a9a08..36cb4f4 100644 --- a/borgmatic/config.py +++ b/borgmatic/config.py @@ -26,6 +26,7 @@ CONFIG_FORMAT = ( ( option('source_directories'), option('one_file_system', value_type=bool, required=False), + option('remote_path', required=False), option('repository'), ), ), diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 1c032bb..0511357 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -14,8 +14,8 @@ def test_initialize_with_passphrase_should_set_environment(): try: os.environ = {} - module.initialize({'encryption_passphrase': 'pass'}, command='attic') - assert os.environ.get('ATTIC_PASSPHRASE') == 'pass' + module.initialize({'encryption_passphrase': 'pass'}, command='borg') + assert os.environ.get('BORG_PASSPHRASE') == 'pass' finally: os.environ = orig_environ @@ -25,8 +25,8 @@ def test_initialize_without_passphrase_should_not_set_environment(): try: os.environ = {} - module.initialize({}, command='attic') - assert os.environ.get('ATTIC_PASSPHRASE') == None + module.initialize({}, command='borg') + assert os.environ.get('BORG_PASSPHRASE') == None finally: os.environ = orig_environ @@ -53,11 +53,11 @@ def insert_datetime_mock(): ).mock -CREATE_COMMAND_WITHOUT_EXCLUDES = ('attic', 'create', 'repo::host-now', 'foo', 'bar') +CREATE_COMMAND_WITHOUT_EXCLUDES = ('borg', 'create', 'repo::host-now', 'foo', 'bar') CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes') -def test_create_archive_should_call_attic_with_parameters(): +def test_create_archive_should_call_borg_with_parameters(): insert_subprocess_mock(CREATE_COMMAND) insert_platform_mock() insert_datetime_mock() @@ -68,7 +68,7 @@ def test_create_archive_should_call_attic_with_parameters(): storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) @@ -83,11 +83,11 @@ def test_create_archive_with_two_spaces_in_source_directories(): storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_none_excludes_filename_should_call_attic_without_excludes(): +def test_create_archive_with_none_excludes_filename_should_call_borg_without_excludes(): insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES) insert_platform_mock() insert_datetime_mock() @@ -98,11 +98,11 @@ def test_create_archive_with_none_excludes_filename_should_call_attic_without_ex storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter(): +def test_create_archive_with_verbosity_some_should_call_borg_with_stats_parameter(): insert_subprocess_mock(CREATE_COMMAND + ('--stats',)) insert_platform_mock() insert_datetime_mock() @@ -113,11 +113,11 @@ def test_create_archive_with_verbosity_some_should_call_attic_with_stats_paramet storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_parameter(): +def test_create_archive_with_verbosity_lots_should_call_borg_with_verbose_parameter(): insert_subprocess_mock(CREATE_COMMAND + ('--verbose', '--stats')) insert_platform_mock() insert_datetime_mock() @@ -128,11 +128,11 @@ def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_param storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_compression_should_call_attic_with_compression_parameters(): +def test_create_archive_with_compression_should_call_borg_with_compression_parameters(): insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) insert_platform_mock() insert_datetime_mock() @@ -143,11 +143,11 @@ def test_create_archive_with_compression_should_call_attic_with_compression_para storage_config={'compression': 'rle'}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_one_file_system_should_call_attic_with_one_file_system_parameters(): +def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters(): insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) insert_platform_mock() insert_datetime_mock() @@ -158,12 +158,28 @@ def test_create_archive_with_one_file_system_should_call_attic_with_one_file_sys storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', one_file_system=True, ) -def test_create_archive_with_umask_should_call_attic_with_umask_parameters(): +def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): + insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1')) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + excludes_filename='excludes', + verbosity=None, + storage_config={}, + source_directories='foo bar', + repository='repo', + command='borg', + remote_path='borg1', + ) + + +def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) insert_platform_mock() insert_datetime_mock() @@ -174,12 +190,12 @@ def test_create_archive_with_umask_should_call_attic_with_umask_parameters(): storage_config={'umask': 740}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) def test_create_archive_with_source_directories_glob_expands(): - insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food')) + insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) @@ -190,12 +206,12 @@ def test_create_archive_with_source_directories_glob_expands(): storage_config={}, source_directories='foo*', repository='repo', - command='attic', + command='borg', ) def test_create_archive_with_non_matching_source_directories_glob_passes_through(): - insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo*')) + insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*')) insert_platform_mock() insert_datetime_mock() flexmock(module).should_receive('glob').with_args('foo*').and_return([]) @@ -206,12 +222,12 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through storage_config={}, source_directories='foo*', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_glob_should_call_attic_with_expanded_directories(): - insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food')) +def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): + insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) @@ -222,7 +238,7 @@ def test_create_archive_with_glob_should_call_attic_with_expanded_directories(): storage_config={}, source_directories='foo*', repository='repo', - command='attic', + command='borg', ) @@ -248,11 +264,11 @@ def test_make_prune_flags_should_return_flags_from_config(): PRUNE_COMMAND = ( - 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', + 'borg', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', ) -def test_prune_archives_should_call_attic_with_parameters(): +def test_prune_archives_should_call_borg_with_parameters(): retention_config = flexmock() flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, @@ -263,11 +279,11 @@ def test_prune_archives_should_call_attic_with_parameters(): verbosity=None, repository='repo', retention_config=retention_config, - command='attic', + command='borg', ) -def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_parameter(): +def test_prune_archives_with_verbosity_some_should_call_borg_with_stats_parameter(): retention_config = flexmock() flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, @@ -278,11 +294,11 @@ def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_paramet repository='repo', verbosity=VERBOSITY_SOME, retention_config=retention_config, - command='attic', + command='borg', ) -def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter(): +def test_prune_archives_with_verbosity_lots_should_call_borg_with_verbose_parameter(): retention_config = flexmock() flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, @@ -293,7 +309,22 @@ def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_param repository='repo', verbosity=VERBOSITY_LOTS, retention_config=retention_config, - command='attic', + command='borg', + ) + +def test_prune_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): + retention_config = flexmock() + flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( + BASE_PRUNE_FLAGS, + ) + insert_subprocess_mock(PRUNE_COMMAND + ('--remote-path', 'borg1')) + + module.prune_archives( + verbosity=None, + repository='repo', + retention_config=retention_config, + command='borg', + remote_path='borg1', ) @@ -345,7 +376,7 @@ def test_make_check_flags_with_last_returns_last_flag(): assert flags == ('--last', 3) -def test_check_archives_should_call_attic_with_parameters(): +def test_check_archives_should_call_borg_with_parameters(): checks = flexmock() check_last = flexmock() consistency_config = flexmock().should_receive('get').and_return(check_last).mock @@ -353,7 +384,7 @@ def test_check_archives_should_call_attic_with_parameters(): flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) stdout = flexmock() insert_subprocess_mock( - ('attic', 'check', 'repo'), + ('borg', 'check', 'repo'), stdout=stdout, stderr=STDOUT, ) insert_platform_mock() @@ -365,16 +396,16 @@ def test_check_archives_should_call_attic_with_parameters(): verbosity=None, repository='repo', consistency_config=consistency_config, - command='attic', + command='borg', ) -def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter(): +def test_check_archives_with_verbosity_some_should_call_borg_with_verbose_parameter(): consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( - ('attic', 'check', 'repo', '--verbose'), + ('borg', 'check', 'repo', '--verbose'), stdout=None, stderr=STDOUT, ) insert_platform_mock() @@ -384,16 +415,16 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param verbosity=VERBOSITY_SOME, repository='repo', consistency_config=consistency_config, - command='attic', + command='borg', ) -def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter(): +def test_check_archives_with_verbosity_lots_should_call_borg_with_verbose_parameter(): consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( - ('attic', 'check', 'repo', '--verbose'), + ('borg', 'check', 'repo', '--verbose'), stdout=None, stderr=STDOUT, ) insert_platform_mock() @@ -403,7 +434,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param verbosity=VERBOSITY_LOTS, repository='repo', consistency_config=consistency_config, - command='attic', + command='borg', ) @@ -416,5 +447,30 @@ def test_check_archives_without_any_checks_should_bail(): verbosity=None, repository='repo', consistency_config=consistency_config, - command='attic', + command='borg', + ) + + +def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters(): + checks = flexmock() + check_last = flexmock() + consistency_config = flexmock().should_receive('get').and_return(check_last).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) + stdout = flexmock() + insert_subprocess_mock( + ('borg', 'check', 'repo', '--remote-path', 'borg1'), + stdout=stdout, stderr=STDOUT, + ) + insert_platform_mock() + insert_datetime_mock() + builtins_mock().should_receive('open').and_return(stdout) + flexmock(module.os).should_receive('devnull') + + module.check_archives( + verbosity=None, + repository='repo', + consistency_config=consistency_config, + command='borg', + remote_path='borg1', ) diff --git a/sample/config b/sample/config index acea8d7..f081e4c 100644 --- a/sample/config +++ b/sample/config @@ -3,10 +3,12 @@ # Globs are expanded. source_directories: /home /etc /var/log/syslog* -# For Borg only, you can specify to stay in same file system (do not cross -# mount points). +# Stay in same file system (do not cross mount points). #one_file_system: True +# Alternate Borg remote executable (defaults to "borg"): +#remote_path: borg1 + # Path to local or remote repository. repository: user@backupserver:sourcehostname.borg @@ -14,10 +16,12 @@ repository: user@backupserver:sourcehostname.borg # Passphrase to unlock the encryption key with. Only use on repositories that # were initialized with passphrase/repokey encryption. #encryption_passphrase: foo + # Type of compression to use when creating archives. See # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create # for details. Defaults to no compression. #compression: lz4 + # Umask to be used for borg create. #umask: 0740 @@ -30,6 +34,7 @@ keep_daily: 7 keep_weekly: 4 keep_monthly: 6 keep_yearly: 1 + #prefix: sourcehostname [consistency] @@ -38,5 +43,6 @@ keep_yearly: 1 # checks. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check # for details. checks: repository archives + # Restrict the number of checked archives to the last n. #check_last: 3 diff --git a/setup.py b/setup.py index 19c1c60..91d6f38 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.0.0' +VERSION = '1.0.1' setup(