From 6096a7181cef640da8278335f4e1430d446d3a70 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 27 Jan 2019 13:47:26 -0800 Subject: [PATCH] Leave exclude_patterns glob expansion to Borg, since doing it in borgmatic leads to confusing behavior (#132). --- NEWS | 2 + borgmatic/borg/create.py | 15 ++++- tests/unit/borg/test_create.py | 110 ++++++++++++++++++++++----------- 3 files changed, 89 insertions(+), 38 deletions(-) diff --git a/NEWS b/NEWS index 6fc3cb9..bdad672 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,7 @@ 1.2.15.dev0 * #127: Remove date echo from schema example, as it's not a substitute for real logging. + * #132: Leave exclude_patterns glob expansion to Borg, since doing it in borgmatic leads to + confusing behavior. * #136: Handle and format validation errors raised during argument parsing. * #139: Allow use of --stats flag when --create or --prune flags are implied. diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 70151e2..9fedbc2 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -32,6 +32,17 @@ def _expand_directories(directories): ) +def _expand_home_directories(directories): + ''' + Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing. + Return the results as a tuple. + ''' + if directories is None: + return () + + return tuple(os.path.expanduser(directory) for directory in directories) + + def _write_pattern_file(patterns=None): ''' Given a sequence of patterns, write them to a named temporary file and return it. Return None @@ -101,7 +112,9 @@ def create_archive( sources = _expand_directories(location_config['source_directories']) pattern_file = _write_pattern_file(location_config.get('patterns')) - exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns'))) + exclude_file = _write_pattern_file( + _expand_home_directories(location_config.get('exclude_patterns')) + ) checkpoint_interval = storage_config.get('checkpoint_interval', None) chunker_params = storage_config.get('chunker_params', None) compression = storage_config.get('compression', None) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 14ccbf7..8d69d3e 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -43,6 +43,21 @@ def test_expand_directories_considers_none_as_no_directories(): assert paths == () +def test_expand_home_directories_expands_tildes(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/bar').and_return('/foo/bar') + flexmock(module.os.path).should_receive('expanduser').with_args('baz').and_return('baz') + + paths = module._expand_home_directories(('~/bar', 'baz')) + + assert paths == ('/foo/bar', 'baz') + + +def test_expand_home_directories_considers_none_as_no_directories(): + paths = module._expand_home_directories(None) + + assert paths == () + + def test_write_pattern_file_does_not_raise(): temporary_file = flexmock(name='filename', write=lambda mode: None, flush=lambda: None) flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file) @@ -149,7 +164,8 @@ CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'fo def test_create_archive_calls_borg_with_parameters(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -169,7 +185,8 @@ def test_create_archive_calls_borg_with_parameters(): def test_create_archive_with_patterns_calls_borg_with_patterns(): pattern_flags = ('--patterns-from', 'patterns') - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return( flexmock(name='/tmp/patterns') ).and_return(None) @@ -191,9 +208,8 @@ def test_create_archive_with_patterns_calls_borg_with_patterns(): def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): exclude_flags = ('--exclude-from', 'excludes') - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return( - ('exclude',) - ) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',)) flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return( flexmock(name='/tmp/excludes') ) @@ -214,7 +230,8 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): def test_create_archive_with_log_info_calls_borg_with_info_parameter(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -235,7 +252,8 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -257,7 +275,8 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -277,9 +296,10 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_parameter(): - """ --dry-run and --stats are mutually exclusive, see: - https://borgbackup.readthedocs.io/en/stable/usage/create.html#description """ - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + # --dry-run and --stats are mutually exclusive, see: + # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -300,9 +320,10 @@ def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_param def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_parameter(): - """ --dry-run and --stats are mutually exclusive, see: - https://borgbackup.readthedocs.io/en/stable/usage/create.html#description """ - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + # --dry-run and --stats are mutually exclusive, see: + # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -325,7 +346,8 @@ def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_para def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -344,7 +366,8 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -363,7 +386,8 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param def test_create_archive_with_compression_calls_borg_with_compression_parameters(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -382,7 +406,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -401,7 +426,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -421,7 +447,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par def test_create_archive_with_read_special_calls_borg_with_read_special_parameter(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -441,7 +468,8 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parameter(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -461,7 +489,8 @@ def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parame def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_parameter(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -481,7 +510,8 @@ def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_paramete def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -501,7 +531,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( def test_create_archive_with_local_path_calls_borg_via_local_path(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -521,7 +552,8 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -541,7 +573,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( def test_create_archive_with_umask_calls_borg_with_umask_parameters(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -560,7 +593,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -579,7 +613,8 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): def test_create_archive_with_json_calls_borg_with_json_parameter(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -599,9 +634,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): def test_create_archive_with_source_directories_glob_expands(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return( - () - ) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -623,7 +657,8 @@ def test_create_archive_with_source_directories_glob_expands(): def test_create_archive_with_non_matching_source_directories_glob_passes_through(): - flexmock(module).should_receive('_expand_directories').and_return(('foo*',)).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo*',)) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -643,9 +678,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through def test_create_archive_with_glob_calls_borg_with_expanded_directories(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return( - () - ) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -666,7 +700,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -685,7 +720,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return([]) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(())