Fix for potential data loss (data not getting backed up) when borgmatic omitted configured source directories in certain situations (#333).

This commit is contained in:
Dan Helfman 2020-06-19 20:16:38 -07:00
parent 7fb4061759
commit 27cbe9dfc0
3 changed files with 97 additions and 29 deletions

4
NEWS
View file

@ -1,4 +1,8 @@
1.5.7.dev0 1.5.7.dev0
* #333: Fix for potential data loss (data not getting backed up) when borgmatic omitted configured
source directories in certain situations. Specifically, this occurred when two source directories
on different filesystems were related by parentage (e.g. "/foo" and "/foo/bar/baz") and the
one_file_system option was enabled.
* Update documentation code fragments theme to better match the rest of the page. * Update documentation code fragments theme to better match the rest of the page.
* Improve configuration reference documentation readability via more aggressive word-wrapping in * Improve configuration reference documentation readability via more aggressive word-wrapping in
configuration schema descriptions. configuration schema descriptions.

View file

@ -44,10 +44,24 @@ def _expand_home_directories(directories):
return tuple(os.path.expanduser(directory) for directory in directories) return tuple(os.path.expanduser(directory) for directory in directories)
def deduplicate_directories(directories): def map_directories_to_devices(directories): # pragma: no cover
''' '''
Given a sequence of directories, return them as a sorted tuple with all duplicate child Given a sequence of directories, return a map from directory to an identifier for the device on
directories removed. For instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',) which that directory resides. This is handy for determining whether two different directories
are on the same filesystem (have the same device identifier).
'''
return {directory: os.stat(directory).st_dev for directory in directories}
def deduplicate_directories(directory_devices):
'''
Given a map from directory to the identifier for the device on which that directory resides,
return the directories as a sorted tuple with all duplicate child directories removed. For
instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
The one exception to this rule is if two paths are on different filesystems (devices). In that
case, they won't get de-duplicated in case they both need to be passed to Borg (e.g. the
location.one_file_system option is true).
The idea is that if Borg is given a parent directory, then it doesn't also need to be given The idea is that if Borg is given a parent directory, then it doesn't also need to be given
child directories, because it will naturally spider the contents of the parent directory. And child directories, because it will naturally spider the contents of the parent directory. And
@ -56,19 +70,23 @@ def deduplicate_directories(directories):
Borg. Borg.
''' '''
deduplicated = set() deduplicated = set()
directories = sorted(directory_devices.keys())
for directory in sorted(directories): for directory in directories:
# If the directory is "/", that contains all child directories, so we can early out.
if directory == os.path.sep:
return (os.path.sep,)
# If no other directories are parents of current directory (even n levels up), then the
# current directory isn't a duplicate.
if not any(
pathlib.PurePath(other_directory) in pathlib.PurePath(directory).parents
for other_directory in directories
):
deduplicated.add(directory) deduplicated.add(directory)
parents = pathlib.PurePath(directory).parents
# If another directory in the given list is a parent of current directory (even n levels
# up) and both are on the same filesystem, then the current directory is a duplicate.
for other_directory in directories:
for parent in parents:
if (
pathlib.PurePath(other_directory) == parent
and directory_devices[other_directory] == directory_devices[directory]
):
if directory in deduplicated:
deduplicated.remove(directory)
break
return tuple(sorted(deduplicated)) return tuple(sorted(deduplicated))
@ -179,11 +197,13 @@ def create_archive(
create command while also triggering the given processes to produce output. create command while also triggering the given processes to produce output.
''' '''
sources = deduplicate_directories( sources = deduplicate_directories(
map_directories_to_devices(
_expand_directories( _expand_directories(
location_config['source_directories'] location_config['source_directories']
+ borgmatic_source_directories(location_config.get('borgmatic_source_directory')) + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
) )
) )
)
pattern_file = _write_pattern_file(location_config.get('patterns')) pattern_file = _write_pattern_file(location_config.get('patterns'))
exclude_file = _write_pattern_file( exclude_file = _write_pattern_file(

View file

@ -63,20 +63,25 @@ def test_expand_home_directories_considers_none_as_no_directories():
@pytest.mark.parametrize( @pytest.mark.parametrize(
'directories,expected_directories', 'directories,expected_directories',
( (
(('/', '/root'), ('/',)), ({'/': 1, '/root': 1}, ('/',)),
(('/', '/root/'), ('/',)), ({'/': 1, '/root/': 1}, ('/',)),
(('/root', '/'), ('/',)), ({'/': 1, '/root': 2}, ('/', '/root')),
(('/root', '/root/foo'), ('/root',)), ({'/root': 1, '/': 1}, ('/',)),
(('/root/', '/root/foo'), ('/root/',)), ({'/root': 1, '/root/foo': 1}, ('/root',)),
(('/root', '/root/foo/'), ('/root',)), ({'/root/': 1, '/root/foo': 1}, ('/root/',)),
(('/root/foo', '/root'), ('/root',)), ({'/root': 1, '/root/foo/': 1}, ('/root',)),
(('/root', '/etc', '/root/foo/bar'), ('/etc', '/root')), ({'/root': 1, '/root/foo': 2}, ('/root', '/root/foo')),
(('/root', '/root/foo', '/root/foo/bar'), ('/root',)), ({'/root/foo': 1, '/root': 1}, ('/root',)),
(('/dup', '/dup'), ('/dup',)), ({'/root': 1, '/etc': 1, '/root/foo/bar': 1}, ('/etc', '/root')),
(('/foo', '/bar'), ('/bar', '/foo')), ({'/root': 1, '/root/foo': 1, '/root/foo/bar': 1}, ('/root',)),
({'/dup': 1, '/dup': 1}, ('/dup',)),
({'/foo': 1, '/bar': 1}, ('/bar', '/foo')),
({'/foo': 1, '/bar': 2}, ('/bar', '/foo')),
), ),
) )
def test_deduplicate_directories_removes_child_paths(directories, expected_directories): def test_deduplicate_directories_removes_child_paths_on_the_same_filesystem(
directories, expected_directories
):
assert module.deduplicate_directories(directories) == expected_directories assert module.deduplicate_directories(directories) == expected_directories
@ -235,6 +240,7 @@ ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
def test_create_archive_calls_borg_with_parameters(): def test_create_archive_calls_borg_with_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -263,6 +269,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
pattern_flags = ('--patterns-from', 'patterns') pattern_flags = ('--patterns-from', 'patterns')
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return( flexmock(module).should_receive('_write_pattern_file').and_return(
@ -293,6 +300,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
exclude_flags = ('--exclude-from', 'excludes') exclude_flags = ('--exclude-from', 'excludes')
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',)) flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return( flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
@ -322,6 +330,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
def test_create_archive_with_log_info_calls_borg_with_info_parameter(): def test_create_archive_with_log_info_calls_borg_with_info_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -351,6 +360,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -381,6 +391,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -409,6 +420,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -438,6 +450,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -468,6 +481,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
# https://borgbackup.readthedocs.io/en/stable/usage/create.html#description # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -498,6 +512,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters(): def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -525,6 +540,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters(): def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -552,6 +568,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
def test_create_archive_with_compression_calls_borg_with_compression_parameters(): def test_create_archive_with_compression_calls_borg_with_compression_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -579,6 +596,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters(): def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -606,6 +624,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter(): def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -634,6 +653,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter(): def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -662,6 +682,7 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
def test_create_archive_with_read_special_calls_borg_with_read_special_parameter(): def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -691,6 +712,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name): def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -720,6 +742,7 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name): def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -748,6 +771,7 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(): def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -776,6 +800,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
def test_create_archive_with_local_path_calls_borg_via_local_path(): def test_create_archive_with_local_path_calls_borg_via_local_path():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -804,6 +829,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(): def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -832,6 +858,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
def test_create_archive_with_umask_calls_borg_with_umask_parameters(): def test_create_archive_with_umask_calls_borg_with_umask_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -859,6 +886,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -886,6 +914,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_output_log_level(): def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_output_log_level():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -914,6 +943,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_and_info_output_log_level(): def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_and_info_output_log_level():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -943,6 +973,7 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_output_log_level(): def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_output_log_level():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -971,6 +1002,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_and_info_output_log_level(): def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_and_info_output_log_level():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1000,6 +1032,7 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list(): def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1029,6 +1062,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
def test_create_archive_with_progress_calls_borg_with_progress_parameter(): def test_create_archive_with_progress_calls_borg_with_progress_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1058,6 +1092,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
processes = flexmock() processes = flexmock()
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1089,6 +1124,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
def test_create_archive_with_json_calls_borg_with_json_parameter(): def test_create_archive_with_json_calls_borg_with_json_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1119,6 +1155,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter(): def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1150,6 +1187,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
def test_create_archive_with_source_directories_glob_expands(): def test_create_archive_with_source_directories_glob_expands():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1178,6 +1216,7 @@ def test_create_archive_with_source_directories_glob_expands():
def test_create_archive_with_non_matching_source_directories_glob_passes_through(): def test_create_archive_with_non_matching_source_directories_glob_passes_through():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo*',)) flexmock(module).should_receive('deduplicate_directories').and_return(('foo*',))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1206,6 +1245,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
def test_create_archive_with_glob_calls_borg_with_expanded_directories(): def test_create_archive_with_glob_calls_borg_with_expanded_directories():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1233,6 +1273,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1260,6 +1301,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1287,6 +1329,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -1315,6 +1358,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes():
processes = flexmock() processes = flexmock()
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)