From b8f6bab12d4c8c5f04954b2352c32f9c11495778 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 14 Jan 2018 15:52:19 -0800 Subject: [PATCH] 49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed includes/excludes. --- NEWS | 4 +- borgmatic/borg/create.py | 48 +++++++--- borgmatic/config/schema.yaml | 24 ++++- borgmatic/tests/unit/borg/test_create.py | 117 ++++++++++++++++++----- 4 files changed, 152 insertions(+), 41 deletions(-) diff --git a/NEWS b/NEWS index 9ba7599..0dad115 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ 1.1.13.dev0 * #54: Fix for incorrect consistency check flags passed to Borg when all three checks ("repository", "archives", and "extract") are specified in borgmatic configuration. + * #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed + includes/excludes. * Moved issue tracker from Taiga to integrated Gitea tracker at https://projects.torsion.org/witten/borgmatic/issues @@ -19,7 +21,7 @@ shuts down if borgmatic is terminated (e.g. due to a system suspend). * #30: Support for using tilde in repository paths to reference home directory. * #43: Support for Borg --files-cache option for setting the files cache operation mode. - * #45: Support for Borg --remote-ratelimit for limiting upload rate. + * #45: Support for Borg --remote-ratelimit option for limiting upload rate. * Log invoked Borg commands when at highest verbosity level. 1.1.9 diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 755aff6..2285a45 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -31,28 +31,45 @@ def _expand_directory(directory): return glob.glob(expanded_directory) or [expanded_directory] -def _write_exclude_file(exclude_patterns=None): +def _write_pattern_file(patterns=None): ''' - Given a sequence of exclude patterns, write them to a named temporary file and return it. Return - None if no patterns are provided. + Given a sequence of patterns, write them to a named temporary file and return it. Return None + if no patterns are provided. ''' - if not exclude_patterns: + if not patterns: return None - exclude_file = tempfile.NamedTemporaryFile('w') - exclude_file.write('\n'.join(exclude_patterns)) - exclude_file.flush() + pattern_file = tempfile.NamedTemporaryFile('w') + pattern_file.write('\n'.join(patterns)) + pattern_file.flush() - return exclude_file + return pattern_file -def _make_exclude_flags(location_config, exclude_patterns_filename=None): +def _make_pattern_flags(location_config, pattern_filename=None): + ''' + Given a location config dict with a potential pattern_from option, and a filename containing any + additional patterns, return the corresponding Borg flags for those files as a tuple. + ''' + pattern_filenames = tuple(location_config.get('patterns_from') or ()) + ( + (pattern_filename,) if pattern_filename else () + ) + + return tuple( + itertools.chain.from_iterable( + ('--pattern-from', pattern_filename) + for pattern_filename in pattern_filenames + ) + ) + + +def _make_exclude_flags(location_config, exclude_filename=None): ''' Given a location config dict with various exclude options, and a filename containing any exclude patterns, return the corresponding Borg flags as a tuple. ''' exclude_filenames = tuple(location_config.get('exclude_from') or ()) + ( - (exclude_patterns_filename,) if exclude_patterns_filename else () + (exclude_filename,) if exclude_filename else () ) exclude_from_flags = tuple( itertools.chain.from_iterable( @@ -81,10 +98,15 @@ def create_archive( ) ) - exclude_patterns_file = _write_exclude_file(location_config.get('exclude_patterns')) + pattern_file = _write_pattern_file(location_config.get('patterns')) + pattern_flags = _make_pattern_flags( + location_config, + pattern_file.name if pattern_file else None, + ) + exclude_file = _write_pattern_file(location_config.get('exclude_patterns')) exclude_flags = _make_exclude_flags( location_config, - exclude_patterns_file.name if exclude_patterns_file else None, + exclude_file.name if exclude_file else None, ) compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () @@ -110,7 +132,7 @@ def create_archive( repository=repository, archive_name_format=archive_name_format, ), - ) + sources + exclude_flags + compression_flags + remote_rate_limit_flags + \ + ) + sources + pattern_flags + exclude_flags + compression_flags + remote_rate_limit_flags + \ one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \ verbosity_flags diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index fadbc4b..d05c9b3 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -42,6 +42,28 @@ map: repositories are backed up to in sequence. example: - user@backupserver:sourcehostname.borg + patterns: + seq: + - type: scalar + desc: | + Any paths matching these patterns are included/excluded from backups. Globs are + expanded. Note that Borg considers this option experimental. See the output of + "borg help patterns" for more details. Quoting any value if it contains leading + punctuation, so it parses correctly. + example: + - 'R /' + - '- /home/*/.cache' + - '+ /home/susan' + - '- /home/*' + patterns_from: + seq: + - type: scalar + desc: | + Read include/exclude patterns from one or more separate named files, one pattern + per line. Note that Borg considers this option experimental. See the output of + "borg help patterns" for more details. + example: + - /etc/borgmatic/patterns exclude_patterns: seq: - type: scalar @@ -57,7 +79,7 @@ map: - type: scalar desc: | Read exclude patterns from one or more separate named files, one pattern per - line. + line. See the output of "borg help patterns" for more details. example: - /etc/borgmatic/excludes exclude_caches: diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py index 30fb3ec..8bca44c 100644 --- a/borgmatic/tests/unit/borg/test_create.py +++ b/borgmatic/tests/unit/borg/test_create.py @@ -58,7 +58,7 @@ def test_expand_directory_with_glob_expands(): assert paths == ['foo', 'food'] -def test_write_exclude_file_does_not_raise(): +def test_write_pattern_file_does_not_raise(): temporary_file = flexmock( name='filename', write=lambda mode: None, @@ -66,11 +66,11 @@ def test_write_exclude_file_does_not_raise(): ) flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file) - module._write_exclude_file(['exclude']) + module._write_pattern_file(['exclude']) -def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise(): - module._write_exclude_file([]) +def test_write_pattern_file_with_empty_exclude_patterns_does_not_raise(): + module._write_pattern_file([]) def insert_subprocess_mock(check_call_command, **kwargs): @@ -78,17 +78,50 @@ def insert_subprocess_mock(check_call_command, **kwargs): subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() +def test_make_pattern_flags_includes_pattern_filename_when_given(): + pattern_flags = module._make_pattern_flags( + location_config={'patterns': ['R /', '- /var']}, + pattern_filename='/tmp/patterns', + ) + + assert pattern_flags == ('--pattern-from', '/tmp/patterns') + + +def test_make_pattern_flags_includes_patterns_from_filenames_when_in_config(): + pattern_flags = module._make_pattern_flags( + location_config={'patterns_from': ['patterns', 'other']}, + ) + + assert pattern_flags == ('--pattern-from', 'patterns', '--pattern-from', 'other') + + +def test_make_pattern_flags_includes_both_filenames_when_patterns_given_and_patterns_from_in_config(): + pattern_flags = module._make_pattern_flags( + location_config={'patterns_from': ['patterns']}, + pattern_filename='/tmp/patterns', + ) + + assert pattern_flags == ('--pattern-from', 'patterns', '--pattern-from', '/tmp/patterns') + + +def test_make_pattern_flags_considers_none_patterns_from_filenames_as_empty(): + pattern_flags = module._make_pattern_flags( + location_config={'patterns_from': None}, + ) + + assert pattern_flags == () + + def test_make_exclude_flags_includes_exclude_patterns_filename_when_given(): exclude_flags = module._make_exclude_flags( location_config={'exclude_patterns': ['*.pyc', '/var']}, - exclude_patterns_filename='/tmp/excludes', + exclude_filename='/tmp/excludes', ) assert exclude_flags == ('--exclude-from', '/tmp/excludes') def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config(): - flexmock(module).should_receive('_write_exclude_file').and_return(None) exclude_flags = module._make_exclude_flags( location_config={'exclude_from': ['excludes', 'other']}, @@ -98,19 +131,15 @@ def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config(): def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config(): - flexmock(module).should_receive('_write_exclude_file').and_return(None) - exclude_flags = module._make_exclude_flags( location_config={'exclude_from': ['excludes']}, - exclude_patterns_filename='/tmp/excludes', + exclude_filename='/tmp/excludes', ) assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes') def test_make_exclude_flags_considers_none_exclude_from_filenames_as_empty(): - flexmock(module).should_receive('_write_exclude_file').and_return(None) - exclude_flags = module._make_exclude_flags( location_config={'exclude_from': None}, ) @@ -154,7 +183,8 @@ CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'fo def test_create_archive_calls_borg_with_parameters(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND) @@ -170,10 +200,31 @@ def test_create_archive_calls_borg_with_parameters(): ) +def test_create_archive_with_patterns_calls_borg_with_patterns(): + pattern_flags = ('--patterns-from', 'patterns') + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_write_pattern_file').and_return(flexmock(name='/tmp/patterns')).and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) + insert_subprocess_mock(CREATE_COMMAND + pattern_flags) + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'patterns': ['pattern'], + }, + storage_config={}, + ) + + def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): exclude_flags = ('--exclude-from', 'excludes') flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes')) + flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(flexmock(name='/tmp/excludes')) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags) insert_subprocess_mock(CREATE_COMMAND + exclude_flags) @@ -191,7 +242,9 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',)) @@ -209,7 +262,8 @@ def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter(): def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats')) @@ -227,7 +281,8 @@ def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter(): def test_create_archive_with_compression_calls_borg_with_compression_parameters(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) @@ -245,7 +300,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--remote-ratelimit', '100')) @@ -263,7 +319,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) @@ -282,7 +339,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--files-cache', 'ctime,size')) @@ -301,7 +359,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1')) @@ -320,7 +379,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( def test_create_archive_with_umask_calls_borg_with_umask_parameters(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) @@ -338,7 +398,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): def test_create_archive_with_source_directories_glob_expands(): flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')) flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) @@ -357,7 +418,8 @@ def test_create_archive_with_source_directories_glob_expands(): def test_create_archive_with_non_matching_source_directories_glob_passes_through(): flexmock(module).should_receive('_expand_directory').and_return(['foo*']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*')) flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([]) @@ -376,7 +438,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through def test_create_archive_with_glob_calls_borg_with_expanded_directories(): flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')) @@ -394,7 +457,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar')) @@ -414,7 +478,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) - flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'))