49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed includes/excludes.
This commit is contained in:
Dan Helfman
parent
50b3240c4f
commit
b8f6bab12d
4 changed files with 152 additions and 41 deletions
4
NEWS
4
NEWS
|
|
@ -1,6 +1,8 @@
|
|||
1.1.13.dev0
|
||||
* #54: Fix for incorrect consistency check flags passed to Borg when all three checks ("repository",
|
||||
"archives", and "extract") are specified in borgmatic configuration.
|
||||
* #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed
|
||||
includes/excludes.
|
||||
* Moved issue tracker from Taiga to integrated Gitea tracker at
|
||||
https://projects.torsion.org/witten/borgmatic/issues
|
||||
|
||||
|
|
@ -19,7 +21,7 @@
|
|||
shuts down if borgmatic is terminated (e.g. due to a system suspend).
|
||||
* #30: Support for using tilde in repository paths to reference home directory.
|
||||
* #43: Support for Borg --files-cache option for setting the files cache operation mode.
|
||||
* #45: Support for Borg --remote-ratelimit for limiting upload rate.
|
||||
* #45: Support for Borg --remote-ratelimit option for limiting upload rate.
|
||||
* Log invoked Borg commands when at highest verbosity level.
|
||||
|
||||
1.1.9
|
||||
|
|
|
|||
|
|
@ -31,28 +31,45 @@ def _expand_directory(directory):
|
|||
return glob.glob(expanded_directory) or [expanded_directory]
|
||||
|
||||
|
||||
def _write_exclude_file(exclude_patterns=None):
|
||||
def _write_pattern_file(patterns=None):
|
||||
'''
|
||||
Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
|
||||
None if no patterns are provided.
|
||||
Given a sequence of patterns, write them to a named temporary file and return it. Return None
|
||||
if no patterns are provided.
|
||||
'''
|
||||
if not exclude_patterns:
|
||||
if not patterns:
|
||||
return None
|
||||
|
||||
exclude_file = tempfile.NamedTemporaryFile('w')
|
||||
exclude_file.write('\n'.join(exclude_patterns))
|
||||
exclude_file.flush()
|
||||
pattern_file = tempfile.NamedTemporaryFile('w')
|
||||
pattern_file.write('\n'.join(patterns))
|
||||
pattern_file.flush()
|
||||
|
||||
return exclude_file
|
||||
return pattern_file
|
||||
|
||||
|
||||
def _make_exclude_flags(location_config, exclude_patterns_filename=None):
|
||||
def _make_pattern_flags(location_config, pattern_filename=None):
|
||||
'''
|
||||
Given a location config dict with a potential pattern_from option, and a filename containing any
|
||||
additional patterns, return the corresponding Borg flags for those files as a tuple.
|
||||
'''
|
||||
pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
|
||||
(pattern_filename,) if pattern_filename else ()
|
||||
)
|
||||
|
||||
return tuple(
|
||||
itertools.chain.from_iterable(
|
||||
('--pattern-from', pattern_filename)
|
||||
for pattern_filename in pattern_filenames
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _make_exclude_flags(location_config, exclude_filename=None):
|
||||
'''
|
||||
Given a location config dict with various exclude options, and a filename containing any exclude
|
||||
patterns, return the corresponding Borg flags as a tuple.
|
||||
'''
|
||||
exclude_filenames = tuple(location_config.get('exclude_from') or ()) + (
|
||||
(exclude_patterns_filename,) if exclude_patterns_filename else ()
|
||||
(exclude_filename,) if exclude_filename else ()
|
||||
)
|
||||
exclude_from_flags = tuple(
|
||||
itertools.chain.from_iterable(
|
||||
|
|
@ -81,10 +98,15 @@ def create_archive(
|
|||
)
|
||||
)
|
||||
|
||||
exclude_patterns_file = _write_exclude_file(location_config.get('exclude_patterns'))
|
||||
pattern_file = _write_pattern_file(location_config.get('patterns'))
|
||||
pattern_flags = _make_pattern_flags(
|
||||
location_config,
|
||||
pattern_file.name if pattern_file else None,
|
||||
)
|
||||
exclude_file = _write_pattern_file(location_config.get('exclude_patterns'))
|
||||
exclude_flags = _make_exclude_flags(
|
||||
location_config,
|
||||
exclude_patterns_file.name if exclude_patterns_file else None,
|
||||
exclude_file.name if exclude_file else None,
|
||||
)
|
||||
compression = storage_config.get('compression', None)
|
||||
compression_flags = ('--compression', compression) if compression else ()
|
||||
|
|
@ -110,7 +132,7 @@ def create_archive(
|
|||
repository=repository,
|
||||
archive_name_format=archive_name_format,
|
||||
),
|
||||
) + sources + exclude_flags + compression_flags + remote_rate_limit_flags + \
|
||||
) + sources + pattern_flags + exclude_flags + compression_flags + remote_rate_limit_flags + \
|
||||
one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \
|
||||
verbosity_flags
|
||||
|
||||
|
|
|
|||
|
|
@ -42,6 +42,28 @@ map:
|
|||
repositories are backed up to in sequence.
|
||||
example:
|
||||
- user@backupserver:sourcehostname.borg
|
||||
patterns:
|
||||
seq:
|
||||
- type: scalar
|
||||
desc: |
|
||||
Any paths matching these patterns are included/excluded from backups. Globs are
|
||||
expanded. Note that Borg considers this option experimental. See the output of
|
||||
"borg help patterns" for more details. Quoting any value if it contains leading
|
||||
punctuation, so it parses correctly.
|
||||
example:
|
||||
- 'R /'
|
||||
- '- /home/*/.cache'
|
||||
- '+ /home/susan'
|
||||
- '- /home/*'
|
||||
patterns_from:
|
||||
seq:
|
||||
- type: scalar
|
||||
desc: |
|
||||
Read include/exclude patterns from one or more separate named files, one pattern
|
||||
per line. Note that Borg considers this option experimental. See the output of
|
||||
"borg help patterns" for more details.
|
||||
example:
|
||||
- /etc/borgmatic/patterns
|
||||
exclude_patterns:
|
||||
seq:
|
||||
- type: scalar
|
||||
|
|
@ -57,7 +79,7 @@ map:
|
|||
- type: scalar
|
||||
desc: |
|
||||
Read exclude patterns from one or more separate named files, one pattern per
|
||||
line.
|
||||
line. See the output of "borg help patterns" for more details.
|
||||
example:
|
||||
- /etc/borgmatic/excludes
|
||||
exclude_caches:
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ def test_expand_directory_with_glob_expands():
|
|||
assert paths == ['foo', 'food']
|
||||
|
||||
|
||||
def test_write_exclude_file_does_not_raise():
|
||||
def test_write_pattern_file_does_not_raise():
|
||||
temporary_file = flexmock(
|
||||
name='filename',
|
||||
write=lambda mode: None,
|
||||
|
|
@ -66,11 +66,11 @@ def test_write_exclude_file_does_not_raise():
|
|||
)
|
||||
flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
|
||||
|
||||
module._write_exclude_file(['exclude'])
|
||||
module._write_pattern_file(['exclude'])
|
||||
|
||||
|
||||
def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
|
||||
module._write_exclude_file([])
|
||||
def test_write_pattern_file_with_empty_exclude_patterns_does_not_raise():
|
||||
module._write_pattern_file([])
|
||||
|
||||
|
||||
def insert_subprocess_mock(check_call_command, **kwargs):
|
||||
|
|
@ -78,17 +78,50 @@ def insert_subprocess_mock(check_call_command, **kwargs):
|
|||
subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
|
||||
|
||||
|
||||
def test_make_pattern_flags_includes_pattern_filename_when_given():
|
||||
pattern_flags = module._make_pattern_flags(
|
||||
location_config={'patterns': ['R /', '- /var']},
|
||||
pattern_filename='/tmp/patterns',
|
||||
)
|
||||
|
||||
assert pattern_flags == ('--pattern-from', '/tmp/patterns')
|
||||
|
||||
|
||||
def test_make_pattern_flags_includes_patterns_from_filenames_when_in_config():
|
||||
pattern_flags = module._make_pattern_flags(
|
||||
location_config={'patterns_from': ['patterns', 'other']},
|
||||
)
|
||||
|
||||
assert pattern_flags == ('--pattern-from', 'patterns', '--pattern-from', 'other')
|
||||
|
||||
|
||||
def test_make_pattern_flags_includes_both_filenames_when_patterns_given_and_patterns_from_in_config():
|
||||
pattern_flags = module._make_pattern_flags(
|
||||
location_config={'patterns_from': ['patterns']},
|
||||
pattern_filename='/tmp/patterns',
|
||||
)
|
||||
|
||||
assert pattern_flags == ('--pattern-from', 'patterns', '--pattern-from', '/tmp/patterns')
|
||||
|
||||
|
||||
def test_make_pattern_flags_considers_none_patterns_from_filenames_as_empty():
|
||||
pattern_flags = module._make_pattern_flags(
|
||||
location_config={'patterns_from': None},
|
||||
)
|
||||
|
||||
assert pattern_flags == ()
|
||||
|
||||
|
||||
def test_make_exclude_flags_includes_exclude_patterns_filename_when_given():
|
||||
exclude_flags = module._make_exclude_flags(
|
||||
location_config={'exclude_patterns': ['*.pyc', '/var']},
|
||||
exclude_patterns_filename='/tmp/excludes',
|
||||
exclude_filename='/tmp/excludes',
|
||||
)
|
||||
|
||||
assert exclude_flags == ('--exclude-from', '/tmp/excludes')
|
||||
|
||||
|
||||
def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
|
||||
exclude_flags = module._make_exclude_flags(
|
||||
location_config={'exclude_from': ['excludes', 'other']},
|
||||
|
|
@ -98,19 +131,15 @@ def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
|
|||
|
||||
|
||||
def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config():
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
|
||||
exclude_flags = module._make_exclude_flags(
|
||||
location_config={'exclude_from': ['excludes']},
|
||||
exclude_patterns_filename='/tmp/excludes',
|
||||
exclude_filename='/tmp/excludes',
|
||||
)
|
||||
|
||||
assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes')
|
||||
|
||||
|
||||
def test_make_exclude_flags_considers_none_exclude_from_filenames_as_empty():
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
|
||||
exclude_flags = module._make_exclude_flags(
|
||||
location_config={'exclude_from': None},
|
||||
)
|
||||
|
|
@ -154,7 +183,8 @@ CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'fo
|
|||
|
||||
def test_create_archive_calls_borg_with_parameters():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(CREATE_COMMAND)
|
||||
|
||||
|
|
@ -170,10 +200,31 @@ def test_create_archive_calls_borg_with_parameters():
|
|||
)
|
||||
|
||||
|
||||
def test_create_archive_with_patterns_calls_borg_with_patterns():
|
||||
pattern_flags = ('--patterns-from', 'patterns')
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(flexmock(name='/tmp/patterns')).and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags)
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(CREATE_COMMAND + pattern_flags)
|
||||
|
||||
module.create_archive(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
location_config={
|
||||
'source_directories': ['foo', 'bar'],
|
||||
'repositories': ['repo'],
|
||||
'patterns': ['pattern'],
|
||||
},
|
||||
storage_config={},
|
||||
)
|
||||
|
||||
|
||||
def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
|
||||
exclude_flags = ('--exclude-from', 'excludes')
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes'))
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(flexmock(name='/tmp/excludes'))
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
|
||||
insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
|
||||
|
||||
|
|
@ -191,7 +242,9 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
|
|||
|
||||
def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
|
||||
|
||||
|
|
@ -209,7 +262,8 @@ def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
|
|||
|
||||
def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
|
||||
|
||||
|
|
@ -227,7 +281,8 @@ def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
|
|||
|
||||
def test_create_archive_with_compression_calls_borg_with_compression_parameters():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
|
||||
|
||||
|
|
@ -245,7 +300,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
|
|||
|
||||
def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(CREATE_COMMAND + ('--remote-ratelimit', '100'))
|
||||
|
||||
|
|
@ -263,7 +319,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
|
|||
|
||||
def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
|
||||
|
||||
|
|
@ -282,7 +339,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
|
|||
|
||||
def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(CREATE_COMMAND + ('--files-cache', 'ctime,size'))
|
||||
|
||||
|
|
@ -301,7 +359,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
|
|||
|
||||
def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
|
||||
|
||||
|
|
@ -320,7 +379,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
|
|||
|
||||
def test_create_archive_with_umask_calls_borg_with_umask_parameters():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
|
||||
|
||||
|
|
@ -338,7 +398,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
|
|||
|
||||
def test_create_archive_with_source_directories_glob_expands():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
|
||||
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
|
||||
|
|
@ -357,7 +418,8 @@ def test_create_archive_with_source_directories_glob_expands():
|
|||
|
||||
def test_create_archive_with_non_matching_source_directories_glob_passes_through():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo*'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'))
|
||||
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
|
||||
|
|
@ -376,7 +438,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
|
|||
|
||||
def test_create_archive_with_glob_calls_borg_with_expanded_directories():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
|
||||
|
||||
|
|
@ -394,7 +457,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
|
|||
|
||||
def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'))
|
||||
|
||||
|
|
@ -414,7 +478,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
|
|||
|
||||
def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
|
||||
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
|
||||
flexmock(module).should_receive('_write_exclude_file').and_return(None)
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'))
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue