Skip auto-exclusion of special files when user explicitly sets read_special to true (#587).

This commit is contained in:
Dan Helfman 2022-10-06 11:07:43 -07:00
parent 902730df46
commit a31ce337e9
3 changed files with 138 additions and 61 deletions

View file

@ -332,7 +332,6 @@ def create_archive(
upload_rate_limit = storage_config.get('upload_rate_limit', None) upload_rate_limit = storage_config.get('upload_rate_limit', None)
umask = storage_config.get('umask', None) umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None) lock_wait = storage_config.get('lock_wait', None)
read_special = True if (location_config.get('read_special') or stream_processes) else False
files_cache = location_config.get('files_cache') files_cache = location_config.get('files_cache')
archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT) archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '') extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
@ -384,7 +383,7 @@ def create_archive(
+ atime_flags + atime_flags
+ (('--noctime',) if location_config.get('ctime') is False else ()) + (('--noctime',) if location_config.get('ctime') is False else ())
+ (('--nobirthtime',) if location_config.get('birthtime') is False else ()) + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
+ (('--read-special',) if read_special else ()) + (('--read-special',) if location_config.get('read_special') or stream_processes else ())
+ noflags_flags + noflags_flags
+ (('--files-cache', files_cache) if files_cache else ()) + (('--files-cache', files_cache) if files_cache else ())
+ (('--remote-path', remote_path) if remote_path else ()) + (('--remote-path', remote_path) if remote_path else ())
@ -410,8 +409,9 @@ def create_archive(
borg_environment = environment.make_environment(storage_config) borg_environment = environment.make_environment(storage_config)
# If read_special is enabled, exclude files that might cause Borg to hang. # If database hooks are enabled (as indicated by streaming processes), exclude files that might
if read_special: # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
if stream_processes and not location_config.get('read_special'):
logger.debug(f'{repository}: Collecting special file paths') logger.debug(f'{repository}: Collecting special file paths')
special_file_paths = collect_special_file_paths( special_file_paths = collect_special_file_paths(
create_command, create_command,

View file

@ -217,7 +217,11 @@ special files are excluded from backups (named pipes, block devices,
character devices, and sockets) to prevent hanging. Try a command like character devices, and sockets) to prevent hanging. Try a command like
`find /your/source/path -type b -or -type c -or -type p -or -type s` to find `find /your/source/path -type b -or -type c -or -type p -or -type s` to find
such files. Common directories to exclude are `/dev` and `/run`, but that may such files. Common directories to exclude are `/dev` and `/run`, but that may
not be exhaustive. not be exhaustive. <span class="minilink minilink-addedin">New in version
1.7.3</span> When database hooks are enabled, borgmatic automatically excludes
special files that may cause Borg to hang, so you no longer need to manually
exclude them. You can override/prevent this behavior by explicitly setting
`read_special` to true.
### Manual restoration ### Manual restoration
@ -273,3 +277,7 @@ Alternatively, if excluding special files is too onerous, you can create two
separate borgmatic configuration files—one for your source files and a separate borgmatic configuration files—one for your source files and a
separate one for backing up databases. That way, the database `read_special` separate one for backing up databases. That way, the database `read_special`
option will not be active when backing up special files. option will not be active when backing up special files.
<span class="minilink minilink-addedin">New in version 1.7.3</span> See
Limitations above about borgmatic's automatic exclusion of special files to
prevent Borg hangs.

View file

@ -1147,62 +1147,6 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
) )
def test_create_archive_with_read_special_adds_special_files_to_excludes():
flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('expand_directories').and_return(())
flexmock(module).should_receive('pattern_root_directories').and_return([])
flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
flexmock(module).should_receive('expand_home_directories').and_return(()).and_return(
('special',)
)
flexmock(module).should_receive('write_pattern_file').and_return(None).and_return(
flexmock(name='/excludes')
)
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module).should_receive('ensure_files_readable')
flexmock(module).should_receive('make_pattern_flags').and_return(())
flexmock(module).should_receive('make_exclude_flags').and_return(()).and_return(
'--exclude-from', '/excludes'
)
flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
(f'repo::{DEFAULT_ARCHIVE_NAME}',)
)
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('collect_special_file_paths').and_return(('special',))
create_command = ('borg', 'create', '--read-special') + REPO_ARCHIVE_WITH_PATHS
flexmock(module).should_receive('execute_command').with_args(
create_command + ('--dry-run', '--list'),
output_log_level=logging.INFO,
output_file=None,
borg_local_path='borg',
working_directory=None,
extra_environment=None,
)
flexmock(module).should_receive('execute_command').with_args(
create_command + ('--exclude-from', '/excludes'),
output_log_level=logging.INFO,
output_file=None,
borg_local_path='borg',
working_directory=None,
extra_environment=None,
)
module.create_archive(
dry_run=False,
repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'read_special': True,
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
)
@pytest.mark.parametrize( @pytest.mark.parametrize(
'option_name,option_value', 'option_name,option_value',
(('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),), (('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),),
@ -1911,6 +1855,131 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log
) )
def test_create_archive_with_stream_processes_adds_special_files_to_excludes():
processes = flexmock()
flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('expand_directories').and_return(())
flexmock(module).should_receive('pattern_root_directories').and_return([])
flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
flexmock(module).should_receive('expand_home_directories').and_return(()).and_return(
('special',)
)
flexmock(module).should_receive('write_pattern_file').and_return(None).and_return(
flexmock(name='/excludes')
)
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module).should_receive('ensure_files_readable')
flexmock(module).should_receive('make_pattern_flags').and_return(())
flexmock(module).should_receive('make_exclude_flags').and_return(()).and_return(
'--exclude-from', '/excludes'
)
flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
(f'repo::{DEFAULT_ARCHIVE_NAME}',)
)
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('collect_special_file_paths').and_return(('special',))
create_command = (
'borg',
'create',
'--one-file-system',
'--read-special',
) + REPO_ARCHIVE_WITH_PATHS
flexmock(module).should_receive('execute_command_with_processes').with_args(
create_command + ('--dry-run', '--list'),
processes=processes,
output_log_level=logging.INFO,
output_file=None,
borg_local_path='borg',
working_directory=None,
extra_environment=None,
)
flexmock(module).should_receive('execute_command_with_processes').with_args(
create_command + ('--exclude-from', '/excludes'),
processes=processes,
output_log_level=logging.INFO,
output_file=None,
borg_local_path='borg',
working_directory=None,
extra_environment=None,
)
module.create_archive(
dry_run=False,
repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
stream_processes=processes,
)
def test_create_archive_with_stream_processes_and_read_special_does_not_add_special_files_to_excludes():
processes = flexmock()
flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('expand_directories').and_return(())
flexmock(module).should_receive('pattern_root_directories').and_return([])
flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
flexmock(module).should_receive('expand_home_directories').and_return(()).and_return(
('special',)
)
flexmock(module).should_receive('write_pattern_file').and_return(None)
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module).should_receive('ensure_files_readable')
flexmock(module).should_receive('make_pattern_flags').and_return(())
flexmock(module).should_receive('make_exclude_flags').and_return(())
flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
(f'repo::{DEFAULT_ARCHIVE_NAME}',)
)
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('collect_special_file_paths').and_return(('special',))
create_command = (
'borg',
'create',
'--one-file-system',
'--read-special',
) + REPO_ARCHIVE_WITH_PATHS
flexmock(module).should_receive('execute_command_with_processes').with_args(
create_command + ('--dry-run', '--list'),
processes=processes,
output_log_level=logging.INFO,
output_file=None,
borg_local_path='borg',
working_directory=None,
extra_environment=None,
)
flexmock(module).should_receive('execute_command_with_processes').with_args(
create_command,
processes=processes,
output_log_level=logging.INFO,
output_file=None,
borg_local_path='borg',
working_directory=None,
extra_environment=None,
)
module.create_archive(
dry_run=False,
repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'read_special': True,
},
storage_config={},
local_borg_version='1.2.3',
stream_processes=processes,
)
def test_create_archive_with_json_calls_borg_with_json_parameter(): def test_create_archive_with_json_calls_borg_with_json_parameter():
flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))