Skip auto-exclusion of special files when user explicitly sets read_special to true (#587).

borgmatic/borg/create.py

@@ -332,7 +332,6 @@ def create_archive(
     upload_rate_limit = storage_config.get('upload_rate_limit', None)
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
-    read_special = True if (location_config.get('read_special') or stream_processes) else False
     files_cache = location_config.get('files_cache')
     archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
     extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
@@ -384,7 +383,7 @@ def create_archive(
         + atime_flags
         + (('--noctime',) if location_config.get('ctime') is False else ())
         + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
-        + (('--read-special',) if read_special else ())
+        + (('--read-special',) if location_config.get('read_special') or stream_processes else ())
         + noflags_flags
         + (('--files-cache', files_cache) if files_cache else ())
         + (('--remote-path', remote_path) if remote_path else ())
@@ -410,8 +409,9 @@ def create_archive(
 
     borg_environment = environment.make_environment(storage_config)
 
-    # If read_special is enabled, exclude files that might cause Borg to hang.
+    # If database hooks are enabled (as indicated by streaming processes), exclude files that might
-    if read_special:
+    # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
+    if stream_processes and not location_config.get('read_special'):
         logger.debug(f'{repository}: Collecting special file paths')
         special_file_paths = collect_special_file_paths(
             create_command,

docs/how-to/backup-your-databases.md

@@ -217,7 +217,11 @@ special files are excluded from backups (named pipes, block devices,
 character devices, and sockets) to prevent hanging. Try a command like
 `find /your/source/path -type b -or -type c -or -type p -or -type s` to find
 such files. Common directories to exclude are `/dev` and `/run`, but that may
-not be exhaustive.
+not be exhaustive. <span class="minilink minilink-addedin">New in version
+1.7.3</span> When database hooks are enabled, borgmatic automatically excludes
+special files that may cause Borg to hang, so you no longer need to manually
+exclude them. You can override/prevent this behavior by explicitly setting
+`read_special` to true.
 
 
 ### Manual restoration
@@ -273,3 +277,7 @@ Alternatively, if excluding special files is too onerous, you can create two
 separate borgmatic configuration files—one for your source files and a
 separate one for backing up databases. That way, the database `read_special`
 option will not be active when backing up special files.
+
+<span class="minilink minilink-addedin">New in version 1.7.3</span> See
+Limitations above about borgmatic's automatic exclusion of special files to
+prevent Borg hangs.

tests/unit/borg/test_create.py

@@ -1147,62 +1147,6 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
     )
 
 
-def test_create_archive_with_read_special_adds_special_files_to_excludes():
-    flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
-    flexmock(module).should_receive('map_directories_to_devices').and_return({})
-    flexmock(module).should_receive('expand_directories').and_return(())
-    flexmock(module).should_receive('pattern_root_directories').and_return([])
-    flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
-    flexmock(module).should_receive('expand_home_directories').and_return(()).and_return(
-        ('special',)
-    )
-    flexmock(module).should_receive('write_pattern_file').and_return(None).and_return(
-        flexmock(name='/excludes')
-    )
-    flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('ensure_files_readable')
-    flexmock(module).should_receive('make_pattern_flags').and_return(())
-    flexmock(module).should_receive('make_exclude_flags').and_return(()).and_return(
-        '--exclude-from', '/excludes'
-    )
-    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
-        (f'repo::{DEFAULT_ARCHIVE_NAME}',)
-    )
-    flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('collect_special_file_paths').and_return(('special',))
-    create_command = ('borg', 'create', '--read-special') + REPO_ARCHIVE_WITH_PATHS
-    flexmock(module).should_receive('execute_command').with_args(
-        create_command + ('--dry-run', '--list'),
-        output_log_level=logging.INFO,
-        output_file=None,
-        borg_local_path='borg',
-        working_directory=None,
-        extra_environment=None,
-    )
-    flexmock(module).should_receive('execute_command').with_args(
-        create_command + ('--exclude-from', '/excludes'),
-        output_log_level=logging.INFO,
-        output_file=None,
-        borg_local_path='borg',
-        working_directory=None,
-        extra_environment=None,
-    )
-
-    module.create_archive(
-        dry_run=False,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'read_special': True,
-            'exclude_patterns': None,
-        },
-        storage_config={},
-        local_borg_version='1.2.3',
-    )
-
-
 @pytest.mark.parametrize(
     'option_name,option_value',
     (('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),),
@@ -1911,6 +1855,131 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log
     )
 
 
+def test_create_archive_with_stream_processes_adds_special_files_to_excludes():
+    processes = flexmock()
+    flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
+    flexmock(module).should_receive('expand_directories').and_return(())
+    flexmock(module).should_receive('pattern_root_directories').and_return([])
+    flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
+    flexmock(module).should_receive('expand_home_directories').and_return(()).and_return(
+        ('special',)
+    )
+    flexmock(module).should_receive('write_pattern_file').and_return(None).and_return(
+        flexmock(name='/excludes')
+    )
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module).should_receive('ensure_files_readable')
+    flexmock(module).should_receive('make_pattern_flags').and_return(())
+    flexmock(module).should_receive('make_exclude_flags').and_return(()).and_return(
+        '--exclude-from', '/excludes'
+    )
+    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
+        (f'repo::{DEFAULT_ARCHIVE_NAME}',)
+    )
+    flexmock(module.environment).should_receive('make_environment')
+    flexmock(module).should_receive('collect_special_file_paths').and_return(('special',))
+    create_command = (
+        'borg',
+        'create',
+        '--one-file-system',
+        '--read-special',
+    ) + REPO_ARCHIVE_WITH_PATHS
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        create_command + ('--dry-run', '--list'),
+        processes=processes,
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+        working_directory=None,
+        extra_environment=None,
+    )
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        create_command + ('--exclude-from', '/excludes'),
+        processes=processes,
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+        working_directory=None,
+        extra_environment=None,
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        local_borg_version='1.2.3',
+        stream_processes=processes,
+    )
+
+
+def test_create_archive_with_stream_processes_and_read_special_does_not_add_special_files_to_excludes():
+    processes = flexmock()
+    flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
+    flexmock(module).should_receive('expand_directories').and_return(())
+    flexmock(module).should_receive('pattern_root_directories').and_return([])
+    flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
+    flexmock(module).should_receive('expand_home_directories').and_return(()).and_return(
+        ('special',)
+    )
+    flexmock(module).should_receive('write_pattern_file').and_return(None)
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module).should_receive('ensure_files_readable')
+    flexmock(module).should_receive('make_pattern_flags').and_return(())
+    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
+        (f'repo::{DEFAULT_ARCHIVE_NAME}',)
+    )
+    flexmock(module.environment).should_receive('make_environment')
+    flexmock(module).should_receive('collect_special_file_paths').and_return(('special',))
+    create_command = (
+        'borg',
+        'create',
+        '--one-file-system',
+        '--read-special',
+    ) + REPO_ARCHIVE_WITH_PATHS
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        create_command + ('--dry-run', '--list'),
+        processes=processes,
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+        working_directory=None,
+        extra_environment=None,
+    )
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        create_command,
+        processes=processes,
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+        working_directory=None,
+        extra_environment=None,
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+            'read_special': True,
+        },
+        storage_config={},
+        local_borg_version='1.2.3',
+        stream_processes=processes,
+    )
+
+
 def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))