#12, #35: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options.

NEWS

@@ -1,3 +1,7 @@
+1.1.6
+
+ * #12, #35: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options.
+
 1.1.5
 
  * #34: New "extract" consistency check that performs a dry-run extraction of the most recent

README.md

@@ -121,7 +121,7 @@ However, see below about special cases.
 borgmatic changed its configuration file format in version 1.1.0 from
 INI-style to YAML. This better supports validation, and has a more natural way
 to express lists of values. To upgrade your existing configuration, first
-upgrade to the new version of borgmatic:
+upgrade to the new version of borgmatic.
 
 As of version 1.1.0, borgmatic no longer supports Python 2. If you were
 already running borgmatic with Python 3, then you can simply upgrade borgmatic

borgmatic/borg/create.py

@@ -31,12 +31,33 @@ def _write_exclude_file(exclude_patterns=None):
     return exclude_file
 
 
+def _make_exclude_flags(location_config, exclude_patterns_filename=None):
+    '''
+    Given a location config dict with various exclude options, and a filename containing any exclude
+    patterns, return the corresponding Borg flags as a tuple.
+    '''
+    exclude_filenames = tuple(location_config.get('exclude_from', ())) + (
+        (exclude_patterns_filename,) if exclude_patterns_filename else ()
+    )
+    exclude_from_flags = tuple(
+        itertools.chain.from_iterable(
+            ('--exclude-from', exclude_filename)
+            for exclude_filename in exclude_filenames
+        )
+    )
+    caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
+    if_present = location_config.get('exclude_if_present')
+    if_present_flags = ('--exclude-if-present', if_present) if if_present else ()
+
+    return exclude_from_flags + caches_flag + if_present_flags
+
+
 def create_archive(
     verbosity, repository, location_config, storage_config,
 ):
     '''
-    Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
+    Given a vebosity flag, a local or remote repository path, a location config dict, and a storage
-    repository path, a list of exclude patterns, create a Borg archive.
+    config dict, create a Borg archive.
     '''
     sources = tuple(
         itertools.chain.from_iterable(
@@ -45,8 +66,11 @@ def create_archive(
         )
     )
 
-    exclude_file = _write_exclude_file(location_config.get('exclude_patterns'))
+    exclude_patterns_file = _write_exclude_file(location_config.get('exclude_patterns'))
-    exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
+    exclude_flags = _make_exclude_flags(
+        location_config,
+        exclude_patterns_file.name if exclude_patterns_file else None,
+    )
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
     umask = storage_config.get('umask', None)

borgmatic/config/schema.yaml

@@ -45,6 +45,24 @@ map:
                     - '*.pyc'
                     - /home/*/.cache
                     - /etc/ssl
+            exclude_from:
+                seq:
+                    - type: scalar
+                desc: |
+                    Read exclude patterns from one or more separate named files, one pattern per
+                    line.
+                example:
+                    - /etc/borgmatic/excludes
+            exclude_caches:
+                type: bool
+                desc: |
+                    Exclude directories that contain a CACHEDIR.TAG file. See
+                    http://www.brynosaurus.com/cachedir/spec.html for details.
+                example: true
+            exclude_if_present:
+                type: scalar
+                desc: Exclude directories that contain a file with the given filename.
+                example: .nobackup
     storage:
         desc: |
             Repository storage options. See

borgmatic/tests/unit/borg/test_create.py

@@ -58,11 +58,72 @@ def insert_datetime_mock():
     ).mock
 
 
+def test_make_exclude_flags_includes_exclude_patterns_filename_when_given():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_patterns': ['*.pyc', '/var']},
+        exclude_patterns_filename='/tmp/excludes',
+    )
+
+    assert exclude_flags == ('--exclude-from', '/tmp/excludes')
+
+
+def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_from': ['excludes', 'other']},
+    )
+
+    assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', 'other')
+
+
+def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config():
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_from': ['excludes']},
+        exclude_patterns_filename='/tmp/excludes',
+    )
+
+    assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes')
+
+
+def test_make_exclude_flags_includes_exclude_caches_when_true_in_config():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_caches': True},
+    )
+
+    assert exclude_flags == ('--exclude-caches',)
+
+
+def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_caches': False},
+    )
+
+    assert exclude_flags == ()
+
+
+def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_if_present': 'exclude_me'},
+    )
+
+    assert exclude_flags == ('--exclude-if-present', 'exclude_me')
+
+
+def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
+    exclude_flags = module._make_exclude_flags(location_config={})
+
+    assert exclude_flags == ()
+
+
 CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
 
 
 def test_create_archive_should_call_borg_with_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND)
     insert_platform_mock()
     insert_datetime_mock()
@@ -80,8 +141,10 @@ def test_create_archive_should_call_borg_with_parameters():
 
 
 def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
-    flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes'))
+    exclude_flags = ('--exclude-from', 'excludes')
-    insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes'))
+    flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes'))
+    flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
+    insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
     insert_platform_mock()
     insert_datetime_mock()
 
@@ -98,7 +161,8 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
 
 
 def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
     insert_platform_mock()
     insert_datetime_mock()
@@ -116,7 +180,8 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter
 
 
 def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -134,7 +199,8 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete
 
 
 def test_create_archive_with_compression_should_call_borg_with_compression_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -152,7 +218,8 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param
 
 
 def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
     insert_platform_mock()
     insert_datetime_mock()
@@ -171,7 +238,8 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst
 
 
 def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -190,7 +258,8 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param
 
 
 def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -208,7 +277,8 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
 
 
 def test_create_archive_with_source_directories_glob_expands():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -227,7 +297,8 @@ def test_create_archive_with_source_directories_glob_expands():
 
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -246,7 +317,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 
 def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
     insert_platform_mock()
     insert_datetime_mock()

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.5'
+VERSION = '1.1.6'
 
 
 setup(