From b40e9b7da20471bc31c98bb4b5596353c5987d29 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 18 Aug 2022 09:59:48 -0700 Subject: [PATCH] Ignore archive filter parameters passed to list action when --archive is given (#557). --- NEWS | 7 +- borgmatic/borg/list.py | 25 +++-- docs/how-to/set-up-backups.md | 3 + tests/unit/borg/test_list.py | 189 +++++++++++++++++++++++++++------- 4 files changed, 177 insertions(+), 47 deletions(-) diff --git a/NEWS b/NEWS index a0505cd..f22d856 100644 --- a/NEWS +++ b/NEWS @@ -3,10 +3,9 @@ like "rcreate" (replaces "init"), "rlist" (list archives in repository), and "rinfo" (show repository info). For the most part, borgmatic tries to smooth over differences between Borg 1 and 2 to make your upgrade process easier. However, there are still a few cases where Borg made - breaking changes, such as moving flags from "borg list" to "borg rlist". See the Borg 2.0 - changelog for more information (https://www.borgbackup.org/releases/borg-2.0.html). If you - install Borg 2, you'll need to manually "borg transfer" or "borgmatic transfer" your existing - Borg 1 repositories before use. + breaking changes. See the Borg 2.0 changelog for more information + (https://www.borgbackup.org/releases/borg-2.0.html). If you install Borg 2, you'll need to + manually "borg transfer" or "borgmatic transfer" your existing Borg 1 repositories before use. * #557: Rename several configuration options to match Borg 2: "remote_rate_limit" is now "upload_rate_limit", "numeric_owner" is "numeric_ids", and "bsd_flags" is "flags". borgmatic still works with the old options. diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index a13b943..360f597 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -9,7 +9,14 @@ from borgmatic.execute import execute_command logger = logging.getLogger(__name__) -MAKE_FLAGS_EXCLUDES = ('repository', 'archive', 'successful', 'paths', 'find_paths') +ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST = ('prefix', 'glob_archives', 'sort_by', 'first', 'last') +MAKE_FLAGS_EXCLUDES = ( + 'repository', + 'archive', + 'successful', + 'paths', + 'find_paths', +) + ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST def make_list_command( @@ -113,11 +120,11 @@ def list_archive( repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path ) - if feature.available(feature.Feature.RLIST, local_borg_version): - for flag_name in ('prefix', 'glob-archives', 'sort-by', 'first', 'last'): - if getattr(list_arguments, flag_name.replace('-', '_'), None): - raise ValueError( - f'The --{flag_name} flag on the list action is not supported when using the --archive/--find flags and Borg 2.x+.' + if list_arguments.archive: + for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST: + if getattr(list_arguments, name, None): + logger.warning( + f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag." ) if list_arguments.json: @@ -169,6 +176,12 @@ def list_archive( archive_arguments = copy.copy(list_arguments) archive_arguments.archive = archive + + # This list call is to show the files in a single archive, not list multiple archives. So + # blank out any archive filtering flags. They'll break anyway in Borg 2. + for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST: + setattr(archive_arguments, name, None) + main_command = make_list_command( repository, storage_config, diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 6508cd6..e1a58ae 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -204,6 +204,9 @@ Or, with Borg 2.x: sudo borgmatic rcreate --encryption repokey-aes-ocb ``` +(Note that `repokey-chacha20-poly1305` may be faster than `repokey-aes-ocb` on +certain platforms like ARM64.) + This uses the borgmatic configuration file you created above to determine which local or remote repository to create, and encrypts it with the encryption passphrase specified there if one is provided. Read about [Borg diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index d1c5d39..0019d97 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -254,7 +254,17 @@ def test_make_find_paths_adds_globs_to_path_fragments(): def test_list_archive_calls_borg_with_parameters(): - list_arguments = argparse.Namespace(archive='archive', paths=None, json=False, find_paths=None) + list_arguments = argparse.Namespace( + archive='archive', + paths=None, + json=False, + find_paths=None, + prefix=None, + glob_archives=None, + sort_by=None, + first=None, + last=None, + ) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -297,7 +307,17 @@ def test_list_archive_with_archive_and_json_errors(): def test_list_archive_calls_borg_with_local_path(): - list_arguments = argparse.Namespace(archive='archive', paths=None, json=False, find_paths=None) + list_arguments = argparse.Namespace( + archive='archive', + paths=None, + json=False, + find_paths=None, + prefix=None, + glob_archives=None, + sort_by=None, + first=None, + last=None, + ) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -346,9 +366,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): output_log_level=None, borg_local_path='borg', extra_environment=None, - ).and_return( - 'archive1 Sun, 2022-05-29 15:27:04 [abc]\narchive2 Mon, 2022-05-30 19:47:15 [xyz]' - ).once() + ).and_return('archive1\narchive2').once() flexmock(module).should_receive('make_list_command').and_return( ('borg', 'list', 'repo::archive1') ).and_return(('borg', 'list', 'repo::archive2')) @@ -376,7 +394,17 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): def test_list_archive_calls_borg_with_archive(): - list_arguments = argparse.Namespace(archive='archive', paths=None, json=False, find_paths=None) + list_arguments = argparse.Namespace( + archive='archive', + paths=None, + json=False, + find_paths=None, + prefix=None, + glob_archives=None, + sort_by=None, + first=None, + last=None, + ) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -461,35 +489,15 @@ def test_list_archive_with_borg_features_without_archive_delegates_to_list_repos @pytest.mark.parametrize( 'archive_filter_flag', ('prefix', 'glob_archives', 'sort_by', 'first', 'last',), ) -def test_list_archive_with_archive_disallows_archive_filter_flag_if_rlist_feature_available( - archive_filter_flag, -): - list_arguments = argparse.Namespace( - archive='archive', paths=None, json=False, find_paths=None, **{archive_filter_flag: 'foo'} - ) - - flexmock(module.feature).should_receive('available').with_args( - module.feature.Feature.RLIST, '1.2.3' - ).and_return(True) - - with pytest.raises(ValueError): - module.list_archive( - repository='repo', - storage_config={}, - local_borg_version='1.2.3', - list_arguments=list_arguments, - ) - - -@pytest.mark.parametrize( - 'archive_filter_flag', ('prefix', 'glob_archives', 'sort_by', 'first', 'last',), -) -def test_list_archive_with_archive_allows_archive_filter_flag_if_rlist_feature_unavailable( - archive_filter_flag, -): - list_arguments = argparse.Namespace( - archive='archive', paths=None, json=False, find_paths=None, **{archive_filter_flag: 'foo'} - ) +def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_flag,): + default_filter_flags = { + 'prefix': None, + 'glob_archives': None, + 'sort_by': None, + 'first': None, + 'last': None, + } + altered_filter_flags = {**default_filter_flags, **{archive_filter_flag: 'foo'}} flexmock(module.feature).should_receive('available').with_args( module.feature.Feature.RLIST, '1.2.3' @@ -498,7 +506,9 @@ def test_list_archive_with_archive_allows_archive_filter_flag_if_rlist_feature_u repository='repo', storage_config={}, local_borg_version='1.2.3', - list_arguments=list_arguments, + list_arguments=argparse.Namespace( + archive='archive', paths=None, json=False, find_paths=None, **default_filter_flags + ), local_path='borg', remote_path=None, ).and_return(('borg', 'list', 'repo::archive')) @@ -515,5 +525,110 @@ def test_list_archive_with_archive_allows_archive_filter_flag_if_rlist_feature_u repository='repo', storage_config={}, local_borg_version='1.2.3', - list_arguments=list_arguments, + list_arguments=argparse.Namespace( + archive='archive', paths=None, json=False, find_paths=None, **altered_filter_flags + ), + ) + + +@pytest.mark.parametrize( + 'archive_filter_flag', ('prefix', 'glob_archives', 'sort_by', 'first', 'last',), +) +def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes_it_to_rlist( + archive_filter_flag, +): + default_filter_flags = { + 'prefix': None, + 'glob_archives': None, + 'sort_by': None, + 'first': None, + 'last': None, + } + altered_filter_flags = {**default_filter_flags, **{archive_filter_flag: 'foo'}} + glob_paths = ('**/*foo.txt*/**',) + flexmock(module.feature).should_receive('available').and_return(True) + + flexmock(module.rlist).should_receive('make_rlist_command').with_args( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=argparse.Namespace( + repository='repo', short=True, format=None, json=None, **altered_filter_flags + ), + local_path='borg', + remote_path=None, + ).and_return(('borg', 'rlist', '--repo', 'repo')) + + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'rlist', '--repo', 'repo'), + output_log_level=None, + borg_local_path='borg', + extra_environment=None, + ).and_return('archive1\narchive2').once() + + flexmock(module).should_receive('make_list_command').with_args( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=argparse.Namespace( + repository='repo', + archive='archive1', + paths=None, + short=True, + format=None, + json=None, + find_paths=['foo.txt'], + **default_filter_flags, + ), + local_path='borg', + remote_path=None, + ).and_return(('borg', 'list', '--repo', 'repo', 'archive1')) + + flexmock(module).should_receive('make_list_command').with_args( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=argparse.Namespace( + repository='repo', + archive='archive2', + paths=None, + short=True, + format=None, + json=None, + find_paths=['foo.txt'], + **default_filter_flags, + ), + local_path='borg', + remote_path=None, + ).and_return(('borg', 'list', '--repo', 'repo', 'archive2')) + + flexmock(module).should_receive('make_find_paths').and_return(glob_paths) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'list', '--repo', 'repo', 'archive1') + glob_paths, + output_log_level=logging.WARNING, + borg_local_path='borg', + extra_environment=None, + ).once() + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'list', '--repo', 'repo', 'archive2') + glob_paths, + output_log_level=logging.WARNING, + borg_local_path='borg', + extra_environment=None, + ).once() + + module.list_archive( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=argparse.Namespace( + repository='repo', + archive=None, + paths=None, + short=True, + format=None, + json=None, + find_paths=['foo.txt'], + **altered_filter_flags, + ), )