List the files within an archive via --list --archive option (#140).

This commit is contained in:
Dan Helfman 2019-02-23 23:02:17 -08:00
parent 26071de2e7
commit 4272c6b077
8 changed files with 153 additions and 55 deletions

3
NEWS
View file

@ -1,3 +1,6 @@
1.2.17
* #140: List the files within an archive via --list --archive option.
1.2.16
* #119: Include a sample borgmatic configuration file in the documentation.
* #123: Support for Borg archive restoration via borgmatic --extract command-line flag.

View file

@ -5,15 +5,17 @@ import subprocess
logger = logging.getLogger(__name__)
def list_archives(repository, storage_config, local_path='borg', remote_path=None, json=False):
def list_archives(
repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False
):
'''
Given a local or remote repository path, and a storage config dict,
list Borg archives in the repository.
Given a local or remote repository path and a storage config dict, list Borg archives in the
repository. Or, if an archive name is given, list the files in that archive.
'''
lock_wait = storage_config.get('lock_wait', None)
full_command = (
(local_path, 'list', repository)
(local_path, 'list', '::'.join((repository, archive)) if archive else repository)
+ (('--remote-path', remote_path) if remote_path else ())
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())

View file

@ -102,27 +102,49 @@ def parse_arguments(*arguments):
help='Create a repository with a fixed storage quota',
)
prune_group = parser.add_argument_group('options for --prune')
stats_argument = prune_group.add_argument(
'--stats',
dest='stats',
default=False,
action='store_true',
help='Display statistics of archive',
)
create_group = parser.add_argument_group('options for --create')
create_group.add_argument(
progress_argument = create_group.add_argument(
'--progress',
dest='progress',
default=False,
action='store_true',
help='Display progress for each file as it is backed up',
help='Display progress for each file as it is processed',
)
create_group._group_actions.append(stats_argument)
json_argument = create_group.add_argument(
'--json', dest='json', default=False, action='store_true', help='Output results as JSON'
)
extract_group = parser.add_argument_group('options for --extract')
extract_group.add_argument(
repository_argument = extract_group.add_argument(
'--repository',
help='Path of repository to restore from, defaults to the configured repository if there is only one',
help='Path of repository to use, defaults to the configured repository if there is only one',
)
extract_group.add_argument('--archive', help='Name of archive to restore')
archive_argument = extract_group.add_argument('--archive', help='Name of archive to operate on')
extract_group.add_argument(
'--restore-path',
nargs='+',
dest='restore_paths',
help='Paths to restore from archive, defaults to the entire archive',
)
extract_group._group_actions.append(progress_argument)
list_group = parser.add_argument_group('options for --list')
list_group._group_actions.append(repository_argument)
list_group._group_actions.append(archive_argument)
list_group._group_actions.append(json_argument)
info_group = parser.add_argument_group('options for --info')
info_group._group_actions.append(json_argument)
common_group = parser.add_argument_group('common options')
common_group.add_argument(
@ -140,20 +162,6 @@ def parse_arguments(*arguments):
dest='excludes_filename',
help='Deprecated in favor of exclude_patterns within configuration',
)
common_group.add_argument(
'--stats',
dest='stats',
default=False,
action='store_true',
help='Display statistics of archive with --create or --prune option',
)
common_group.add_argument(
'--json',
dest='json',
default=False,
action='store_true',
help='Output results from the --create, --list, or --info options as json',
)
common_group.add_argument(
'-n',
'--dry-run',
@ -196,10 +204,15 @@ def parse_arguments(*arguments):
raise ValueError('The --encryption option is required with the --init option')
if not args.extract:
if args.repository:
raise ValueError('The --repository option can only be used with the --extract option')
if args.archive:
raise ValueError('The --archive option can only be used with the --extract option')
if not args.list:
if args.repository:
raise ValueError(
'The --repository option can only be used with the --extract and --list options'
)
if args.archive:
raise ValueError(
'The --archive option can only be used with the --extract and --list options'
)
if args.restore_paths:
raise ValueError('The --restore-path option can only be used with the --extract option')
if args.extract and not args.archive:
@ -360,14 +373,20 @@ def _run_commands_on_repository(
progress=args.progress,
)
if args.list:
logger.info('{}: Listing archives'.format(repository))
output = borg_list.list_archives(
repository, storage, local_path=local_path, remote_path=remote_path, json=args.json
)
if args.json:
json_results.append(json.loads(output))
else:
sys.stdout.write(output)
if args.repository is None or repository == args.repository:
logger.info('{}: Listing archives'.format(repository))
output = borg_list.list_archives(
repository,
storage,
args.archive,
local_path=local_path,
remote_path=remote_path,
json=args.json,
)
if args.json:
json_results.append(json.loads(output))
else:
sys.stdout.write(output)
if args.info:
logger.info('{}: Displaying summary info for archives'.format(repository))
output = borg_info.display_archives_info(
@ -388,6 +407,7 @@ def collect_configuration_run_summary_logs(config_filenames, args):
# Dict mapping from config filename to corresponding parsed config dict.
configs = collections.OrderedDict()
# Parse and load each configuration file.
for config_filename in config_filenames:
try:
logger.info('{}: Parsing configuration file'.format(config_filename))
@ -403,13 +423,15 @@ def collect_configuration_run_summary_logs(config_filenames, args):
)
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
if args.extract:
# Run cross-file validation checks.
if args.extract or (args.list and args.archive):
try:
validate.guard_configuration_contains_repository(args.repository, configs)
except ValueError as error:
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
return
# Execute the actions corresponding to each configuration file.
for config_filename, config in configs.items():
try:
run_configuration(config_filename, config, args)

View file

@ -130,7 +130,7 @@ def guard_configuration_contains_repository(repository, configurations):
if count > 1:
raise ValueError(
'Can\'t determine which repository to extract. Use --repository option to disambiguate'.format(
'Can\'t determine which repository to use. Use --repository option to disambiguate'.format(
repository
)
)

View file

@ -1,7 +1,7 @@
from setuptools import setup, find_packages
VERSION = '1.2.16'
VERSION = '1.2.17'
setup(

View file

@ -142,14 +142,28 @@ def test_parse_arguments_disallows_init_and_dry_run():
)
def test_parse_arguments_disallows_repository_without_extract():
def test_parse_arguments_disallows_repository_without_extract_or_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg')
def test_parse_arguments_disallows_archive_without_extract():
def test_parse_arguments_allows_repository_with_extract():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments(
'--config', 'myconfig', '--extract', '--repository', 'test.borg', '--archive', 'test'
)
def test_parse_arguments_allows_repository_with_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--config', 'myconfig', '--list', '--repository', 'test.borg')
def test_parse_arguments_disallows_archive_without_extract_or_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
@ -169,6 +183,12 @@ def test_parse_arguments_allows_archive_with_extract():
module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test')
def test_parse_arguments_allows_archive_with_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--config', 'myconfig', '--list', '--archive', 'test')
def test_parse_arguments_requires_archive_with_extract():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
@ -177,51 +197,73 @@ def test_parse_arguments_requires_archive_with_extract():
def test_parse_arguments_allows_progress_and_create():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--progress', '--create', '--list')
def test_parse_arguments_allows_progress_and_extract():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--progress', '--extract', '--archive', 'test', '--list')
def test_parse_arguments_disallows_progress_without_create():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--progress', '--list')
def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--stats', '--create', '--list')
def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--stats', '--prune', '--list')
def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--stats', '--list')
def test_parse_arguments_with_just_stats_flag_does_not_raise():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--stats')
def test_parse_arguments_allows_json_with_list_or_info():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--list', '--json')
module.parse_arguments('--info', '--json')
def test_parse_arguments_disallows_json_without_list_or_info():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--json')
def test_parse_arguments_disallows_json_with_both_list_and_info():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--list', '--info', '--json')
def test_borgmatic_version_matches_news_version():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
borgmatic_version = subprocess.check_output(('borgmatic', '--version')).decode('ascii')
news_version = open('NEWS').readline()

View file

@ -34,10 +34,18 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
module.list_archives(repository='repo', storage_config={})
def test_list_archives_with_json_calls_borg_with_json_parameter():
insert_subprocess_mock(LIST_COMMAND + ('--json',))
def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
storage_config = {'lock_wait': 5}
insert_subprocess_mock(LIST_COMMAND + ('--lock-wait', '5'))
module.list_archives(repository='repo', storage_config={}, json=True)
module.list_archives(repository='repo', storage_config=storage_config)
def test_list_archives_with_archive_calls_borg_with_archive_parameter():
storage_config = {}
insert_subprocess_mock(('borg', 'list', 'repo::archive'))
module.list_archives(repository='repo', storage_config=storage_config, archive='archive')
def test_list_archives_with_local_path_calls_borg_via_local_path():
@ -52,8 +60,7 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters()
module.list_archives(repository='repo', storage_config={}, remote_path='borg1')
def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
storage_config = {'lock_wait': 5}
insert_subprocess_mock(LIST_COMMAND + ('--lock-wait', '5'))
def test_list_archives_with_json_calls_borg_with_json_parameter():
insert_subprocess_mock(LIST_COMMAND + ('--json',))
module.list_archives(repository='repo', storage_config=storage_config)
module.list_archives(repository='repo', storage_config={}, json=True)

View file

@ -50,22 +50,22 @@ def test_run_commands_handles_multiple_json_outputs_in_array():
def test_collect_configuration_run_summary_logs_info_for_success():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module).should_receive('run_configuration')
args = flexmock(extract=False)
args = flexmock(extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.INFO)
assert all(log for log in logs if log.levelno == module.logging.INFO)
def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module.validate).should_receive('guard_configuration_contains_repository')
flexmock(module).should_receive('run_configuration')
args = flexmock(extract=True, repository='repo')
args = flexmock(extract=True, list=False, repository='repo')
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.INFO)
assert all(log for log in logs if log.levelno == module.logging.INFO)
def test_collect_configuration_run_summary_logs_critical_for_extract_with_repository_error():
@ -73,16 +73,38 @@ def test_collect_configuration_run_summary_logs_critical_for_extract_with_reposi
flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
ValueError
)
args = flexmock(extract=True, repository='repo')
args = flexmock(extract=True, list=False, repository='repo')
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_and_repository_error():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
ValueError
)
args = flexmock(extract=False, list=True, repository='repo', archive='test')
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
def test_collect_configuration_run_summary_logs_info_for_success_with_list():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module).should_receive('run_configuration')
args = flexmock(extract=False, list=True, repository='repo', archive=None)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert all(log for log in logs if log.levelno == module.logging.INFO)
def test_collect_configuration_run_summary_logs_critical_for_parse_error():
flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError)
args = flexmock(extract=False)
args = flexmock(extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
@ -93,7 +115,7 @@ def test_collect_configuration_run_summary_logs_critical_for_run_error():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module.validate).should_receive('guard_configuration_contains_repository')
flexmock(module).should_receive('run_configuration').and_raise(ValueError)
args = flexmock(extract=False)
args = flexmock(extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
@ -103,7 +125,7 @@ def test_collect_configuration_run_summary_logs_critical_for_run_error():
def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module).should_receive('run_configuration')
args = flexmock(config_paths=(), extract=False)
args = flexmock(config_paths=(), extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(config_filenames=(), args=args))