psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Store configs used to create an archive in the archive and add borgmatic bootstrap (#697).

Browse source 
Merge pull request #71 from diivi/feat/store-config-in-archive
This commit is contained in:
Dan Helfman 2023-06-10 14:39:53 -07:00 committed by GitHub
commit ef409ad23c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 615 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

84
borgmatic/actions/config/bootstrap.py Normal file
View file

@ -0,0 +1,84 @@
import json
import logging
import os
import borgmatic.borg.extract
import borgmatic.borg.rlist
import borgmatic.config.validate
import borgmatic.hooks.command
from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
logger = logging.getLogger(__name__)
def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version):
'''
Given:
The bootstrap arguments, which include the repository and archive name, borgmatic source directory,
destination directory, and whether to strip components.
The global arguments, which include the dry run flag
and the local borg version,
Return:
The config paths from the manifest.json file in the borgmatic source directory after extracting it from the
repository.
'''
borgmatic_source_directory = (
bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY
)
borgmatic_manifest_path = os.path.expanduser(
os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
)
extract_process = borgmatic.borg.extract.extract_archive(
global_arguments.dry_run,
bootstrap_arguments.repository,
borgmatic.borg.rlist.resolve_archive_name(
bootstrap_arguments.repository,
bootstrap_arguments.archive,
{},
local_borg_version,
global_arguments,
),
[borgmatic_manifest_path],
{},
{},
local_borg_version,
global_arguments,
extract_to_stdout=True,
)
manifest_data = json.loads(extract_process.stdout.read())
return manifest_data['config_paths']
def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
'''
Run the "bootstrap" action for the given repository.
'''
manifest_config_paths = get_config_paths(
bootstrap_arguments, global_arguments, local_borg_version
)
for config_path in manifest_config_paths:
logger.info('Bootstrapping config path %s', config_path)
borgmatic.borg.extract.extract_archive(
global_arguments.dry_run,
bootstrap_arguments.repository,
borgmatic.borg.rlist.resolve_archive_name(
bootstrap_arguments.repository,
bootstrap_arguments.archive,
{},
local_borg_version,
global_arguments,
),
[config_path],
{},
{},
local_borg_version,
global_arguments,
extract_to_stdout=False,
destination_path=bootstrap_arguments.destination,
strip_components=bootstrap_arguments.strip_components,
progress=bootstrap_arguments.progress,
)

39
borgmatic/actions/create.py
View file

@ -1,15 +1,51 @@
import json
import logging
import os
try:
import importlib_metadata
except ModuleNotFoundError: # pragma: nocover
import importlib.metadata as importlib_metadata
import borgmatic.borg.create
import borgmatic.config.validate
import borgmatic.hooks.command
import borgmatic.hooks.dispatch
import borgmatic.hooks.dump
from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
logger = logging.getLogger(__name__)
def create_borgmatic_manifest(location, config_paths, dry_run):
'''
Create a borgmatic manifest file to store the paths to the configuration files used to create
the archive.
'''
if dry_run:
return
borgmatic_source_directory = location.get(
'borgmatic_source_directory', DEFAULT_BORGMATIC_SOURCE_DIRECTORY
)
borgmatic_manifest_path = os.path.expanduser(
os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
)
if not os.path.exists(borgmatic_manifest_path):
os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True)
with open(borgmatic_manifest_path, 'w') as config_list_file:
json.dump(
{
'borgmatic_version': importlib_metadata.version('borgmatic'),
'config_paths': config_paths,
},
config_list_file,
)
def run_create(
config_filename,
repository,
@ -59,6 +95,9 @@ def run_create(
location,
global_arguments.dry_run,
)
create_borgmatic_manifest(
location, global_arguments.used_config_paths, global_arguments.dry_run
)
stream_processes = [process for processes in active_dumps.values() for process in processes]
json_output = borgmatic.borg.create.create_archive(

4
borgmatic/borg/create.py
View file

@ -351,7 +351,9 @@ def create_archive(
sources = deduplicate_directories(
map_directories_to_devices(
expand_directories(
tuple(location_config.get('source_directories', ())) + borgmatic_source_directories
tuple(location_config.get('source_directories', ()))
+ borgmatic_source_directories
+ tuple(global_arguments.used_config_paths)
)
),
additional_directory_devices=map_directories_to_devices(

158
borgmatic/commands/arguments.py
View file

@ -1,4 +1,6 @@
import argparse
import collections
import itertools
from argparse import Action, ArgumentParser
from borgmatic.config import collect
@ -9,6 +11,8 @@ SUBPARSER_ALIASES = {
'compact': [],
'create': ['-C'],
'check': ['-k'],
'config': [],
'config_bootstrap': [],
'extract': ['-x'],
'export-tar': [],
'mount': ['-m'],
@ -24,6 +28,27 @@ SUBPARSER_ALIASES = {
}
def get_unparsable_arguments(remaining_subparser_arguments):
'''
Determine the remaining arguments that no subparsers have consumed.
'''
if remaining_subparser_arguments:
remaining_arguments = [
argument
for argument in dict.fromkeys(
itertools.chain.from_iterable(remaining_subparser_arguments)
).keys()
if all(
argument in subparser_arguments
for subparser_arguments in remaining_subparser_arguments
)
]
else:
remaining_arguments = []
return remaining_arguments
def parse_subparser_arguments(unparsed_arguments, subparsers):
'''
Given a sequence of arguments and a dict from subparser name to argparse.ArgumentParser
@ -40,6 +65,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
for subparser_name, aliases in SUBPARSER_ALIASES.items()
for alias in aliases
}
subcommand_parsers_mapping = {
'config': ['bootstrap'],
}
# If the "borg" action is used, skip all other subparsers. This avoids confusion like
# "borg list" triggering borgmatic's own list action.
@ -56,7 +84,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
# If a parsed value happens to be the same as the name of a subparser, remove it from the
# remaining arguments. This prevents, for instance, "check --only extract" from triggering
# the "extract" subparser.
parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
parsed, unused_remaining = subparser.parse_known_args(
[argument for argument in unparsed_arguments if argument != canonical_name]
)
for value in vars(parsed).values():
if isinstance(value, str):
if value in subparsers:
@ -66,7 +96,16 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
if item in subparsers:
remaining_arguments.remove(item)
arguments[canonical_name] = parsed
arguments[canonical_name] = None if canonical_name in subcommand_parsers_mapping else parsed
for argument in arguments:
if not arguments[argument]:
if not any(
subcommand in arguments for subcommand in subcommand_parsers_mapping[argument]
):
raise ValueError(
f'Missing subcommand for {argument}. Expected one of {subcommand_parsers_mapping[argument]}'
)
# If no actions are explicitly requested, assume defaults.
if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
@ -77,13 +116,22 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
remaining_arguments = list(unparsed_arguments)
# Now ask each subparser, one by one, to greedily consume arguments.
for subparser_name, subparser in subparsers.items():
# Now ask each subparser, one by one, to greedily consume arguments, from last to first. This
# allows subparsers to consume arguments before their parent subparsers do.
remaining_subparser_arguments = []
for subparser_name, subparser in reversed(subparsers.items()):
if subparser_name not in arguments.keys():
continue
subparser = subparsers[subparser_name]
unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
unused_parsed, remaining = subparser.parse_known_args(
[argument for argument in unparsed_arguments if argument != subparser_name]
)
remaining_subparser_arguments.append(remaining)
if remaining_subparser_arguments:
remaining_arguments = get_unparsable_arguments(remaining_subparser_arguments)
# Special case: If "borg" is present in the arguments, consume all arguments after (+1) the
# "borg" action.
@ -109,7 +157,7 @@ class Extend_action(Action):
items = getattr(namespace, self.dest, None)
if items:
items.extend(values)
items.extend(values) # pragma: no cover
else:
setattr(namespace, self.dest, list(values))
@ -563,6 +611,71 @@ def make_parsers():
'-h', '--help', action='help', help='Show this help message and exit'
)
config_parser = subparsers.add_parser(
'config',
aliases=SUBPARSER_ALIASES['config'],
help='Perform configuration file related operations',
description='Perform configuration file related operations',
add_help=False,
)
config_group = config_parser.add_argument_group('config arguments')
config_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
config_subparsers = config_parser.add_subparsers(
title='config subcommands',
description='Valid subcommands for config',
help='Additional help',
)
config_bootstrap_parser = config_subparsers.add_parser(
'bootstrap',
aliases=SUBPARSER_ALIASES['config_bootstrap'],
help='Extract the config files used to create a borgmatic repository',
description='Extract config files that were used to create a borgmatic repository during the "create" operation',
add_help=False,
)
config_bootstrap_group = config_bootstrap_parser.add_argument_group(
'config bootstrap arguments'
)
config_bootstrap_group.add_argument(
'--repository',
help='Path of repository to extract config files from',
required=True,
)
config_bootstrap_group.add_argument(
'--borgmatic-source-directory',
help='Path that stores the config files used to create an archive and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic',
)
config_bootstrap_group.add_argument(
'--archive',
help='Name of archive to extract config files from, defaults to "latest"',
default='latest',
)
config_bootstrap_group.add_argument(
'--destination',
metavar='PATH',
dest='destination',
help='Directory to extract config files into, defaults to /',
default='/',
)
config_bootstrap_group.add_argument(
'--strip-components',
type=lambda number: number if number == 'all' else int(number),
metavar='NUMBER',
help='Number of leading path components to remove from each extracted path or "all" to strip all leading path components. Skip paths with fewer elements',
)
config_bootstrap_group.add_argument(
'--progress',
dest='progress',
default=False,
action='store_true',
help='Display progress for each file as it is extracted',
)
config_bootstrap_group.add_argument(
'-h', '--help', action='help', help='Show this help message and exit'
)
export_tar_parser = subparsers.add_parser(
'export-tar',
aliases=SUBPARSER_ALIASES['export-tar'],
@ -973,7 +1086,28 @@ def make_parsers():
)
borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
return top_level_parser, subparsers
merged_subparsers = argparse._SubParsersAction(
None, None, metavar=None, dest='merged', parser_class=None
)
merged_subparsers = merge_subparsers(subparsers, config_subparsers)
return top_level_parser, merged_subparsers
def merge_subparsers(*subparsers):
'''
Merge multiple subparsers into a single subparser.
'''
merged_subparsers = argparse._SubParsersAction(
None, None, metavar=None, dest='merged', parser_class=None
)
for subparser in subparsers:
for name, subparser in subparser.choices.items():
merged_subparsers._name_parser_map[name] = subparser
return merged_subparsers
def parse_arguments(*unparsed_arguments):
@ -986,6 +1120,16 @@ def parse_arguments(*unparsed_arguments):
arguments, remaining_arguments = parse_subparser_arguments(
unparsed_arguments, subparsers.choices
)
if (
'bootstrap' in arguments.keys()
and 'config' in arguments.keys()
and len(arguments.keys()) > 2
):
raise ValueError(
'The bootstrap action cannot be combined with other actions. Please run it separately.'
)
arguments['global'] = top_level_parser.parse_args(remaining_arguments)
if arguments['global'].excludes_filename:

30
borgmatic/commands/borgmatic.py
View file

@ -18,6 +18,7 @@ import borgmatic.actions.borg
import borgmatic.actions.break_lock
import borgmatic.actions.check
import borgmatic.actions.compact
import borgmatic.actions.config.bootstrap
import borgmatic.actions.create
import borgmatic.actions.export_tar
import borgmatic.actions.extract
@ -622,11 +623,37 @@ def collect_configuration_run_summary_logs(configs, arguments):
if 'extract' in arguments or 'mount' in arguments:
validate.guard_single_repository_selected(repository, configs)
validate.guard_configuration_contains_repository(repository, configs)
if 'bootstrap' not in arguments:
validate.guard_configuration_contains_repository(repository, configs)
except ValueError as error:
yield from log_error_records(str(error))
return
if 'bootstrap' in arguments:
# no configuration file is needed for bootstrap
local_borg_version = borg_version.local_borg_version({}, 'borg')
try:
borgmatic.actions.config.bootstrap.run_bootstrap(
arguments['bootstrap'], arguments['global'], local_borg_version
)
yield logging.makeLogRecord(
dict(
levelno=logging.INFO,
levelname='INFO',
msg='Bootstrap successful',
)
)
except (
CalledProcessError,
ValueError,
OSError,
json.JSONDecodeError,
KeyError,
) as error:
yield from log_error_records('Error running bootstrap', error)
return
if not configs:
yield from log_error_records(
f"{' '.join(arguments['global'].config_paths)}: No valid configuration files found",
@ -733,6 +760,7 @@ def main(): # pragma: no cover
sys.exit(0)
config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
global_arguments.used_config_paths = list(config_filenames)
configs, parse_logs = load_configurations(
config_filenames, global_arguments.overrides, global_arguments.resolve_env
)

32
tests/integration/commands/test_arguments.py
View file

@ -1,3 +1,5 @@
import argparse
import pytest
from flexmock import flexmock
@ -298,6 +300,13 @@ def test_parse_arguments_disallows_paths_unless_action_consumes_it():
module.parse_arguments('--config', 'myconfig', '--path', 'test')
def test_parse_arguments_disallows_other_actions_with_config_bootstrap():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('config', 'bootstrap', '--repository', 'test.borg', 'list')
def test_parse_arguments_allows_archive_with_extract():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
@ -523,3 +532,26 @@ def test_parse_arguments_extract_with_check_only_extract_does_not_raise():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract')
def test_merging_two_subparser_collections_merges_their_choices():
top_level_parser = argparse.ArgumentParser()
subparsers = top_level_parser.add_subparsers()
subparser1 = subparsers.add_parser('subparser1')
subparser2 = subparsers.add_parser('subparser2')
subsubparsers = subparser2.add_subparsers()
subsubparser1 = subsubparsers.add_parser('subsubparser1')
merged_subparsers = argparse._SubParsersAction(
None, None, metavar=None, dest='merged', parser_class=None
)
merged_subparsers = module.merge_subparsers(subparsers, subsubparsers)
assert merged_subparsers.choices == {
'subparser1': subparser1,
'subparser2': subparser2,
'subsubparser1': subsubparser1,
}

56
tests/unit/actions/config/test_bootstrap.py Normal file
View file

@ -0,0 +1,56 @@
from flexmock import flexmock
from borgmatic.actions.config import bootstrap as module
def test_get_config_paths_returns_list_of_config_paths():
bootstrap_arguments = flexmock(
borgmatic_source_directory=None,
repository='repo',
archive='archive',
)
global_arguments = flexmock(
dry_run=False,
)
local_borg_version = flexmock()
extract_process = flexmock(
stdout=flexmock(
read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}',
),
)
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
extract_process
)
flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
'archive'
)
assert module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) == [
'/borgmatic/config.yaml'
]
def test_run_bootstrap_does_not_raise():
bootstrap_arguments = flexmock(
repository='repo',
archive='archive',
destination='dest',
strip_components=1,
progress=False,
borgmatic_source_directory='/borgmatic',
)
global_arguments = flexmock(
dry_run=False,
)
local_borg_version = flexmock()
extract_process = flexmock(
stdout=flexmock(
read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}',
),
)
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
extract_process
).twice()
flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
'archive'
)
module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version)

50
tests/unit/actions/test_create.py
View file

@ -1,3 +1,5 @@
import sys
from flexmock import flexmock
from borgmatic.actions import create as module
@ -7,6 +9,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
flexmock(module.logger).answer = lambda message: None
flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
flexmock(module.borgmatic.borg.create).should_receive('create_archive').once()
flexmock(module).should_receive('create_borgmatic_manifest').once()
flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({})
flexmock(module.borgmatic.hooks.dispatch).should_receive(
@ -19,7 +22,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
json=flexmock(),
list_files=flexmock(),
)
global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[])
list(
module.run_create(
@ -45,6 +48,7 @@ def test_run_create_runs_with_selected_repository():
'repositories_match'
).once().and_return(True)
flexmock(module.borgmatic.borg.create).should_receive('create_archive').once()
flexmock(module).should_receive('create_borgmatic_manifest').once()
create_arguments = flexmock(
repository=flexmock(),
progress=flexmock(),
@ -52,7 +56,7 @@ def test_run_create_runs_with_selected_repository():
json=flexmock(),
list_files=flexmock(),
)
global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[])
list(
module.run_create(
@ -78,6 +82,7 @@ def test_run_create_bails_if_repository_does_not_match():
'repositories_match'
).once().and_return(False)
flexmock(module.borgmatic.borg.create).should_receive('create_archive').never()
flexmock(module).should_receive('create_borgmatic_manifest').never()
create_arguments = flexmock(
repository=flexmock(),
progress=flexmock(),
@ -85,7 +90,7 @@ def test_run_create_bails_if_repository_does_not_match():
json=flexmock(),
list_files=flexmock(),
)
global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[])
list(
module.run_create(
@ -103,3 +108,42 @@ def test_run_create_bails_if_repository_does_not_match():
remote_path=None,
)
)
def test_create_borgmatic_manifest_creates_manifest_file():
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.os).should_receive('makedirs').and_return(True)
flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0')
flexmock(module.json).should_receive('dump').and_return(True)
module.create_borgmatic_manifest({}, 'test.yaml', False)
def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_source_directory():
flexmock(module.os.path).should_receive('join').with_args(
'/borgmatic', 'bootstrap', 'manifest.json'
).and_return('/borgmatic/bootstrap/manifest.json')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.os).should_receive('makedirs').and_return(True)
flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0')
flexmock(sys.modules['builtins']).should_receive('open').with_args(
'/borgmatic/bootstrap/manifest.json', 'w'
).and_return(
flexmock(
__enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None),
__exit__=lambda *args: None,
)
)
flexmock(module.json).should_receive('dump').and_return(True)
module.create_borgmatic_manifest(
{'borgmatic_source_directory': '/borgmatic'}, 'test.yaml', False
)
def test_create_borgmatic_manifest_does_not_create_manifest_file_on_dry_run():
flexmock(module.os.path).should_receive('expanduser').never()
module.create_borgmatic_manifest({}, 'test.yaml', True)

143
tests/unit/borg/test_create.py
View file

@ -492,7 +492,7 @@ def test_create_archive_calls_borg_with_parameters():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -536,7 +536,7 @@ def test_create_archive_calls_borg_with_environment():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -582,7 +582,56 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sources_and_config_paths():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(
('foo', 'bar', '/etc/borgmatic/config.yaml')
)
flexmock(module).should_receive('map_directories_to_devices').and_return({})
flexmock(module).should_receive('expand_directories').with_args([]).and_return(())
flexmock(module).should_receive('expand_directories').with_args(
('foo', 'bar', '/etc/borgmatic/config.yaml')
).and_return(('foo', 'bar', '/etc/borgmatic/config.yaml'))
flexmock(module).should_receive('expand_directories').with_args([]).and_return(())
flexmock(module).should_receive('pattern_root_directories').and_return([])
flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
flexmock(module).should_receive('expand_home_directories').and_return(())
flexmock(module).should_receive('write_pattern_file').and_return(None)
flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module).should_receive('ensure_files_readable')
flexmock(module).should_receive('make_pattern_flags').and_return(())
flexmock(module).should_receive('make_exclude_flags').and_return(())
flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
(f'repo::{DEFAULT_ARCHIVE_NAME}',)
)
environment = {'BORG_THINGY': 'YUP'}
flexmock(module.environment).should_receive('make_environment').and_return(environment)
flexmock(module).should_receive('execute_command').with_args(
('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('/etc/borgmatic/config.yaml',),
output_log_level=logging.INFO,
output_file=None,
borg_local_path='borg',
working_directory=None,
extra_environment=environment,
)
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=['/etc/borgmatic/config.yaml']),
)
@ -628,7 +677,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -672,7 +721,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -713,7 +762,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
json=True,
)
@ -758,7 +807,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -799,7 +848,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
json=True,
)
@ -843,7 +892,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -889,7 +938,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stats=True,
)
@ -933,7 +982,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
},
storage_config={'checkpoint_interval': 600},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -976,7 +1025,7 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume
},
storage_config={'checkpoint_volume': 1024},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1019,7 +1068,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
},
storage_config={'chunker_params': '1,2,3,4'},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1062,7 +1111,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
},
storage_config={'compression': 'rle'},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1111,7 +1160,7 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_
},
storage_config={'upload_rate_limit': 100},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1157,7 +1206,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1201,7 +1250,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1251,7 +1300,7 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter(
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1305,7 +1354,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1361,7 +1410,7 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1416,7 +1465,7 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1471,7 +1520,7 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1515,7 +1564,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1558,7 +1607,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
local_path='borg1',
)
@ -1602,7 +1651,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
remote_path='borg1',
)
@ -1646,7 +1695,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
},
storage_config={'umask': 740},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1689,7 +1738,7 @@ def test_create_archive_with_log_json_calls_borg_with_log_json_parameters():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=True),
global_arguments=flexmock(log_json=True, used_config_paths=[]),
)
@ -1732,7 +1781,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
},
storage_config={'lock_wait': 5},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1775,7 +1824,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stats=True,
)
@ -1819,7 +1868,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
list_files=True,
)
@ -1869,7 +1918,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
progress=True,
)
@ -1913,7 +1962,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
progress=True,
)
@ -1974,7 +2023,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
progress=True,
stream_processes=processes,
)
@ -2039,7 +2088,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stream_processes=processes,
)
@ -2107,7 +2156,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stream_processes=processes,
)
@ -2172,7 +2221,7 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stream_processes=processes,
)
@ -2213,7 +2262,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
json=True,
)
@ -2256,7 +2305,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
json=True,
stats=True,
)
@ -2304,7 +2353,7 @@ def test_create_archive_with_source_directories_glob_expands():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2348,7 +2397,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2391,7 +2440,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2434,7 +2483,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
},
storage_config={'archive_name_format': 'ARCHIVE_NAME'},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2478,7 +2527,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
},
storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2522,7 +2571,7 @@ def test_create_archive_with_repository_accepts_borg_placeholders():
},
storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2565,7 +2614,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
},
storage_config={'extra_borg_options': {'create': '--extra --options'}},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2626,7 +2675,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stream_processes=processes,
)
@ -2652,7 +2701,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)

43
tests/unit/commands/test_arguments.py
View file

@ -1,5 +1,6 @@
import collections
import pytest
from flexmock import flexmock
from borgmatic.commands import arguments as module
@ -164,3 +165,45 @@ def test_parse_subparser_arguments_parses_borg_options_and_skips_other_subparser
assert arguments == {'borg': action_namespace}
assert arguments['borg'].options == ['list']
assert remaining_arguments == []
def test_parse_subparser_arguments_raises_error_when_no_subparser_is_specified():
action_namespace = flexmock(options=[])
subparsers = {
'config': flexmock(parse_known_args=lambda arguments: (action_namespace, ['config'])),
}
with pytest.raises(ValueError):
module.parse_subparser_arguments(('config',), subparsers)
@pytest.mark.parametrize(
'arguments, expected',
[
(
(
('--latest', 'archive', 'prune', 'extract', 'list', '--test-flag'),
('--latest', 'archive', 'check', 'extract', 'list', '--test-flag'),
('prune', 'check', 'list', '--test-flag'),
('prune', 'check', 'extract', '--test-flag'),
),
[
'--test-flag',
],
),
(
(
('--latest', 'archive', 'prune', 'extract', 'list'),
('--latest', 'archive', 'check', 'extract', 'list'),
('prune', 'check', 'list'),
('prune', 'check', 'extract'),
),
[],
),
((), []),
],
)
def test_get_unparsable_arguments_returns_remaining_arguments_that_no_subparser_can_parse(
arguments, expected
):
assert module.get_unparsable_arguments(arguments) == expected

35
tests/unit/commands/test_borgmatic.py
View file

@ -1000,6 +1000,41 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
assert {log.levelno for log in logs} == {logging.INFO}
def test_collect_configuration_run_summary_logs_info_for_success_with_bootstrap():
flexmock(module.validate).should_receive('guard_single_repository_selected').never()
flexmock(module.validate).should_receive('guard_configuration_contains_repository').never()
flexmock(module).should_receive('run_configuration').never()
flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap')
arguments = {
'bootstrap': flexmock(repository='repo'),
'global': flexmock(monitoring_verbosity=1, dry_run=False),
}
logs = tuple(
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert {log.levelno for log in logs} == {logging.INFO}
def test_collect_configuration_run_summary_logs_error_on_bootstrap_failure():
flexmock(module.validate).should_receive('guard_single_repository_selected').never()
flexmock(module.validate).should_receive('guard_configuration_contains_repository').never()
flexmock(module).should_receive('run_configuration').never()
flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').and_raise(
ValueError
)
arguments = {
'bootstrap': flexmock(repository='repo'),
'global': flexmock(monitoring_verbosity=1, dry_run=False),
}
logs = tuple(
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert {log.levelno for log in logs} == {logging.CRITICAL}
def test_collect_configuration_run_summary_logs_extract_with_repository_error():
flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
ValueError