diff --git a/.drone.yml b/.drone.yml index cf4c358..9a7630a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,6 +24,8 @@ clone: steps: - name: build image: alpine:3.13 + environment: + TEST_CONTAINER: true pull: always commands: - scripts/run-full-tests diff --git a/.eleventy.js b/.eleventy.js index bde54af..30057a4 100644 --- a/.eleventy.js +++ b/.eleventy.js @@ -1,4 +1,5 @@ const pluginSyntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight"); +const codeClipboard = require("eleventy-plugin-code-clipboard"); const inclusiveLangPlugin = require("@11ty/eleventy-plugin-inclusive-language"); const navigationPlugin = require("@11ty/eleventy-navigation"); @@ -6,6 +7,7 @@ module.exports = function(eleventyConfig) { eleventyConfig.addPlugin(pluginSyntaxHighlight); eleventyConfig.addPlugin(inclusiveLangPlugin); eleventyConfig.addPlugin(navigationPlugin); + eleventyConfig.addPlugin(codeClipboard); let markdownIt = require("markdown-it"); let markdownItAnchor = require("markdown-it-anchor"); @@ -31,6 +33,7 @@ module.exports = function(eleventyConfig) { markdownIt(markdownItOptions) .use(markdownItAnchor, markdownItAnchorOptions) .use(markdownItReplaceLink) + .use(codeClipboard.markdownItCopyButton) ); eleventyConfig.addPassthroughCopy({"docs/static": "static"}); diff --git a/NEWS b/NEWS index 0bb47e2..4a27d07 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,66 @@ -1.7.10.dev0 +1.7.13.dev0 + * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" + flag. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas + +1.7.12 + * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. + See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ + * #666, #670: Fix error when running the "info" action with the "--match-archives" or "--archive" + flags. Also fix the "--match-archives"/"--archive" flags to correctly override the + "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. + * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix" + options set. + * #672: Selectively shallow merge certain mappings or sequences when including configuration files. + See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge + * #672: Selectively omit list values when including configuration files. See the documentation for + more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#list-merge + * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag. + See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes + * Add optional support for running end-to-end tests and building documentation with rootless Podman + instead of Docker. + +1.7.11 + * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives + get used for borgmatic actions that operate on multiple archives. Override this behavior with the + new "match_archives" option in the storage section. This change is "breaking" in that it silently + changes which archives get considered for "rlist", "prune", "check", etc. See the documentation + for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming + * #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format" + auto-matching behavior and the "match_archives" option. + * #658: Add "--log-file-format" flag for customizing the log message format. See the documentation + for more information: + https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#logging-to-file + * #662: Fix regression in which the "check_repositories" option failed to match repositories. + * #663: Fix regression in which the "transfer" action produced a traceback. + * Add spellchecking of source code during test runs. + +1.7.10 + * #396: When a database command errors, display and log the error message instead of swallowing it. * #501: Optionally error if a source directory does not exist via "source_directories_must_exist" option in borgmatic's location configuration. * #576: Add support for "file://" paths within "repositories" option. + * #612: Define and use custom constants in borgmatic configuration files. See the documentation for + more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#constant-interpolation * #618: Add support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option in borgmatic's storage configuration. * #623: Fix confusing message when an error occurs running actions for a configuration file. + * #635: Add optional repository labels so you can select a repository via "--repository yourlabel" + at the command-line. See the configuration reference for more information: + https://torsion.org/borgmatic/docs/reference/configuration/ + * #649: Add documentation on backing up a database running in a container: + https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers + * #655: Fix error when databases are configured and a source directory doesn't exist. + * Add code style plugins to enforce use of Python f-strings and prevent single-letter variables. + To join in the pedantry, refresh your test environment with "tox --recreate". + * Rename scripts/run-full-dev-tests to scripts/run-end-to-end-dev-tests and make it run end-to-end + tests only. Continue using tox to run unit and integration tests. 1.7.9 * #295: Add a SQLite database dump/restore hook. @@ -374,7 +430,7 @@ configuration schema descriptions. 1.5.6 - * #292: Allow before_backup and similiar hooks to exit with a soft failure without altering the + * #292: Allow before_backup and similar hooks to exit with a soft failure without altering the monitoring status on Healthchecks or other providers. Support this by waiting to ping monitoring services with a "start" status until after before_* hooks finish. Failures in before_* hooks still trigger a monitoring "fail" status. @@ -443,7 +499,7 @@ * For "list" and "info" actions, show repository names even at verbosity 0. 1.4.22 - * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON ouput. + * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON output. * After a backup of a database dump in directory format, properly remove the dump directory. * In "borgmatic --help", don't expand $HOME in listing of default "--config" paths. @@ -815,7 +871,7 @@ * #77: Skip non-"*.yaml" config filenames in /etc/borgmatic.d/ so as not to parse backup files, editor swap files, etc. * #81: Document user-defined hooks run before/after backup, or on error. - * Add code style guidelines to the documention. + * Add code style guidelines to the documentation. 1.2.0 * #61: Support for Borg --list option via borgmatic command-line to list all archives. diff --git a/README.md b/README.md index 27fc6cd..eb827ae 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,10 @@ location: # Paths of local or remote repositories to backup to. repositories: - - ssh://1234@usw-s001.rsync.net/./backups.borg - - ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo - - /var/lib/backups/local.borg + - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo + label: borgbase + - path: /var/lib/backups/local.borg + label: local retention: # Retention policy for how many backups to keep. diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index a50dd28..3d2998b 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_borg( - repository, storage, local_borg_version, borg_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + borg_arguments, + local_path, + remote_path, ): ''' Run the "borg" action for the given repository. @@ -16,9 +21,9 @@ def run_borg( if borg_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, borg_arguments.repository ): - logger.info('{}: Running arbitrary Borg command'.format(repository)) + logger.info(f'{repository["path"]}: Running arbitrary Borg command') archive_name = borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], borg_arguments.archive, storage, local_borg_version, @@ -26,7 +31,7 @@ def run_borg( remote_path, ) borgmatic.borg.borg.run_arbitrary_borg( - repository, + repository['path'], storage, local_borg_version, options=borg_arguments.options, diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index 65384d7..2174161 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -7,7 +7,12 @@ logger = logging.getLogger(__name__) def run_break_lock( - repository, storage, local_borg_version, break_lock_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + break_lock_arguments, + local_path, + remote_path, ): ''' Run the "break-lock" action for the given repository. @@ -15,7 +20,11 @@ def run_break_lock( if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, break_lock_arguments.repository ): - logger.info(f'{repository}: Breaking repository and cache locks') + logger.info(f'{repository["path"]}: Breaking repository and cache locks') borgmatic.borg.break_lock.break_lock( - repository, storage, local_borg_version, local_path=local_path, remote_path=remote_path, + repository['path'], + storage, + local_borg_version, + local_path=local_path, + remote_path=remote_path, ) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index f357239..1696e07 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -37,9 +37,9 @@ def run_check( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Running consistency checks'.format(repository)) + logger.info(f'{repository["path"]}: Running consistency checks') borgmatic.borg.check.check_archives( - repository, + repository['path'], location, storage, consistency, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 7a25b82..95334c5 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -39,10 +39,10 @@ def run_compact( **hook_context, ) if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): - logger.info('{}: Compacting segments{}'.format(repository, dry_run_label)) + logger.info(f'{repository["path"]}: Compacting segments{dry_run_label}') borgmatic.borg.compact.compact_segments( global_arguments.dry_run, - repository, + repository['path'], storage, local_borg_version, local_path=local_path, @@ -52,7 +52,7 @@ def run_compact( threshold=compact_arguments.threshold, ) else: # pragma: nocover - logger.info('{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository)) + logger.info(f'{repository["path"]}: Skipping compact (only available/needed in Borg 1.2+)') borgmatic.hooks.command.execute_hook( hooks.get('after_compact'), hooks.get('umask'), diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 96a4852..3fbe31e 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -42,11 +42,11 @@ def run_create( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) + logger.info(f'{repository["path"]}: Creating archive{dry_run_label}') borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, @@ -54,7 +54,7 @@ def run_create( active_dumps = borgmatic.hooks.dispatch.call_hooks( 'dump_databases', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, @@ -63,7 +63,7 @@ def run_create( json_output = borgmatic.borg.create.create_archive( global_arguments.dry_run, - repository, + repository['path'], location, storage, local_borg_version, diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index ae34920..ff9f31b 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -23,13 +23,13 @@ def run_export_tar( repository, export_tar_arguments.repository ): logger.info( - '{}: Exporting archive {} as tar file'.format(repository, export_tar_arguments.archive) + f'{repository["path"]}: Exporting archive {export_tar_arguments.archive} as tar file' ) borgmatic.borg.export_tar.export_tar_archive( global_arguments.dry_run, - repository, + repository['path'], borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], export_tar_arguments.archive, storage, local_borg_version, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index a3d89a5..cc1516c 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -35,12 +35,12 @@ def run_extract( if extract_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, extract_arguments.repository ): - logger.info('{}: Extracting archive {}'.format(repository, extract_arguments.archive)) + logger.info(f'{repository["path"]}: Extracting archive {extract_arguments.archive}') borgmatic.borg.extract.extract_archive( global_arguments.dry_run, - repository, + repository['path'], borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], extract_arguments.archive, storage, local_borg_version, diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index ab4fe42..5402312 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -9,7 +9,12 @@ logger = logging.getLogger(__name__) def run_info( - repository, storage, local_borg_version, info_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + info_arguments, + local_path, + remote_path, ): ''' Run the "info" action for the given repository and archive. @@ -20,9 +25,9 @@ def run_info( repository, info_arguments.repository ): if not info_arguments.json: # pragma: nocover - logger.answer(f'{repository}: Displaying archive summary information') + logger.answer(f'{repository["path"]}: Displaying archive summary information') info_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], info_arguments.archive, storage, local_borg_version, @@ -30,7 +35,7 @@ def run_info( remote_path, ) json_output = borgmatic.borg.info.display_archives_info( - repository, + repository['path'], storage, local_borg_version, info_arguments=info_arguments, diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 78efdf5..359f3b6 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_list( - repository, storage, local_borg_version, list_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + list_arguments, + local_path, + remote_path, ): ''' Run the "list" action for the given repository and archive. @@ -20,11 +25,11 @@ def run_list( ): if not list_arguments.json: # pragma: nocover if list_arguments.find_paths: - logger.answer(f'{repository}: Searching archives') + logger.answer(f'{repository["path"]}: Searching archives') elif not list_arguments.archive: - logger.answer(f'{repository}: Listing archives') + logger.answer(f'{repository["path"]}: Listing archives') list_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], list_arguments.archive, storage, local_borg_version, @@ -32,7 +37,7 @@ def run_list( remote_path, ) json_output = borgmatic.borg.list.list_archive( - repository, + repository['path'], storage, local_borg_version, list_arguments=list_arguments, diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index b1a1132..ec472eb 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_mount( - repository, storage, local_borg_version, mount_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + mount_arguments, + local_path, + remote_path, ): ''' Run the "mount" action for the given repository. @@ -17,14 +22,14 @@ def run_mount( repository, mount_arguments.repository ): if mount_arguments.archive: - logger.info('{}: Mounting archive {}'.format(repository, mount_arguments.archive)) + logger.info(f'{repository["path"]}: Mounting archive {mount_arguments.archive}') else: # pragma: nocover - logger.info('{}: Mounting repository'.format(repository)) + logger.info(f'{repository["path"]}: Mounting repository') borgmatic.borg.mount.mount_archive( - repository, + repository['path'], borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], mount_arguments.archive, storage, local_borg_version, diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index 09666ee..644fd2a 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -37,10 +37,10 @@ def run_prune( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Pruning archives{}'.format(repository, dry_run_label)) + logger.info(f'{repository["path"]}: Pruning archives{dry_run_label}') borgmatic.borg.prune.prune_archives( global_arguments.dry_run, - repository, + repository['path'], storage, retention, local_borg_version, diff --git a/borgmatic/actions/rcreate.py b/borgmatic/actions/rcreate.py index 0052b4b..6220631 100644 --- a/borgmatic/actions/rcreate.py +++ b/borgmatic/actions/rcreate.py @@ -23,10 +23,10 @@ def run_rcreate( ): return - logger.info('{}: Creating repository'.format(repository)) + logger.info(f'{repository["path"]}: Creating repository') borgmatic.borg.rcreate.create_repository( global_arguments.dry_run, - repository, + repository['path'], storage, local_borg_version, rcreate_arguments.encryption_mode, diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 7a05809..f061dca 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -114,7 +114,13 @@ def restore_single_database( def collect_archive_database_names( - repository, archive, location, storage, local_borg_version, local_path, remote_path, + repository, + archive, + location, + storage, + local_borg_version, + local_path, + remote_path, ): ''' Given a local or remote repository path, a resolved archive name, a location configuration dict, @@ -180,7 +186,7 @@ def find_databases_to_restore(requested_database_names, archive_database_names): if 'all' in restore_names[UNSPECIFIED_HOOK]: restore_names[UNSPECIFIED_HOOK].remove('all') - for (hook_name, database_names) in archive_database_names.items(): + for hook_name, database_names in archive_database_names.items(): restore_names.setdefault(hook_name, []).extend(database_names) # If a database is to be restored as part of "all", then remove it from restore names so @@ -256,22 +262,34 @@ def run_restore( return logger.info( - '{}: Restoring databases from archive {}'.format(repository, restore_arguments.archive) + f'{repository["path"]}: Restoring databases from archive {restore_arguments.archive}' ) + borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, ) archive_name = borgmatic.borg.rlist.resolve_archive_name( - repository, restore_arguments.archive, storage, local_borg_version, local_path, remote_path, + repository['path'], + restore_arguments.archive, + storage, + local_borg_version, + local_path, + remote_path, ) archive_database_names = collect_archive_database_names( - repository, archive_name, location, storage, local_borg_version, local_path, remote_path, + repository['path'], + archive_name, + location, + storage, + local_borg_version, + local_path, + remote_path, ) restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names) found_names = set() @@ -291,7 +309,7 @@ def run_restore( found_names.add(database_name) restore_single_database( - repository, + repository['path'], location, storage, hooks, @@ -301,7 +319,7 @@ def run_restore( remote_path, archive_name, found_hook_name or hook_name, - found_database, + dict(found_database, **{'schemas': restore_arguments.schemas}), ) # For any database that weren't found via exact matches in the hooks configuration, try to @@ -320,7 +338,7 @@ def run_restore( database['name'] = database_name restore_single_database( - repository, + repository['path'], location, storage, hooks, @@ -330,13 +348,13 @@ def run_restore( remote_path, archive_name, found_hook_name or hook_name, - database, + dict(database, **{'schemas': restore_arguments.schemas}), ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 611d1bc..0947ec3 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_rinfo( - repository, storage, local_borg_version, rinfo_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + rinfo_arguments, + local_path, + remote_path, ): ''' Run the "rinfo" action for the given repository. @@ -19,9 +24,10 @@ def run_rinfo( repository, rinfo_arguments.repository ): if not rinfo_arguments.json: # pragma: nocover - logger.answer('{}: Displaying repository summary information'.format(repository)) + logger.answer(f'{repository["path"]}: Displaying repository summary information') + json_output = borgmatic.borg.rinfo.display_repository_info( - repository, + repository['path'], storage, local_borg_version, rinfo_arguments=rinfo_arguments, diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index 72d5206..10d06a5 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_rlist( - repository, storage, local_borg_version, rlist_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + rlist_arguments, + local_path, + remote_path, ): ''' Run the "rlist" action for the given repository. @@ -19,9 +24,10 @@ def run_rlist( repository, rlist_arguments.repository ): if not rlist_arguments.json: # pragma: nocover - logger.answer('{}: Listing repository'.format(repository)) + logger.answer(f'{repository["path"]}: Listing repository') + json_output = borgmatic.borg.rlist.list_repository( - repository, + repository['path'], storage, local_borg_version, rlist_arguments=rlist_arguments, diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 628f273..8089fd4 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,10 +17,10 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' - logger.info(f'{repository}: Transferring archives to repository') + logger.info(f'{repository["path"]}: Transferring archives to repository') borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, - repository, + repository['path'], storage, local_borg_version, transfer_arguments, diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index 460d9d6..f19d655 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -13,7 +13,7 @@ BORG_SUBCOMMANDS_WITHOUT_REPOSITORY = (('debug', 'info'), ('debug', 'convert-pro def run_arbitrary_borg( - repository, + repository_path, storage_config, local_borg_version, options, @@ -44,10 +44,10 @@ def run_arbitrary_borg( repository_archive_flags = () elif archive: repository_archive_flags = flags.make_repository_archive_flags( - repository, archive, local_borg_version + repository_path, archive, local_borg_version ) else: - repository_archive_flags = flags.make_repository_flags(repository, local_borg_version) + repository_archive_flags = flags.make_repository_flags(repository_path, local_borg_version) full_command = ( (local_path,) diff --git a/borgmatic/borg/break_lock.py b/borgmatic/borg/break_lock.py index 820b1c5..7099af8 100644 --- a/borgmatic/borg/break_lock.py +++ b/borgmatic/borg/break_lock.py @@ -7,7 +7,11 @@ logger = logging.getLogger(__name__) def break_lock( - repository, storage_config, local_borg_version, local_path='borg', remote_path=None, + repository_path, + storage_config, + local_borg_version, + local_path='borg', + remote_path=None, ): ''' Given a local or remote repository path, a storage configuration dict, the local Borg version, @@ -24,7 +28,7 @@ def break_lock( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) borg_environment = environment.make_environment(storage_config) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index d9beaa6..cee9d92 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -12,7 +12,6 @@ DEFAULT_CHECKS = ( {'name': 'repository', 'frequency': '1 month'}, {'name': 'archives', 'frequency': '1 month'}, ) -DEFAULT_PREFIX = '{hostname}-' logger = logging.getLogger(__name__) @@ -146,9 +145,10 @@ def filter_checks_on_frequency( return tuple(filtered_checks) -def make_check_flags(local_borg_version, checks, check_last=None, prefix=None): +def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None): ''' - Given the local Borg version and a parsed sequence of checks, transform the checks into tuple of + Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the + check last value, and a consistency check prefix, transform the checks into tuple of command-line flags. For example, given parsed checks of: @@ -174,10 +174,21 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None): if 'archives' in checks: last_flags = ('--last', str(check_last)) if check_last else () - if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): - match_archives_flags = ('--match-archives', f'sh:{prefix}*') if prefix else () - else: - match_archives_flags = ('--glob-archives', f'{prefix}*') if prefix else () + match_archives_flags = ( + ( + ('--match-archives', f'sh:{prefix}*') + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) + else ('--glob-archives', f'{prefix}*') + ) + if prefix + else ( + flags.make_match_archives_flags( + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, + ) + ) + ) else: last_flags = () match_archives_flags = () @@ -196,7 +207,7 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None): return common_flags return ( - tuple('--{}-only'.format(check) for check in checks if check in ('repository', 'archives')) + tuple(f'--{check}-only' for check in checks if check in ('repository', 'archives')) + common_flags ) @@ -243,7 +254,7 @@ def read_check_time(path): def check_archives( - repository, + repository_path, location_config, storage_config, consistency_config, @@ -268,7 +279,7 @@ def check_archives( try: borg_repository_id = json.loads( rinfo.display_repository_info( - repository, + repository_path, storage_config, local_borg_version, argparse.Namespace(json=True), @@ -277,7 +288,7 @@ def check_archives( ) )['repository']['id'] except (json.JSONDecodeError, KeyError): - raise ValueError(f'Cannot determine Borg repository ID for {repository}') + raise ValueError(f'Cannot determine Borg repository ID for {repository_path}') checks = filter_checks_on_frequency( location_config, @@ -291,7 +302,7 @@ def check_archives( extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') if set(checks).intersection({'repository', 'archives', 'data'}): - lock_wait = storage_config.get('lock_wait', None) + lock_wait = storage_config.get('lock_wait') verbosity_flags = () if logger.isEnabledFor(logging.INFO): @@ -299,18 +310,18 @@ def check_archives( if logger.isEnabledFor(logging.DEBUG): verbosity_flags = ('--debug', '--show-rc') - prefix = consistency_config.get('prefix', DEFAULT_PREFIX) + prefix = consistency_config.get('prefix') full_command = ( (local_path, 'check') + (('--repair',) if repair else ()) - + make_check_flags(local_borg_version, checks, check_last, prefix) + + make_check_flags(local_borg_version, storage_config, checks, check_last, prefix) + (('--remote-path', remote_path) if remote_path else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags + (('--progress',) if progress else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) borg_environment = environment.make_environment(storage_config) @@ -329,6 +340,6 @@ def check_archives( if 'extract' in checks: extract.extract_last_archive_dry_run( - storage_config, local_borg_version, repository, lock_wait, local_path, remote_path + storage_config, local_borg_version, repository_path, lock_wait, local_path, remote_path ) write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract')) diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index 847ed26..0e9d3e8 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def compact_segments( dry_run, - repository, + repository_path, storage_config, local_borg_version, local_path='borg', @@ -36,11 +36,11 @@ def compact_segments( + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) if dry_run: - logging.info(f'{repository}: Skipping compact (dry run)') + logging.info(f'{repository_path}: Skipping compact (dry run)') return execute_command( diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 87a0fdd..8782dc6 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -217,7 +217,7 @@ def make_list_filter_flags(local_borg_version, dry_run): return f'{base_flags}-' -DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' +DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' # noqa: FS003 def collect_borgmatic_source_directories(borgmatic_source_directory): @@ -322,7 +322,7 @@ def check_all_source_directories_exist(source_directories): def create_archive( dry_run, - repository, + repository_path, location_config, storage_config, local_borg_version, @@ -411,7 +411,7 @@ def create_archive( if stream_processes and location_config.get('read_special') is False: logger.warning( - f'{repository}: Ignoring configured "read_special" value of false, as true is needed for database hooks.' + f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.' ) create_command = ( @@ -446,7 +446,9 @@ def create_archive( ) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_archive_flags(repository, archive_name_format, local_borg_version) + + flags.make_repository_archive_flags( + repository_path, archive_name_format, local_borg_version + ) + (sources if not pattern_file else ()) ) @@ -466,7 +468,7 @@ def create_archive( # If database hooks are enabled (as indicated by streaming processes), exclude files that might # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True. if stream_processes and not location_config.get('read_special'): - logger.debug(f'{repository}: Collecting special file paths') + logger.debug(f'{repository_path}: Collecting special file paths') special_file_paths = collect_special_file_paths( create_command, local_path, @@ -477,7 +479,7 @@ def create_archive( if special_file_paths: logger.warning( - f'{repository}: Excluding special files to prevent Borg from hanging: {", ".join(special_file_paths)}' + f'{repository_path}: Excluding special files to prevent Borg from hanging: {", ".join(special_file_paths)}' ) exclude_file = write_pattern_file( expand_home_directories( @@ -507,7 +509,9 @@ def create_archive( ) elif output_log_level is None: return execute_command_and_capture_output( - create_command, working_directory=working_directory, extra_environment=borg_environment, + create_command, + working_directory=working_directory, + extra_environment=borg_environment, ) else: execute_command( diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index 43ea4ac..a624f07 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def export_tar_archive( dry_run, - repository, + repository_path, archive, paths, destination_path, @@ -45,7 +45,11 @@ def export_tar_archive( + (('--dry-run',) if dry_run else ()) + (('--tar-filter', tar_filter) if tar_filter else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) - + flags.make_repository_archive_flags(repository, archive, local_borg_version,) + + flags.make_repository_archive_flags( + repository_path, + archive, + local_borg_version, + ) + (destination_path,) + (tuple(paths) if paths else ()) ) @@ -56,7 +60,7 @@ def export_tar_archive( output_log_level = logging.INFO if dry_run: - logging.info('{}: Skipping export to tar file (dry run)'.format(repository)) + logging.info(f'{repository_path}: Skipping export to tar file (dry run)') return execute_command( diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 6c32f7f..f947141 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) def extract_last_archive_dry_run( storage_config, local_borg_version, - repository, + repository_path, lock_wait=None, local_path='borg', remote_path=None, @@ -30,7 +30,7 @@ def extract_last_archive_dry_run( try: last_archive_name = rlist.resolve_archive_name( - repository, 'latest', storage_config, local_borg_version, local_path, remote_path + repository_path, 'latest', storage_config, local_borg_version, local_path, remote_path ) except ValueError: logger.warning('No archives found. Skipping extract consistency check.') @@ -44,7 +44,9 @@ def extract_last_archive_dry_run( + lock_wait_flags + verbosity_flags + list_flag - + flags.make_repository_archive_flags(repository, last_archive_name, local_borg_version) + + flags.make_repository_archive_flags( + repository_path, last_archive_name, local_borg_version + ) ) execute_command( @@ -106,7 +108,11 @@ def extract_archive( + (('--strip-components', str(strip_components)) if strip_components else ()) + (('--progress',) if progress else ()) + (('--stdout',) if extract_to_stdout else ()) - + flags.make_repository_archive_flags(repository, archive, local_borg_version,) + + flags.make_repository_archive_flags( + repository, + archive, + local_borg_version, + ) + (tuple(paths) if paths else ()) ) diff --git a/borgmatic/borg/feature.py b/borgmatic/borg/feature.py index 5294121..b9311cd 100644 --- a/borgmatic/borg/feature.py +++ b/borgmatic/borg/feature.py @@ -1,6 +1,6 @@ from enum import Enum -from pkg_resources import parse_version +from packaging.version import parse class Feature(Enum): @@ -18,17 +18,17 @@ class Feature(Enum): FEATURE_TO_MINIMUM_BORG_VERSION = { - Feature.COMPACT: parse_version('1.2.0a2'), # borg compact - Feature.ATIME: parse_version('1.2.0a7'), # borg create --atime - Feature.NOFLAGS: parse_version('1.2.0a8'), # borg create --noflags - Feature.NUMERIC_IDS: parse_version('1.2.0b3'), # borg create/extract/mount --numeric-ids - Feature.UPLOAD_RATELIMIT: parse_version('1.2.0b3'), # borg create --upload-ratelimit - Feature.SEPARATE_REPOSITORY_ARCHIVE: parse_version('2.0.0a2'), # --repo with separate archive - Feature.RCREATE: parse_version('2.0.0a2'), # borg rcreate - Feature.RLIST: parse_version('2.0.0a2'), # borg rlist - Feature.RINFO: parse_version('2.0.0a2'), # borg rinfo - Feature.MATCH_ARCHIVES: parse_version('2.0.0b3'), # borg --match-archives - Feature.EXCLUDED_FILES_MINUS: parse_version('2.0.0b5'), # --list --filter uses "-" for excludes + Feature.COMPACT: parse('1.2.0a2'), # borg compact + Feature.ATIME: parse('1.2.0a7'), # borg create --atime + Feature.NOFLAGS: parse('1.2.0a8'), # borg create --noflags + Feature.NUMERIC_IDS: parse('1.2.0b3'), # borg create/extract/mount --numeric-ids + Feature.UPLOAD_RATELIMIT: parse('1.2.0b3'), # borg create --upload-ratelimit + Feature.SEPARATE_REPOSITORY_ARCHIVE: parse('2.0.0a2'), # --repo with separate archive + Feature.RCREATE: parse('2.0.0a2'), # borg rcreate + Feature.RLIST: parse('2.0.0a2'), # borg rlist + Feature.RINFO: parse('2.0.0a2'), # borg rinfo + Feature.MATCH_ARCHIVES: parse('2.0.0b3'), # borg --match-archives + Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'), # --list --filter uses "-" for excludes } @@ -37,4 +37,4 @@ def available(feature, borg_version): Given a Borg Feature constant and a Borg version string, return whether that feature is available in that version of Borg. ''' - return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse_version(borg_version) + return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse(borg_version) diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py index 81b6a6b..986531b 100644 --- a/borgmatic/borg/flags.py +++ b/borgmatic/borg/flags.py @@ -1,4 +1,5 @@ import itertools +import re from borgmatic.borg import feature @@ -10,7 +11,7 @@ def make_flags(name, value): if not value: return () - flag = '--{}'.format(name.replace('_', '-')) + flag = f"--{name.replace('_', '-')}" if value is True: return (flag,) @@ -33,7 +34,7 @@ def make_flags_from_arguments(arguments, excludes=()): ) -def make_repository_flags(repository, local_borg_version): +def make_repository_flags(repository_path, local_borg_version): ''' Given the path of a Borg repository and the local Borg version, return Borg-version-appropriate command-line flags (as a tuple) for selecting that repository. @@ -42,17 +43,41 @@ def make_repository_flags(repository, local_borg_version): ('--repo',) if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) else () - ) + (repository,) + ) + (repository_path,) -def make_repository_archive_flags(repository, archive, local_borg_version): +def make_repository_archive_flags(repository_path, archive, local_borg_version): ''' Given the path of a Borg repository, an archive name or pattern, and the local Borg version, return Borg-version-appropriate command-line flags (as a tuple) for selecting that repository and archive. ''' return ( - ('--repo', repository, archive) + ('--repo', repository_path, archive) if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) - else (f'{repository}::{archive}',) + else (f'{repository_path}::{archive}',) ) + + +def make_match_archives_flags(match_archives, archive_name_format, local_borg_version): + ''' + Return match archives flags based on the given match archives value, if any. If it isn't set, + return match archives flags to match archives created with the given archive name format, if + any. This is done by replacing certain archive name format placeholders for ephemeral data (like + "{now}") with globs. + ''' + if match_archives: + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): + return ('--match-archives', match_archives) + else: + return ('--glob-archives', re.sub(r'^sh:', '', match_archives)) + + if not archive_name_format: + return () + + derived_match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format) + + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): + return ('--match-archives', f'sh:{derived_match_archives}') + else: + return ('--glob-archives', f'{derived_match_archives}') diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index bcde24c..ef2c0c4 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def display_archives_info( - repository, + repository_path, storage_config, local_borg_version, info_arguments, @@ -44,22 +44,26 @@ def display_archives_info( else flags.make_flags('glob-archives', f'{info_arguments.prefix}*') ) if info_arguments.prefix - else () + else ( + flags.make_match_archives_flags( + info_arguments.match_archives + or info_arguments.archive + or storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, + ) + ) ) + flags.make_flags_from_arguments( - info_arguments, excludes=('repository', 'archive', 'prefix') - ) - + flags.make_repository_flags(repository, local_borg_version) - + ( - flags.make_flags('match-archives', info_arguments.archive) - if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) - else flags.make_flags('glob-archives', info_arguments.archive) + info_arguments, excludes=('repository', 'archive', 'prefix', 'match_archives') ) + + flags.make_repository_flags(repository_path, local_borg_version) ) if info_arguments.json: return execute_command_and_capture_output( - full_command, extra_environment=environment.make_environment(storage_config), + full_command, + extra_environment=environment.make_environment(storage_config), ) else: execute_command( diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index fedd365..908f8fe 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -21,7 +21,7 @@ MAKE_FLAGS_EXCLUDES = ( def make_list_command( - repository, + repository_path, storage_config, local_borg_version, list_arguments, @@ -52,10 +52,10 @@ def make_list_command( + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES) + ( flags.make_repository_archive_flags( - repository, list_arguments.archive, local_borg_version + repository_path, list_arguments.archive, local_borg_version ) if list_arguments.archive - else flags.make_repository_flags(repository, local_borg_version) + else flags.make_repository_flags(repository_path, local_borg_version) ) + (tuple(list_arguments.paths) if list_arguments.paths else ()) ) @@ -86,7 +86,7 @@ def make_find_paths(find_paths): def capture_archive_listing( - repository, + repository_path, archive, storage_config, local_borg_version, @@ -104,16 +104,16 @@ def capture_archive_listing( return tuple( execute_command_and_capture_output( make_list_command( - repository, + repository_path, storage_config, local_borg_version, argparse.Namespace( - repository=repository, + repository=repository_path, archive=archive, paths=[f'sh:{list_path}'], find_paths=None, json=None, - format='{path}{NL}', + format='{path}{NL}', # noqa: FS003 ), local_path, remote_path, @@ -126,7 +126,7 @@ def capture_archive_listing( def list_archive( - repository, + repository_path, storage_config, local_borg_version, list_arguments, @@ -149,7 +149,7 @@ def list_archive( ) rlist_arguments = argparse.Namespace( - repository=repository, + repository=repository_path, short=list_arguments.short, format=list_arguments.format, json=list_arguments.json, @@ -160,7 +160,12 @@ def list_archive( last=list_arguments.last, ) return rlist.list_repository( - repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path + repository_path, + storage_config, + local_borg_version, + rlist_arguments, + local_path, + remote_path, ) if list_arguments.archive: @@ -181,7 +186,7 @@ def list_archive( # getting a list of archives to search. if list_arguments.find_paths and not list_arguments.archive: rlist_arguments = argparse.Namespace( - repository=repository, + repository=repository_path, short=True, format=None, json=None, @@ -196,7 +201,7 @@ def list_archive( archive_lines = tuple( execute_command_and_capture_output( rlist.make_rlist_command( - repository, + repository_path, storage_config, local_borg_version, rlist_arguments, @@ -213,7 +218,7 @@ def list_archive( # For each archive listed by Borg, run list on the contents of that archive. for archive in archive_lines: - logger.answer(f'{repository}: Listing archive {archive}') + logger.answer(f'{repository_path}: Listing archive {archive}') archive_arguments = copy.copy(list_arguments) archive_arguments.archive = archive @@ -224,7 +229,7 @@ def list_archive( setattr(archive_arguments, name, None) main_command = make_list_command( - repository, + repository_path, storage_config, local_borg_version, archive_arguments, diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 2797534..2f6132d 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) def mount_archive( - repository, + repository_path, archive, mount_arguments, storage_config, @@ -40,7 +40,7 @@ def mount_archive( + (('-o', mount_arguments.options) if mount_arguments.options else ()) + ( ( - flags.make_repository_flags(repository, local_borg_version) + flags.make_repository_flags(repository_path, local_borg_version) + ( ('--match-archives', archive) if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) @@ -49,9 +49,9 @@ def mount_archive( ) if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) else ( - flags.make_repository_archive_flags(repository, archive, local_borg_version) + flags.make_repository_archive_flags(repository_path, archive, local_borg_version) if archive - else flags.make_repository_flags(repository, local_borg_version) + else flags.make_repository_flags(repository_path, local_borg_version) ) ) + (mount_arguments.mount_point,) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index b8b1d6b..c9a4635 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -7,10 +7,10 @@ from borgmatic.execute import execute_command logger = logging.getLogger(__name__) -def make_prune_flags(retention_config, local_borg_version): +def make_prune_flags(storage_config, retention_config, local_borg_version): ''' - Given a retention config dict mapping from option name to value, tranform it into an iterable of - command-line name-value flag pairs. + Given a retention config dict mapping from option name to value, transform it into an sequence of + command-line flags. For example, given a retention config of: @@ -24,22 +24,32 @@ def make_prune_flags(retention_config, local_borg_version): ) ''' config = retention_config.copy() - prefix = config.pop('prefix', '{hostname}-') + prefix = config.pop('prefix', None) - if prefix: - if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): - config['match_archives'] = f'sh:{prefix}*' - else: - config['glob_archives'] = f'{prefix}*' - - return ( + flag_pairs = ( ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items() ) + return tuple(element for pair in flag_pairs for element in pair) + ( + ( + ('--match-archives', f'sh:{prefix}*') + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) + else ('--glob-archives', f'{prefix}*') + ) + if prefix + else ( + flags.make_match_archives_flags( + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, + ) + ) + ) + def prune_archives( dry_run, - repository, + repository_path, storage_config, retention_config, local_borg_version, @@ -59,11 +69,7 @@ def prune_archives( full_command = ( (local_path, 'prune') - + tuple( - element - for pair in make_prune_flags(retention_config, local_borg_version) - for element in pair - ) + + make_prune_flags(storage_config, retention_config, local_borg_version) + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) @@ -78,7 +84,7 @@ def prune_archives( + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) if prune_arguments.stats or prune_arguments.list_archives: diff --git a/borgmatic/borg/rcreate.py b/borgmatic/borg/rcreate.py index d3a8f7a..7510529 100644 --- a/borgmatic/borg/rcreate.py +++ b/borgmatic/borg/rcreate.py @@ -13,7 +13,7 @@ RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2 def create_repository( dry_run, - repository, + repository_path, storage_config, local_borg_version, encryption_mode, @@ -33,14 +33,14 @@ def create_repository( ''' try: rinfo.display_repository_info( - repository, + repository_path, storage_config, local_borg_version, argparse.Namespace(json=True), local_path, remote_path, ) - logger.info(f'{repository}: Repository already exists. Skipping creation.') + logger.info(f'{repository_path}: Repository already exists. Skipping creation.') return except subprocess.CalledProcessError as error: if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE: @@ -65,11 +65,11 @@ def create_repository( + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + (('--remote-path', remote_path) if remote_path else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) if dry_run: - logging.info(f'{repository}: Skipping repository creation (dry run)') + logging.info(f'{repository_path}: Skipping repository creation (dry run)') return # Do not capture output here, so as to support interactive prompts. diff --git a/borgmatic/borg/rinfo.py b/borgmatic/borg/rinfo.py index 7bc9a5e..97d7a66 100644 --- a/borgmatic/borg/rinfo.py +++ b/borgmatic/borg/rinfo.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def display_repository_info( - repository, + repository_path, storage_config, local_borg_version, rinfo_arguments, @@ -43,14 +43,15 @@ def display_repository_info( + flags.make_flags('remote-path', remote_path) + flags.make_flags('lock-wait', lock_wait) + (('--json',) if rinfo_arguments.json else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) extra_environment = environment.make_environment(storage_config) if rinfo_arguments.json: return execute_command_and_capture_output( - full_command, extra_environment=extra_environment, + full_command, + extra_environment=extra_environment, ) else: execute_command( diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 2a465fb..c051a9a 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def resolve_archive_name( - repository, archive, storage_config, local_borg_version, local_path='borg', remote_path=None + repository_path, + archive, + storage_config, + local_borg_version, + local_path='borg', + remote_path=None, ): ''' Given a local or remote repository path, an archive name, a storage config dict, a local Borg @@ -31,27 +36,28 @@ def resolve_archive_name( + flags.make_flags('lock-wait', lock_wait) + flags.make_flags('last', 1) + ('--short',) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) output = execute_command_and_capture_output( - full_command, extra_environment=environment.make_environment(storage_config), + full_command, + extra_environment=environment.make_environment(storage_config), ) try: latest_archive = output.strip().splitlines()[-1] except IndexError: raise ValueError('No archives found in the repository') - logger.debug('{}: Latest archive is {}'.format(repository, latest_archive)) + logger.debug(f'{repository_path}: Latest archive is {latest_archive}') return latest_archive -MAKE_FLAGS_EXCLUDES = ('repository', 'prefix') +MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives') def make_rlist_command( - repository, + repository_path, storage_config, local_borg_version, rlist_arguments, @@ -89,15 +95,21 @@ def make_rlist_command( else flags.make_flags('glob-archives', f'{rlist_arguments.prefix}*') ) if rlist_arguments.prefix - else () + else ( + flags.make_match_archives_flags( + rlist_arguments.match_archives or storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, + ) + ) ) + flags.make_flags_from_arguments(rlist_arguments, excludes=MAKE_FLAGS_EXCLUDES) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) def list_repository( - repository, + repository_path, storage_config, local_borg_version, rlist_arguments, @@ -113,11 +125,16 @@ def list_repository( borg_environment = environment.make_environment(storage_config) main_command = make_rlist_command( - repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path + repository_path, + storage_config, + local_borg_version, + rlist_arguments, + local_path, + remote_path, ) if rlist_arguments.json: - return execute_command_and_capture_output(main_command, extra_environment=borg_environment,) + return execute_command_and_capture_output(main_command, extra_environment=borg_environment) else: execute_command( main_command, diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index bad02d0..9fd05b7 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def transfer_archives( dry_run, - repository, + repository_path, storage_config, local_borg_version, transfer_arguments, @@ -28,17 +28,22 @@ def transfer_archives( + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + flags.make_flags('remote-path', remote_path) + flags.make_flags('lock-wait', storage_config.get('lock_wait', None)) - + (('--progress',) if transfer_arguments.progress else ()) + ( - flags.make_flags( - 'match-archives', transfer_arguments.match_archives or transfer_arguments.archive + flags.make_flags_from_arguments( + transfer_arguments, + excludes=('repository', 'source_repository', 'archive', 'match_archives'), + ) + or ( + flags.make_match_archives_flags( + transfer_arguments.match_archives + or transfer_arguments.archive + or storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, + ) ) ) - + flags.make_flags_from_arguments( - transfer_arguments, - excludes=('repository', 'source_repository', 'archive', 'match_archives'), - ) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) + flags.make_flags('other-repo', transfer_arguments.source_repository) + flags.make_flags('dry-run', dry_run) ) diff --git a/borgmatic/borg/version.py b/borgmatic/borg/version.py index 6d6c302..d90a7aa 100644 --- a/borgmatic/borg/version.py +++ b/borgmatic/borg/version.py @@ -19,7 +19,8 @@ def local_borg_version(storage_config, local_path='borg'): + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) ) output = execute_command_and_capture_output( - full_command, extra_environment=environment.make_environment(storage_config), + full_command, + extra_environment=environment.make_environment(storage_config), ) try: diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index db743dc..7711958 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -131,9 +131,7 @@ def make_parsers(): nargs='*', dest='config_paths', default=config_paths, - help='Configuration filenames or directories, defaults to: {}'.format( - ' '.join(unexpanded_config_paths) - ), + help=f"Configuration filenames or directories, defaults to: {' '.join(unexpanded_config_paths)}", ) global_group.add_argument( '--excludes', @@ -182,9 +180,13 @@ def make_parsers(): global_group.add_argument( '--log-file', type=str, - default=None, help='Write log messages to this file instead of syslog', ) + global_group.add_argument( + '--log-file-format', + type=str, + help='Log format string used for log messages written to the log file', + ) global_group.add_argument( '--override', metavar='SECTION.OPTION=VALUE', @@ -225,7 +227,7 @@ def make_parsers(): subparsers = top_level_parser.add_subparsers( title='actions', metavar='', - help='Specify zero or more actions. Defaults to creat, prune, compact, and check. Use --help with action for details:', + help='Specify zero or more actions. Defaults to create, prune, compact, and check. Use --help with action for details:', ) rcreate_parser = subparsers.add_parser( 'rcreate', @@ -258,10 +260,13 @@ def make_parsers(): help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key [Borg 2.x+ only]', ) rcreate_group.add_argument( - '--append-only', action='store_true', help='Create an append-only repository', + '--append-only', + action='store_true', + help='Create an append-only repository', ) rcreate_group.add_argument( - '--storage-quota', help='Create a repository with a fixed storage quota', + '--storage-quota', + help='Create a repository with a fixed storage quota', ) rcreate_group.add_argument( '--make-parent-dirs', @@ -295,7 +300,7 @@ def make_parsers(): ) transfer_group.add_argument( '--upgrader', - help='Upgrader type used to convert the transfered data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion', + help='Upgrader type used to convert the transferred data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion', ) transfer_group.add_argument( '--progress', @@ -673,6 +678,13 @@ def make_parsers(): dest='databases', help="Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic's configuration", ) + restore_group.add_argument( + '--schema', + metavar='NAME', + nargs='+', + dest='schemas', + help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases', + ) restore_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' ) @@ -686,7 +698,8 @@ def make_parsers(): ) rlist_group = rlist_parser.add_argument_group('rlist arguments') rlist_group.add_argument( - '--repository', help='Path of repository to list, defaults to the configured repositories', + '--repository', + help='Path of repository to list, defaults to the configured repositories', ) rlist_group.add_argument( '--short', default=False, action='store_true', help='Output only archive names' @@ -696,7 +709,7 @@ def make_parsers(): '--json', default=False, action='store_true', help='Output results as JSON' ) rlist_group.add_argument( - '-P', '--prefix', help='Only list archive names starting with this prefix' + '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix' ) rlist_group.add_argument( '-a', @@ -763,7 +776,7 @@ def make_parsers(): '--json', default=False, action='store_true', help='Output results as JSON' ) list_group.add_argument( - '-P', '--prefix', help='Only list archive names starting with this prefix' + '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix' ) list_group.add_argument( '-a', @@ -835,7 +848,9 @@ def make_parsers(): '--json', dest='json', default=False, action='store_true', help='Output results as JSON' ) info_group.add_argument( - '-P', '--prefix', help='Only show info for archive names starting with this prefix' + '-P', + '--prefix', + help='Deprecated. Only show info for archive names starting with this prefix', ) info_group.add_argument( '-a', @@ -945,7 +960,17 @@ def parse_arguments(*unparsed_arguments): and arguments['transfer'].match_archives ): raise ValueError( - 'With the transfer action, only one of --archive and --glob-archives flags can be used.' + 'With the transfer action, only one of --archive and --match-archives flags can be used.' + ) + + if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives): + raise ValueError( + 'With the list action, only one of --prefix or --match-archives flags can be used.' + ) + + if 'rlist' in arguments and (arguments['rlist'].prefix and arguments['rlist'].match_archives): + raise ValueError( + 'With the rlist action, only one of --prefix or --match-archives flags can be used.' ) if 'info' in arguments and ( diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index fbea260..999e9d8 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -8,7 +8,11 @@ from queue import Queue from subprocess import CalledProcessError import colorama -import pkg_resources + +try: + import importlib_metadata +except ModuleNotFoundError: # pragma: nocover + import importlib.metadata as importlib_metadata import borgmatic.actions.borg import borgmatic.actions.break_lock @@ -70,9 +74,7 @@ def run_configuration(config_filename, config, arguments): try: local_borg_version = borg_version.local_borg_version(storage, local_path) except (OSError, CalledProcessError, ValueError) as error: - yield from log_error_records( - '{}: Error getting local Borg version'.format(config_filename), error - ) + yield from log_error_records(f'{config_filename}: Error getting local Borg version', error) return try: @@ -100,15 +102,18 @@ def run_configuration(config_filename, config, arguments): return encountered_error = error - yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error) + yield from log_error_records(f'{config_filename}: Error pinging monitor', error) if not encountered_error: repo_queue = Queue() for repo in location['repositories']: - repo_queue.put((repo, 0),) + repo_queue.put( + (repo, 0), + ) while not repo_queue.empty(): - repository_path, retry_num = repo_queue.get() + repository, retry_num = repo_queue.get() + logger.debug(f'{repository["path"]}: Running actions for repository') timeout = retry_num * retry_wait if timeout: logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') @@ -125,14 +130,16 @@ def run_configuration(config_filename, config, arguments): local_path=local_path, remote_path=remote_path, local_borg_version=local_borg_version, - repository_path=repository_path, + repository=repository, ) except (OSError, CalledProcessError, ValueError) as error: if retry_num < retries: - repo_queue.put((repository_path, retry_num + 1),) + repo_queue.put( + (repository, retry_num + 1), + ) tuple( # Consume the generator so as to trigger logging. log_error_records( - '{}: Error running actions for repository'.format(repository_path), + f'{repository["path"]}: Error running actions for repository', error, levelno=logging.WARNING, log_command_error_output=True, @@ -147,10 +154,10 @@ def run_configuration(config_filename, config, arguments): return yield from log_error_records( - '{}: Error running actions for repository'.format(repository_path), error + f'{repository["path"]}: Error running actions for repository', error ) encountered_error = error - error_repository = repository_path + error_repository = repository['path'] try: if using_primary_action: @@ -169,7 +176,7 @@ def run_configuration(config_filename, config, arguments): return encountered_error = error - yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error) + yield from log_error_records(f'{repository["path"]}: Error pinging monitor', error) if not encountered_error: try: @@ -196,7 +203,7 @@ def run_configuration(config_filename, config, arguments): return encountered_error = error - yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error) + yield from log_error_records(f'{config_filename}: Error pinging monitor', error) if encountered_error and using_primary_action: try: @@ -231,9 +238,7 @@ def run_configuration(config_filename, config, arguments): if command.considered_soft_failure(config_filename, error): return - yield from log_error_records( - '{}: Error running on-error hook'.format(config_filename), error - ) + yield from log_error_records(f'{config_filename}: Error running on-error hook', error) def run_actions( @@ -248,7 +253,7 @@ def run_actions( local_path, remote_path, local_borg_version, - repository_path, + repository, ): ''' Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration @@ -263,13 +268,14 @@ def run_actions( invalid. ''' add_custom_log_levels() - repository = os.path.expanduser(repository_path) + repository_path = os.path.expanduser(repository['path']) global_arguments = arguments['global'] dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else '' hook_context = { 'repository': repository_path, # Deprecated: For backwards compatibility with borgmatic < 1.6.0. - 'repositories': ','.join(location['repositories']), + 'repositories': ','.join([repo['path'] for repo in location['repositories']]), + 'log_file': global_arguments.log_file if global_arguments.log_file else '', } command.execute_hook( @@ -281,7 +287,7 @@ def run_actions( **hook_context, ) - for (action_name, action_arguments) in arguments.items(): + for action_name, action_arguments in arguments.items(): if action_name == 'rcreate': borgmatic.actions.rcreate.run_rcreate( repository, @@ -410,19 +416,39 @@ def run_actions( ) elif action_name == 'rlist': yield from borgmatic.actions.rlist.run_rlist( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'list': yield from borgmatic.actions.list.run_list( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'rinfo': yield from borgmatic.actions.rinfo.run_rinfo( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'info': yield from borgmatic.actions.info.run_info( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'break-lock': borgmatic.actions.break_lock.run_break_lock( @@ -435,7 +461,12 @@ def run_actions( ) elif action_name == 'borg': borgmatic.actions.borg.run_borg( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) command.execute_hook( @@ -472,9 +503,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): dict( levelno=logging.WARNING, levelname='WARNING', - msg='{}: Insufficient permissions to read configuration file'.format( - config_filename - ), + msg=f'{config_filename}: Insufficient permissions to read configuration file', ) ), ] @@ -486,7 +515,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): dict( levelno=logging.CRITICAL, levelname='CRITICAL', - msg='{}: Error parsing configuration file'.format(config_filename), + msg=f'{config_filename}: Error parsing configuration file', ) ), logging.makeLogRecord( @@ -587,9 +616,7 @@ def collect_configuration_run_summary_logs(configs, arguments): if not configs: yield from log_error_records( - '{}: No valid configuration files found'.format( - ' '.join(arguments['global'].config_paths) - ) + f"{' '.join(arguments['global'].config_paths)}: No valid configuration files found", ) return @@ -615,24 +642,25 @@ def collect_configuration_run_summary_logs(configs, arguments): error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord)) if error_logs: - yield from log_error_records('{}: An error occurred'.format(config_filename)) + yield from log_error_records(f'{config_filename}: An error occurred') yield from error_logs else: yield logging.makeLogRecord( dict( levelno=logging.INFO, levelname='INFO', - msg='{}: Successfully ran configuration file'.format(config_filename), + msg=f'{config_filename}: Successfully ran configuration file', ) ) if results: json_results.extend(results) if 'umount' in arguments: - logger.info('Unmounting mount point {}'.format(arguments['umount'].mount_point)) + logger.info(f"Unmounting mount point {arguments['umount'].mount_point}") try: borg_umount.unmount_archive( - mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs), + mount_point=arguments['umount'].mount_point, + local_path=get_local_path(configs), ) except (CalledProcessError, OSError) as error: yield from log_error_records('Error unmounting mount point', error) @@ -677,12 +705,12 @@ def main(): # pragma: no cover if error.code == 0: raise error configure_logging(logging.CRITICAL) - logger.critical('Error parsing arguments: {}'.format(' '.join(sys.argv))) + logger.critical(f"Error parsing arguments: {' '.join(sys.argv)}") exit_with_help_link() global_arguments = arguments['global'] if global_arguments.version: - print(pkg_resources.require('borgmatic')[0].version) + print(importlib_metadata.version('borgmatic')) sys.exit(0) if global_arguments.bash_completion: print(borgmatic.commands.completion.bash_completion()) @@ -707,10 +735,11 @@ def main(): # pragma: no cover verbosity_to_log_level(global_arguments.log_file_verbosity), verbosity_to_log_level(global_arguments.monitoring_verbosity), global_arguments.log_file, + global_arguments.log_file_format, ) except (FileNotFoundError, PermissionError) as error: configure_logging(logging.CRITICAL) - logger.critical('Error configuring logging: {}'.format(error)) + logger.critical(f'Error configuring logging: {error}') exit_with_help_link() logger.debug('Ensuring legacy configuration is upgraded') diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 0ff1f3e..1fc976b 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -34,7 +34,7 @@ def bash_completion(): ' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"', ' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];' - ' then cat << EOF\n%s\nEOF' % UPGRADE_MESSAGE, + f' then cat << EOF\n{UPGRADE_MESSAGE}\nEOF', ' fi', '}', 'complete_borgmatic() {', @@ -48,7 +48,7 @@ def bash_completion(): for action, subparser in subparsers.choices.items() ) + ( - ' COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))' + ' COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))' # noqa: FS003 % (actions, global_flags), ' (check_version &)', '}', diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py index 093d4e3..64a8948 100644 --- a/borgmatic/commands/convert_config.py +++ b/borgmatic/commands/convert_config.py @@ -28,9 +28,7 @@ def parse_arguments(*arguments): '--source-config', dest='source_config_filename', default=DEFAULT_SOURCE_CONFIG_FILENAME, - help='Source INI-style configuration filename. Default: {}'.format( - DEFAULT_SOURCE_CONFIG_FILENAME - ), + help=f'Source INI-style configuration filename. Default: {DEFAULT_SOURCE_CONFIG_FILENAME}', ) parser.add_argument( '-e', @@ -46,9 +44,7 @@ def parse_arguments(*arguments): '--destination-config', dest='destination_config_filename', default=DEFAULT_DESTINATION_CONFIG_FILENAME, - help='Destination YAML configuration filename. Default: {}'.format( - DEFAULT_DESTINATION_CONFIG_FILENAME - ), + help=f'Destination YAML configuration filename. Default: {DEFAULT_DESTINATION_CONFIG_FILENAME}', ) return parser.parse_args(arguments) @@ -59,19 +55,15 @@ TEXT_WRAP_CHARACTERS = 80 def display_result(args): # pragma: no cover result_lines = textwrap.wrap( - 'Your borgmatic configuration has been upgraded. Please review the result in {}.'.format( - args.destination_config_filename - ), + f'Your borgmatic configuration has been upgraded. Please review the result in {args.destination_config_filename}.', TEXT_WRAP_CHARACTERS, ) + excludes_phrase = ( + f' and {args.source_excludes_filename}' if args.source_excludes_filename else '' + ) delete_lines = textwrap.wrap( - 'Once you are satisfied, you can safely delete {}{}.'.format( - args.source_config_filename, - ' and {}'.format(args.source_excludes_filename) - if args.source_excludes_filename - else '', - ), + f'Once you are satisfied, you can safely delete {args.source_config_filename}{excludes_phrase}.', TEXT_WRAP_CHARACTERS, ) diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py index 13a5cba..78c32f0 100644 --- a/borgmatic/commands/generate_config.py +++ b/borgmatic/commands/generate_config.py @@ -23,9 +23,7 @@ def parse_arguments(*arguments): '--destination', dest='destination_filename', default=DEFAULT_DESTINATION_CONFIG_FILENAME, - help='Destination YAML configuration file, default: {}'.format( - DEFAULT_DESTINATION_CONFIG_FILENAME - ), + help=f'Destination YAML configuration file, default: {DEFAULT_DESTINATION_CONFIG_FILENAME}', ) parser.add_argument( '--overwrite', @@ -48,17 +46,13 @@ def main(): # pragma: no cover overwrite=args.overwrite, ) - print('Generated a sample configuration file at {}.'.format(args.destination_filename)) + print(f'Generated a sample configuration file at {args.destination_filename}.') print() if args.source_filename: - print( - 'Merged in the contents of configuration file at {}.'.format(args.source_filename) - ) + print(f'Merged in the contents of configuration file at {args.source_filename}.') print('To review the changes made, run:') print() - print( - ' diff --unified {} {}'.format(args.source_filename, args.destination_filename) - ) + print(f' diff --unified {args.source_filename} {args.destination_filename}') print() print('This includes all available configuration options with example values. The few') print('required options are indicated. Please edit the file to suit your needs.') diff --git a/borgmatic/commands/validate_config.py b/borgmatic/commands/validate_config.py index 00ea9f4..8aa8d32 100644 --- a/borgmatic/commands/validate_config.py +++ b/borgmatic/commands/validate_config.py @@ -2,6 +2,7 @@ import logging import sys from argparse import ArgumentParser +import borgmatic.config.generate from borgmatic.config import collect, validate logger = logging.getLogger(__name__) @@ -21,20 +22,24 @@ def parse_arguments(*arguments): nargs='+', dest='config_paths', default=config_paths, - help='Configuration filenames or directories, defaults to: {}'.format( - ' '.join(config_paths) - ), + help=f'Configuration filenames or directories, defaults to: {config_paths}', + ) + parser.add_argument( + '-s', + '--show', + action='store_true', + help='Show the validated configuration after all include merging has occurred', ) return parser.parse_args(arguments) def main(): # pragma: no cover - args = parse_arguments(*sys.argv[1:]) + arguments = parse_arguments(*sys.argv[1:]) logging.basicConfig(level=logging.INFO, format='%(message)s') - config_filenames = tuple(collect.collect_config_filenames(args.config_paths)) + config_filenames = tuple(collect.collect_config_filenames(arguments.config_paths)) if len(config_filenames) == 0: logger.critical('No files to validate found') sys.exit(1) @@ -42,15 +47,22 @@ def main(): # pragma: no cover found_issues = False for config_filename in config_filenames: try: - validate.parse_configuration(config_filename, validate.schema_filename()) + config, parse_logs = validate.parse_configuration( + config_filename, validate.schema_filename() + ) except (ValueError, OSError, validate.Validation_error) as error: - logging.critical('{}: Error parsing configuration file'.format(config_filename)) + logging.critical(f'{config_filename}: Error parsing configuration file') logging.critical(error) found_issues = True + else: + for log in parse_logs: + logger.handle(log) + + if arguments.show: + print('---') + print(borgmatic.config.generate.render_configuration(config)) if found_issues: sys.exit(1) - else: - logger.info( - 'All given configuration files are valid: {}'.format(', '.join(config_filenames)) - ) + + logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}") diff --git a/borgmatic/config/collect.py b/borgmatic/config/collect.py index a13472e..bd38fee 100644 --- a/borgmatic/config/collect.py +++ b/borgmatic/config/collect.py @@ -16,8 +16,8 @@ def get_default_config_paths(expand_home=True): return [ '/etc/borgmatic/config.yaml', '/etc/borgmatic.d', - '%s/borgmatic/config.yaml' % user_config_directory, - '%s/borgmatic.d' % user_config_directory, + os.path.join(user_config_directory, 'borgmatic/config.yaml'), + os.path.join(user_config_directory, 'borgmatic.d'), ] diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index e4e55e4..093ad0c 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -43,7 +43,7 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema): ] ) - # Split space-seperated values into actual lists, make "repository" into a list, and merge in + # Split space-separated values into actual lists, make "repository" into a list, and merge in # excludes. location = destination_config['location'] location['source_directories'] = source_config.location['source_directories'].split(' ') diff --git a/borgmatic/config/environment.py b/borgmatic/config/environment.py index 3a58566..a2857bb 100644 --- a/borgmatic/config/environment.py +++ b/borgmatic/config/environment.py @@ -14,11 +14,14 @@ def _resolve_string(matcher): if matcher.group('escape') is not None: # in case of escaped envvar, unescape it return matcher.group('variable') + # resolve the env var name, default = matcher.group('name'), matcher.group('default') out = os.getenv(name, default=default) + if out is None: - raise ValueError('Cannot find variable ${name} in environment'.format(name=name)) + raise ValueError(f'Cannot find variable {name} in environment') + return out diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index e864a3c..d486f23 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -48,7 +48,7 @@ def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): config, schema, indent=indent, skip_first=parent_is_sequence ) else: - raise ValueError('Schema at level {} is unsupported: {}'.format(level, schema)) + raise ValueError(f'Schema at level {level} is unsupported: {schema}') return config @@ -84,7 +84,7 @@ def _comment_out_optional_configuration(rendered_config): for line in rendered_config.split('\n'): # Upon encountering an optional configuration option, comment out lines until the next blank # line. - if line.strip().startswith('# {}'.format(COMMENTED_OUT_SENTINEL)): + if line.strip().startswith(f'# {COMMENTED_OUT_SENTINEL}'): optional = True continue @@ -117,9 +117,7 @@ def write_configuration(config_filename, rendered_config, mode=0o600, overwrite= ''' if not overwrite and os.path.exists(config_filename): raise FileExistsError( - '{} already exists. Aborting. Use --overwrite to replace the file.'.format( - config_filename - ) + f'{config_filename} already exists. Aborting. Use --overwrite to replace the file.' ) try: @@ -218,7 +216,7 @@ def remove_commented_out_sentinel(config, field_name): except KeyError: return - if last_comment_value == '# {}\n'.format(COMMENTED_OUT_SENTINEL): + if last_comment_value == f'# {COMMENTED_OUT_SENTINEL}\n': config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX].pop() diff --git a/borgmatic/config/legacy.py b/borgmatic/config/legacy.py index 9135278..ec1e50a 100644 --- a/borgmatic/config/legacy.py +++ b/borgmatic/config/legacy.py @@ -70,13 +70,11 @@ def validate_configuration_format(parser, config_format): section_format.name for section_format in config_format ) if unknown_section_names: - raise ValueError( - 'Unknown config sections found: {}'.format(', '.join(unknown_section_names)) - ) + raise ValueError(f"Unknown config sections found: {', '.join(unknown_section_names)}") missing_section_names = set(required_section_names) - section_names if missing_section_names: - raise ValueError('Missing config sections: {}'.format(', '.join(missing_section_names))) + raise ValueError(f"Missing config sections: {', '.join(missing_section_names)}") for section_format in config_format: if section_format.name not in section_names: @@ -91,9 +89,7 @@ def validate_configuration_format(parser, config_format): if unexpected_option_names: raise ValueError( - 'Unexpected options found in config section {}: {}'.format( - section_format.name, ', '.join(sorted(unexpected_option_names)) - ) + f"Unexpected options found in config section {section_format.name}: {', '.join(sorted(unexpected_option_names))}", ) missing_option_names = tuple( @@ -105,9 +101,7 @@ def validate_configuration_format(parser, config_format): if missing_option_names: raise ValueError( - 'Required options missing from config section {}: {}'.format( - section_format.name, ', '.join(missing_option_names) - ) + f"Required options missing from config section {section_format.name}: {', '.join(missing_option_names)}", ) @@ -137,7 +131,7 @@ def parse_configuration(config_filename, config_format): ''' parser = RawConfigParser() if not parser.read(config_filename): - raise ValueError('Configuration file cannot be opened: {}'.format(config_filename)) + raise ValueError(f'Configuration file cannot be opened: {config_filename}') validate_configuration_format(parser, config_format) diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index 04461af..f6290de 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -1,4 +1,5 @@ import functools +import json import logging import os @@ -37,6 +38,37 @@ def include_configuration(loader, filename_node, include_directory): return load_configuration(include_filename) +def raise_retain_node_error(loader, node): + ''' + Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!retain" usage. + + Raise ValueError if a mapping or sequence node is given, as that indicates that "!retain" was + used in a configuration file without a merge. In configuration files with a merge, mapping and + sequence nodes with "!retain" tags are handled by deep_merge_nodes() below. + + Also raise ValueError if a scalar node is given, as "!retain" is not supported on scalar nodes. + ''' + if isinstance(node, (ruamel.yaml.nodes.MappingNode, ruamel.yaml.nodes.SequenceNode)): + raise ValueError( + 'The !retain tag may only be used within a configuration file containing a merged !include tag.' + ) + + raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.') + + +def raise_omit_node_error(loader, node): + ''' + Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!omit" usage. + + Raise ValueError unconditionally, as an "!omit" node here indicates it was used in a + configuration file without a merge. In configuration files with a merge, nodes with "!omit" + tags are handled by deep_merge_nodes() below. + ''' + raise ValueError( + 'The !omit tag may only be used on a scalar (e.g., string) list element within a configuration file containing a merged !include tag.' + ) + + class Include_constructor(ruamel.yaml.SafeConstructor): ''' A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including @@ -49,6 +81,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor): '!include', functools.partial(include_configuration, include_directory=include_directory), ) + self.add_constructor('!retain', raise_retain_node_error) + self.add_constructor('!omit', raise_omit_node_error) def flatten_mapping(self, node): ''' @@ -81,11 +115,13 @@ class Include_constructor(ruamel.yaml.SafeConstructor): def load_configuration(filename): ''' Load the given configuration file and return its contents as a data structure of nested dicts - and lists. + and lists. Also, replace any "{constant}" strings with the value of the "constant" key in the + "constants" section of the configuration file. Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError if there are too many recursive includes. ''' + # Use an embedded derived class for the include constructor so as to capture the filename # value. (functools.partial doesn't work for this use case because yaml.Constructor has to be # an actual class.) @@ -98,7 +134,29 @@ def load_configuration(filename): yaml = ruamel.yaml.YAML(typ='safe') yaml.Constructor = Include_constructor_with_include_directory - return yaml.load(open(filename)) + with open(filename) as file: + file_contents = file.read() + config = yaml.load(file_contents) + + if config and 'constants' in config: + for key, value in config['constants'].items(): + value = json.dumps(value) + file_contents = file_contents.replace(f'{{{key}}}', value.strip('"')) + + config = yaml.load(file_contents) + del config['constants'] + + return config + + +def filter_omitted_nodes(nodes): + ''' + Given a list of nodes, return a filtered list omitting any nodes with an "!omit" tag or with a + value matching such nodes. + ''' + omitted_values = tuple(node.value for node in nodes if node.tag == '!omit') + + return [node for node in nodes if node.value not in omitted_values] DELETED_NODE = object() @@ -162,6 +220,8 @@ def deep_merge_nodes(nodes): ), ] + If a mapping or sequence node has a YAML "!retain" tag, then that node is not merged. + The purpose of deep merging like this is to support, for instance, merging one borgmatic configuration file into another for reuse, such that a configuration section ("retention", etc.) does not completely replace the corresponding section in a merged file. @@ -184,32 +244,42 @@ def deep_merge_nodes(nodes): # If we're dealing with MappingNodes, recurse and merge its values as well. if isinstance(b_value, ruamel.yaml.nodes.MappingNode): - replaced_nodes[(b_key, b_value)] = ( - b_key, - ruamel.yaml.nodes.MappingNode( - tag=b_value.tag, - value=deep_merge_nodes(a_value.value + b_value.value), - start_mark=b_value.start_mark, - end_mark=b_value.end_mark, - flow_style=b_value.flow_style, - comment=b_value.comment, - anchor=b_value.anchor, - ), - ) + # A "!retain" tag says to skip deep merging for this node. Replace the tag so + # downstream schema validation doesn't break on our application-specific tag. + if b_value.tag == '!retain': + b_value.tag = 'tag:yaml.org,2002:map' + else: + replaced_nodes[(b_key, b_value)] = ( + b_key, + ruamel.yaml.nodes.MappingNode( + tag=b_value.tag, + value=deep_merge_nodes(a_value.value + b_value.value), + start_mark=b_value.start_mark, + end_mark=b_value.end_mark, + flow_style=b_value.flow_style, + comment=b_value.comment, + anchor=b_value.anchor, + ), + ) # If we're dealing with SequenceNodes, merge by appending one sequence to the other. elif isinstance(b_value, ruamel.yaml.nodes.SequenceNode): - replaced_nodes[(b_key, b_value)] = ( - b_key, - ruamel.yaml.nodes.SequenceNode( - tag=b_value.tag, - value=a_value.value + b_value.value, - start_mark=b_value.start_mark, - end_mark=b_value.end_mark, - flow_style=b_value.flow_style, - comment=b_value.comment, - anchor=b_value.anchor, - ), - ) + # A "!retain" tag says to skip deep merging for this node. Replace the tag so + # downstream schema validation doesn't break on our application-specific tag. + if b_value.tag == '!retain': + b_value.tag = 'tag:yaml.org,2002:seq' + else: + replaced_nodes[(b_key, b_value)] = ( + b_key, + ruamel.yaml.nodes.SequenceNode( + tag=b_value.tag, + value=filter_omitted_nodes(a_value.value + b_value.value), + start_mark=b_value.start_mark, + end_mark=b_value.end_mark, + flow_style=b_value.flow_style, + comment=b_value.comment, + anchor=b_value.anchor, + ), + ) return [ replaced_nodes.get(node, node) for node in nodes if replaced_nodes.get(node) != DELETED_NODE diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index a143a19..147e4e4 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -57,9 +57,15 @@ def normalize(config_filename, config): # Upgrade remote repositories to ssh:// syntax, required in Borg 2. repositories = location.get('repositories') if repositories: + if isinstance(repositories[0], str): + config['location']['repositories'] = [ + {'path': repository} for repository in repositories + ] + repositories = config['location']['repositories'] config['location']['repositories'] = [] - for repository in repositories: - if '~' in repository: + for repository_dict in repositories: + repository_path = repository_dict['path'] + if '~' in repository_path: logs.append( logging.makeLogRecord( dict( @@ -69,26 +75,37 @@ def normalize(config_filename, config): ) ) ) - if ':' in repository: - if repository.startswith('file://'): - config['location']['repositories'].append( - os.path.abspath(repository.partition('file://')[-1]) + if ':' in repository_path: + if repository_path.startswith('file://'): + updated_repository_path = os.path.abspath( + repository_path.partition('file://')[-1] ) - elif repository.startswith('ssh://'): - config['location']['repositories'].append(repository) + config['location']['repositories'].append( + dict( + repository_dict, + path=updated_repository_path, + ) + ) + elif repository_path.startswith('ssh://'): + config['location']['repositories'].append(repository_dict) else: - rewritten_repository = f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" + rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" logs.append( logging.makeLogRecord( dict( levelno=logging.WARNING, levelname='WARNING', - msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository}" as "{rewritten_repository}"', + msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository_path}" as "{rewritten_repository_path}"', ) ) ) - config['location']['repositories'].append(rewritten_repository) + config['location']['repositories'].append( + dict( + repository_dict, + path=rewritten_repository_path, + ) + ) else: - config['location']['repositories'].append(repository) + config['location']['repositories'].append(repository_dict) return logs diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 8b2a1ab..aacf375 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -57,7 +57,12 @@ def parse_overrides(raw_overrides): for raw_override in raw_overrides: try: raw_keys, value = raw_override.split('=', 1) - parsed_overrides.append((tuple(raw_keys.split('.')), convert_value_type(value),)) + parsed_overrides.append( + ( + tuple(raw_keys.split('.')), + convert_value_type(value), + ) + ) except ValueError: raise ValueError( f"Invalid override '{raw_override}'. Make sure you use the form: SECTION.OPTION=VALUE" @@ -75,5 +80,5 @@ def apply_overrides(config, raw_overrides): ''' overrides = parse_overrides(raw_overrides) - for (keys, value) in overrides: + for keys, value in overrides: set_values(config, keys, value) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index d4d57ab..2650085 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -3,6 +3,17 @@ required: - location additionalProperties: false properties: + constants: + type: object + description: | + Constants to use in the configuration file. All occurrences of the + constant name within culy braces will be replaced with the value. + For example, if you have a constant named "hostname" with the value + "myhostname", then the string "{hostname}" will be replaced with + "myhostname" in the configuration file. + example: + hostname: myhostname + prefix: myprefix location: type: object description: | @@ -29,19 +40,32 @@ properties: repositories: type: array items: - type: string + type: object + required: + - path + properties: + path: + type: string + example: ssh://user@backupserver/./{fqdn} + label: + type: string + example: backupserver description: | - Paths to local or remote repositories (required). Tildes are - expanded. Multiple repositories are backed up to in - sequence. Borg placeholders can be used. See the output of - "borg help placeholders" for details. See ssh_command for - SSH options like identity file or port. If systemd service - is used, then add local repository paths in the systemd - service file to the ReadWritePaths list. + A required list of local or remote repositories with paths + and optional labels (which can be used with the --repository + flag to select a repository). Tildes are expanded. Multiple + repositories are backed up to in sequence. Borg placeholders + can be used. See the output of "borg help placeholders" for + details. See ssh_command for SSH options like identity file + or port. If systemd service is used, then add local + repository paths in the systemd service file to the + ReadWritePaths list. Prior to borgmatic 1.7.10, repositories + was just a list of plain path strings. example: - - ssh://user@backupserver/./sourcehostname.borg - - ssh://user@backupserver/./{fqdn} - - /var/local/backups/local.borg + - path: ssh://user@backupserver/./sourcehostname.borg + label: backupserver + - path: /mnt/backup + label: local working_directory: type: string description: | @@ -354,12 +378,21 @@ properties: description: | Name of the archive. Borg placeholders can be used. See the output of "borg help placeholders" for details. Defaults to - "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this - option, consider also specifying a prefix in the retention - and consistency sections to avoid accidental - pruning/checking of archives with different archive name - formats. + "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". When running + actions like rlist, info, or check, borgmatic automatically + tries to match only archives created with this name format. example: "{hostname}-documents-{now}" + match_archives: + type: string + description: | + A Borg pattern for filtering down the archives used by + borgmatic actions that operate on multiple archives. For + Borg 1.x, use a shell pattern here and see the output of + "borg help placeholders" for details. For Borg 2.x, see the + output of "borg help match-archives". If match_archives is + not specified, borgmatic defaults to deriving the + match_archives value from archive_name_format. + example: "sh:{hostname}-*" relocated_repo_access_is_ok: type: boolean description: | @@ -453,10 +486,12 @@ properties: prefix: type: string description: | - When pruning, only consider archive names starting with this - prefix. Borg placeholders can be used. See the output of - "borg help placeholders" for details. Defaults to - "{hostname}-". Use an empty value to disable the default. + Deprecated. When pruning, only consider archive names + starting with this prefix. Borg placeholders can be used. + See the output of "borg help placeholders" for details. + If a prefix is not specified, borgmatic defaults to + matching archives based on the archive_name_format (see + above). example: sourcehostname consistency: type: object @@ -514,12 +549,12 @@ properties: items: type: string description: | - Paths to a subset of the repositories in the location - section on which to run consistency checks. Handy in case - some of your repositories are very large, and so running - consistency checks on them would take too long. Defaults to - running consistency checks on all repositories configured in - the location section. + Paths or labels for a subset of the repositories in the + location section on which to run consistency checks. Handy + in case some of your repositories are very large, and so + running consistency checks on them would take too long. + Defaults to running consistency checks on all repositories + configured in the location section. example: - user@backupserver:sourcehostname.borg check_last: @@ -532,11 +567,12 @@ properties: prefix: type: string description: | - When performing the "archives" check, only consider archive - names starting with this prefix. Borg placeholders can be - used. See the output of "borg help placeholders" for - details. Defaults to "{hostname}-". Use an empty value to - disable the default. + Deprecated. When performing the "archives" check, only + consider archive names starting with this prefix. Borg + placeholders can be used. See the output of "borg help + placeholders" for details. If a prefix is not specified, + borgmatic defaults to matching archives based on the + archive_name_format (see above). example: sourcehostname output: type: object @@ -905,14 +941,14 @@ properties: type: string enum: ['sql'] description: | - Database dump output format. Currenly only "sql" - is supported. Defaults to "sql" for a single - database. Or, when database name is "all" and - format is blank, dumps all databases to a single - file. But if a format is specified with an "all" - database name, dumps each database to a separate - file of that format, allowing more convenient - restores of individual databases. + Database dump output format. Currently only + "sql" is supported. Defaults to "sql" for a + single database. Or, when database name is "all" + and format is blank, dumps all databases to a + single file. But if a format is specified with + an "all" database name, dumps each database to a + separate file of that format, allowing more + convenient restores of individual databases. example: directory add_drop_database: type: boolean diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 5828380..537f4be 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -1,9 +1,13 @@ import os import jsonschema -import pkg_resources import ruamel.yaml +try: + import importlib_metadata +except ModuleNotFoundError: # pragma: nocover + import importlib.metadata as importlib_metadata + from borgmatic.config import environment, load, normalize, override @@ -11,8 +15,17 @@ def schema_filename(): ''' Path to the installed YAML configuration schema file, used to validate and parse the configuration. + + Raise FileNotFoundError when the schema path does not exist. ''' - return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml') + try: + return next( + str(path.locate()) + for path in importlib_metadata.files('borgmatic') + if path.match('config/schema.yaml') + ) + except StopIteration: + raise FileNotFoundError('Configuration file schema could not be found') def format_json_error_path_element(path_element): @@ -20,9 +33,9 @@ def format_json_error_path_element(path_element): Given a path element into a JSON data structure, format it for display as a string. ''' if isinstance(path_element, int): - return str('[{}]'.format(path_element)) + return str(f'[{path_element}]') - return str('.{}'.format(path_element)) + return str(f'.{path_element}') def format_json_error(error): @@ -30,10 +43,10 @@ def format_json_error(error): Given an instance of jsonschema.exceptions.ValidationError, format it for display as a string. ''' if not error.path: - return 'At the top level: {}'.format(error.message) + return f'At the top level: {error.message}' formatted_path = ''.join(format_json_error_path_element(element) for element in error.path) - return "At '{}': {}".format(formatted_path.lstrip('.'), error.message) + return f"At '{formatted_path.lstrip('.')}': {error.message}" class Validation_error(ValueError): @@ -54,9 +67,10 @@ class Validation_error(ValueError): ''' Render a validation error as a user-facing string. ''' - return 'An error occurred while parsing a configuration file at {}:\n'.format( - self.config_filename - ) + '\n'.join(error for error in self.errors) + return ( + f'An error occurred while parsing a configuration file at {self.config_filename}:\n' + + '\n'.join(error for error in self.errors) + ) def apply_logical_validation(config_filename, parsed_configuration): @@ -68,13 +82,14 @@ def apply_logical_validation(config_filename, parsed_configuration): location_repositories = parsed_configuration.get('location', {}).get('repositories') check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', []) for repository in check_repositories: - if repository not in location_repositories: + if not any( + repositories_match(repository, config_repository) + for config_repository in location_repositories + ): raise Validation_error( config_filename, ( - 'Unknown repository in the "consistency" section\'s "check_repositories": {}'.format( - repository - ), + f'Unknown repository in the "consistency" section\'s "check_repositories": {repository}', ), ) @@ -138,9 +153,17 @@ def normalize_repository_path(repository): def repositories_match(first, second): ''' - Given two repository paths (relative and/or absolute), return whether they match. + Given two repository dicts with keys 'path' (relative and/or absolute), + and 'label', or two repository paths, return whether they match. ''' - return normalize_repository_path(first) == normalize_repository_path(second) + if isinstance(first, str): + first = {'path': first, 'label': first} + if isinstance(second, str): + second = {'path': second, 'label': second} + return (first.get('label') == second.get('label')) or ( + normalize_repository_path(first.get('path')) + == normalize_repository_path(second.get('path')) + ) def guard_configuration_contains_repository(repository, configurations): @@ -160,14 +183,14 @@ def guard_configuration_contains_repository(repository, configurations): config_repository for config in configurations.values() for config_repository in config['location']['repositories'] - if repositories_match(repository, config_repository) + if repositories_match(config_repository, repository) ) ) if count == 0: - raise ValueError('Repository {} not found in configuration files'.format(repository)) + raise ValueError(f'Repository {repository} not found in configuration files') if count > 1: - raise ValueError('Repository {} found in multiple configuration files'.format(repository)) + raise ValueError(f'Repository {repository} found in multiple configuration files') def guard_single_repository_selected(repository, configurations): diff --git a/borgmatic/execute.py b/borgmatic/execute.py index d4b04bf..39691da 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -11,7 +11,7 @@ ERROR_OUTPUT_MAX_LINE_COUNT = 25 BORG_ERROR_EXIT_CODE = 2 -def exit_code_indicates_error(process, exit_code, borg_local_path=None): +def exit_code_indicates_error(command, exit_code, borg_local_path=None): ''' Return True if the given exit code from running a command corresponds to an error. If a Borg local path is given and matches the process' command, then treat exit code 1 as a warning @@ -20,8 +20,6 @@ def exit_code_indicates_error(process, exit_code, borg_local_path=None): if exit_code is None: return False - command = process.args.split(' ') if isinstance(process.args, str) else process.args - if borg_local_path and command[0] == borg_local_path: return bool(exit_code < 0 or exit_code >= BORG_ERROR_EXIT_CODE) @@ -45,6 +43,23 @@ def output_buffer_for_process(process, exclude_stdouts): return process.stderr if process.stdout in exclude_stdouts else process.stdout +def append_last_lines(last_lines, captured_output, line, output_log_level): + ''' + Given a rolling list of last lines, a list of captured output, a line to append, and an output + log level, append the line to the last lines and (if necessary) the captured output. Then log + the line at the requested output log level. + ''' + last_lines.append(line) + + if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT: + last_lines.pop(0) + + if output_log_level is None: + captured_output.append(line) + else: + logger.log(output_log_level, line) + + def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): ''' Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each @@ -100,15 +115,12 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): # Keep the last few lines of output in case the process errors, and we need the output for # the exception below. - last_lines = buffer_last_lines[ready_buffer] - last_lines.append(line) - if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT: - last_lines.pop(0) - - if output_log_level is None: - captured_outputs[ready_process].append(line) - else: - logger.log(output_log_level, line) + append_last_lines( + buffer_last_lines[ready_buffer], + captured_outputs[ready_process], + line, + output_log_level, + ) if not still_running: break @@ -121,13 +133,24 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): if exit_code is None: still_running = True + command = process.args.split(' ') if isinstance(process.args, str) else process.args # If any process errors, then raise accordingly. - if exit_code_indicates_error(process, exit_code, borg_local_path): + if exit_code_indicates_error(command, exit_code, borg_local_path): # If an error occurs, include its output in the raised exception so that we don't # inadvertently hide error output. output_buffer = output_buffer_for_process(process, exclude_stdouts) - last_lines = buffer_last_lines[output_buffer] if output_buffer else [] + + # Collect any straggling output lines that came in since we last gathered output. + while output_buffer: # pragma: no cover + line = output_buffer.readline().rstrip().decode() + if not line: + break + + append_last_lines( + last_lines, captured_outputs[process], line, output_log_level=logging.ERROR + ) + if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT: last_lines.insert(0, '...') @@ -155,8 +178,8 @@ def log_command(full_command, input_file=None, output_file=None): ''' logger.debug( ' '.join(full_command) - + (' < {}'.format(getattr(input_file, 'name', '')) if input_file else '') - + (' > {}'.format(getattr(output_file, 'name', '')) if output_file else '') + + (f" < {getattr(input_file, 'name', '')}" if input_file else '') + + (f" > {getattr(output_file, 'name', '')}" if output_file else '') ) @@ -213,7 +236,11 @@ def execute_command( def execute_command_and_capture_output( - full_command, capture_stderr=False, shell=False, extra_environment=None, working_directory=None, + full_command, + capture_stderr=False, + shell=False, + extra_environment=None, + working_directory=None, ): ''' Execute the given command (a sequence of command/argument strings), capturing and returning its @@ -228,13 +255,18 @@ def execute_command_and_capture_output( environment = {**os.environ, **extra_environment} if extra_environment else None command = ' '.join(full_command) if shell else full_command - output = subprocess.check_output( - command, - stderr=subprocess.STDOUT if capture_stderr else None, - shell=shell, - env=environment, - cwd=working_directory, - ) + try: + output = subprocess.check_output( + command, + stderr=subprocess.STDOUT if capture_stderr else None, + shell=shell, + env=environment, + cwd=working_directory, + ) + except subprocess.CalledProcessError as error: + if exit_code_indicates_error(command, error.returncode): + raise + output = error.output return output.decode() if output is not None else None diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index 756f877..05f7d2f 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -16,7 +16,7 @@ def interpolate_context(config_filename, hook_description, command, context): names/values, interpolate the values by "{name}" into the command and return the result. ''' for name, value in context.items(): - command = command.replace('{%s}' % name, str(value)) + command = command.replace(f'{{{name}}}', str(value)) for unsupported_variable in re.findall(r'{\w+}', command): logger.warning( @@ -38,7 +38,7 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte Raise subprocesses.CalledProcessError if an error occurs in a hook. ''' if not commands: - logger.debug('{}: No commands to run for {} hook'.format(config_filename, description)) + logger.debug(f'{config_filename}: No commands to run for {description} hook') return dry_run_label = ' (dry run; not actually running hooks)' if dry_run else '' @@ -49,19 +49,15 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte ] if len(commands) == 1: - logger.info( - '{}: Running command for {} hook{}'.format(config_filename, description, dry_run_label) - ) + logger.info(f'{config_filename}: Running command for {description} hook{dry_run_label}') else: logger.info( - '{}: Running {} commands for {} hook{}'.format( - config_filename, len(commands), description, dry_run_label - ) + f'{config_filename}: Running {len(commands)} commands for {description} hook{dry_run_label}', ) if umask: parsed_umask = int(str(umask), 8) - logger.debug('{}: Set hook umask to {}'.format(config_filename, oct(parsed_umask))) + logger.debug(f'{config_filename}: Set hook umask to {oct(parsed_umask)}') original_umask = os.umask(parsed_umask) else: original_umask = None @@ -93,9 +89,7 @@ def considered_soft_failure(config_filename, error): if exit_code == SOFT_FAIL_EXIT_CODE: logger.info( - '{}: Command hook exited with soft failure exit code ({}); skipping remaining actions'.format( - config_filename, SOFT_FAIL_EXIT_CODE - ) + f'{config_filename}: Command hook exited with soft failure exit code ({SOFT_FAIL_EXIT_CODE}); skipping remaining actions', ) return True diff --git a/borgmatic/hooks/cronhub.py b/borgmatic/hooks/cronhub.py index cd0ffa5..05ada57 100644 --- a/borgmatic/hooks/cronhub.py +++ b/borgmatic/hooks/cronhub.py @@ -34,17 +34,15 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' - formatted_state = '/{}/'.format(MONITOR_STATE_TO_CRONHUB[state]) + formatted_state = f'/{MONITOR_STATE_TO_CRONHUB[state]}/' ping_url = ( hook_config['ping_url'] .replace('/start/', formatted_state) .replace('/ping/', formatted_state) ) - logger.info( - '{}: Pinging Cronhub {}{}'.format(config_filename, state.name.lower(), dry_run_label) - ) - logger.debug('{}: Using Cronhub ping URL {}'.format(config_filename, ping_url)) + logger.info(f'{config_filename}: Pinging Cronhub {state.name.lower()}{dry_run_label}') + logger.debug(f'{config_filename}: Using Cronhub ping URL {ping_url}') if not dry_run: logging.getLogger('urllib3').setLevel(logging.ERROR) diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index 633b4c3..d669c09 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -34,12 +34,10 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' - ping_url = '{}/{}'.format(hook_config['ping_url'], MONITOR_STATE_TO_CRONITOR[state]) + ping_url = f"{hook_config['ping_url']}/{MONITOR_STATE_TO_CRONITOR[state]}" - logger.info( - '{}: Pinging Cronitor {}{}'.format(config_filename, state.name.lower(), dry_run_label) - ) - logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url)) + logger.info(f'{config_filename}: Pinging Cronitor {state.name.lower()}{dry_run_label}') + logger.debug(f'{config_filename}: Using Cronitor ping URL {ping_url}') if not dry_run: logging.getLogger('urllib3').setLevel(logging.ERROR) diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index 88a99eb..fa7bd9b 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -43,9 +43,9 @@ def call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs): try: module = HOOK_NAME_TO_MODULE[hook_name] except KeyError: - raise ValueError('Unknown hook name: {}'.format(hook_name)) + raise ValueError(f'Unknown hook name: {hook_name}') - logger.debug('{}: Calling {} hook function {}'.format(log_prefix, hook_name, function_name)) + logger.debug(f'{log_prefix}: Calling {hook_name} hook function {function_name}') return getattr(module, function_name)(config, log_prefix, *args, **kwargs) diff --git a/borgmatic/hooks/dump.py b/borgmatic/hooks/dump.py index 43686d3..015ed69 100644 --- a/borgmatic/hooks/dump.py +++ b/borgmatic/hooks/dump.py @@ -33,7 +33,7 @@ def make_database_dump_filename(dump_path, name, hostname=None): Raise ValueError if the database name is invalid. ''' if os.path.sep in name: - raise ValueError('Invalid database name {}'.format(name)) + raise ValueError(f'Invalid database name {name}') return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name) @@ -60,9 +60,7 @@ def remove_database_dumps(dump_path, database_type_name, log_prefix, dry_run): ''' dry_run_label = ' (dry run; not actually removing anything)' if dry_run else '' - logger.debug( - '{}: Removing {} database dumps{}'.format(log_prefix, database_type_name, dry_run_label) - ) + logger.debug(f'{log_prefix}: Removing {database_type_name} database dumps{dry_run_label}') expanded_path = os.path.expanduser(dump_path) @@ -78,4 +76,4 @@ def convert_glob_patterns_to_borg_patterns(patterns): Convert a sequence of shell glob patterns like "/etc/*" to the corresponding Borg archive patterns like "sh:etc/*". ''' - return ['sh:{}'.format(pattern.lstrip(os.path.sep)) for pattern in patterns] + return [f'sh:{pattern.lstrip(os.path.sep)}' for pattern in patterns] diff --git a/borgmatic/hooks/healthchecks.py b/borgmatic/hooks/healthchecks.py index 6ad8449..4cafc49 100644 --- a/borgmatic/hooks/healthchecks.py +++ b/borgmatic/hooks/healthchecks.py @@ -99,7 +99,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ ping_url = ( hook_config['ping_url'] if hook_config['ping_url'].startswith('http') - else 'https://hc-ping.com/{}'.format(hook_config['ping_url']) + else f"https://hc-ping.com/{hook_config['ping_url']}" ) dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' @@ -111,12 +111,10 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ healthchecks_state = MONITOR_STATE_TO_HEALTHCHECKS.get(state) if healthchecks_state: - ping_url = '{}/{}'.format(ping_url, healthchecks_state) + ping_url = f'{ping_url}/{healthchecks_state}' - logger.info( - '{}: Pinging Healthchecks {}{}'.format(config_filename, state.name.lower(), dry_run_label) - ) - logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url)) + logger.info(f'{config_filename}: Pinging Healthchecks {state.name.lower()}{dry_run_label}') + logger.debug(f'{config_filename}: Using Healthchecks ping URL {ping_url}') if state in (monitor.State.FINISH, monitor.State.FAIL, monitor.State.LOG): payload = format_buffered_logs_for_payload() diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index 8c3cab7..781e5f2 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -27,7 +27,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): ''' dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - logger.info('{}: Dumping MongoDB databases{}'.format(log_prefix, dry_run_label)) + logger.info(f'{log_prefix}: Dumping MongoDB databases{dry_run_label}') processes = [] for database in databases: @@ -38,9 +38,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): dump_format = database.get('format', 'archive') logger.debug( - '{}: Dumping MongoDB database {} to {}{}'.format( - log_prefix, name, dump_filename, dry_run_label - ) + f'{log_prefix}: Dumping MongoDB database {name} to {dump_filename}{dry_run_label}', ) if dry_run: continue @@ -126,9 +124,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) restore_command = build_restore_command(extract_process, database, dump_filename) - logger.debug( - '{}: Restoring MongoDB database {}{}'.format(log_prefix, database['name'], dry_run_label) - ) + logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}") if dry_run: return @@ -165,4 +161,7 @@ def build_restore_command(extract_process, database, dump_filename): command.extend(('--authenticationDatabase', database['authentication_database'])) if 'restore_options' in database: command.extend(database['restore_options'].split(' ')) + if database['schemas']: + for schema in database['schemas']: + command.extend(('--nsInclude', schema)) return command diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index e53b896..793b78b 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -88,9 +88,7 @@ def execute_dump_command( + (('--user', database['username']) if 'username' in database else ()) + ('--databases',) + database_names - # Use shell redirection rather than execute_command(output_file=open(...)) to prevent - # the open() call on a named pipe from hanging the main borgmatic process. - + ('>', dump_filename) + + ('--result-file', dump_filename) ) logger.debug( @@ -102,7 +100,9 @@ def execute_dump_command( dump.create_named_pipe_for_dump(dump_filename) return execute_command( - dump_command, shell=True, extra_environment=extra_environment, run_to_completion=False, + dump_command, + extra_environment=extra_environment, + run_to_completion=False, ) @@ -119,7 +119,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] - logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label)) + logger.info(f'{log_prefix}: Dumping MySQL databases{dry_run_label}') for database in databases: dump_path = make_dump_path(location_config) @@ -209,9 +209,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None - logger.debug( - '{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label) - ) + logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}") if dry_run: return diff --git a/borgmatic/hooks/pagerduty.py b/borgmatic/hooks/pagerduty.py index fbb67fb..561b1e2 100644 --- a/borgmatic/hooks/pagerduty.py +++ b/borgmatic/hooks/pagerduty.py @@ -29,14 +29,12 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ ''' if state != monitor.State.FAIL: logger.debug( - '{}: Ignoring unsupported monitoring {} in PagerDuty hook'.format( - config_filename, state.name.lower() - ) + f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in PagerDuty hook', ) return dry_run_label = ' (dry run; not actually sending)' if dry_run else '' - logger.info('{}: Sending failure event to PagerDuty {}'.format(config_filename, dry_run_label)) + logger.info(f'{config_filename}: Sending failure event to PagerDuty {dry_run_label}') if dry_run: return @@ -50,7 +48,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ 'routing_key': hook_config['integration_key'], 'event_action': 'trigger', 'payload': { - 'summary': 'backup failed on {}'.format(hostname), + 'summary': f'backup failed on {hostname}', 'severity': 'error', 'source': hostname, 'timestamp': local_timestamp, @@ -65,7 +63,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ }, } ) - logger.debug('{}: Using PagerDuty payload: {}'.format(config_filename, payload)) + logger.debug(f'{config_filename}: Using PagerDuty payload: {payload}') logging.getLogger('urllib3').setLevel(logging.ERROR) try: diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 3d3676f..bcc48ef 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -1,4 +1,5 @@ import csv +import itertools import logging import os @@ -93,7 +94,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] - logger.info('{}: Dumping PostgreSQL databases{}'.format(log_prefix, dry_run_label)) + logger.info(f'{log_prefix}: Dumping PostgreSQL databases{dry_run_label}') for database in databases: extra_environment = make_extra_environment(database) @@ -122,7 +123,12 @@ def dump_databases(databases, log_prefix, location_config, dry_run): continue command = ( - (dump_command, '--no-password', '--clean', '--if-exists',) + ( + dump_command, + '--no-password', + '--clean', + '--if-exists', + ) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) @@ -145,7 +151,9 @@ def dump_databases(databases, log_prefix, location_config, dry_run): if dump_format == 'directory': dump.create_parent_directory_for_dump(dump_filename) execute_command( - command, shell=True, extra_environment=extra_environment, + command, + shell=True, + extra_environment=extra_environment, ) else: dump.create_named_pipe_for_dump(dump_filename) @@ -225,12 +233,16 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (('--username', database['username']) if 'username' in database else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (() if extract_process else (dump_filename,)) + + tuple( + itertools.chain.from_iterable(('--schema', schema) for schema in database['schemas']) + if database['schemas'] + else () + ) ) + extra_environment = make_extra_environment(database) - logger.debug( - '{}: Restoring PostgreSQL database {}{}'.format(log_prefix, database['name'], dry_run_label) - ) + logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") if dry_run: return diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py index 9e7ecf3..d9f105d 100644 --- a/borgmatic/hooks/sqlite.py +++ b/borgmatic/hooks/sqlite.py @@ -26,7 +26,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] - logger.info('{}: Dumping SQLite databases{}'.format(log_prefix, dry_run_label)) + logger.info(f'{log_prefix}: Dumping SQLite databases{dry_run_label}') for database in databases: database_path = database['path'] diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 0916bfa..5206592 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -68,7 +68,7 @@ class Multi_stream_handler(logging.Handler): def emit(self, record): ''' - Dispatch the log record to the approriate stream handler for the record's log level. + Dispatch the log record to the appropriate stream handler for the record's log level. ''' self.log_level_to_handler[record.levelno].emit(record) @@ -108,7 +108,7 @@ def color_text(color, message): if not color: return message - return '{}{}{}'.format(color, message, colorama.Style.RESET_ALL) + return f'{color}{message}{colorama.Style.RESET_ALL}' def add_logging_level(level_name, level_number): @@ -156,6 +156,7 @@ def configure_logging( log_file_log_level=None, monitoring_log_level=None, log_file=None, + log_file_format=None, ): ''' Configure logging to go to both the console and (syslog or log file). Use the given log levels, @@ -200,12 +201,18 @@ def configure_logging( if syslog_path and not interactive_console(): syslog_handler = logging.handlers.SysLogHandler(address=syslog_path) - syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s')) + syslog_handler.setFormatter( + logging.Formatter('borgmatic: {levelname} {message}', style='{') # noqa: FS003 + ) syslog_handler.setLevel(syslog_log_level) handlers = (console_handler, syslog_handler) elif log_file: file_handler = logging.handlers.WatchedFileHandler(log_file) - file_handler.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s')) + file_handler.setFormatter( + logging.Formatter( + log_file_format or '[{asctime}] {levelname}: {message}', style='{' # noqa: FS003 + ) + ) file_handler.setLevel(log_file_log_level) handlers = (console_handler, file_handler) else: diff --git a/docs/Dockerfile b/docs/Dockerfile index 8800cc1..b612596 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.17.1 as borgmatic +FROM docker.io/alpine:3.17.1 as borgmatic COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib @@ -8,7 +8,7 @@ RUN borgmatic --help > /command-line.txt \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ && borgmatic "$action" --help >> /command-line.txt; done -FROM node:19.5.0-alpine as html +FROM docker.io/node:19.5.0-alpine as html ARG ENVIRONMENT=production @@ -18,6 +18,7 @@ RUN npm install @11ty/eleventy \ @11ty/eleventy-plugin-syntaxhighlight \ @11ty/eleventy-plugin-inclusive-language \ @11ty/eleventy-navigation \ + eleventy-plugin-code-clipboard \ markdown-it \ markdown-it-anchor \ markdown-it-replace-link @@ -27,7 +28,7 @@ COPY . /source RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \ && mv /output/docs/index.html /output/index.html -FROM nginx:1.22.1-alpine +FROM docker.io/nginx:1.22.1-alpine COPY --from=html /output /usr/share/nginx/html COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml diff --git a/docs/_includes/components/toc.css b/docs/_includes/components/toc.css index 039673f..82cf15e 100644 --- a/docs/_includes/components/toc.css +++ b/docs/_includes/components/toc.css @@ -94,7 +94,7 @@ display: block; } -/* Footer catgory navigation */ +/* Footer category navigation */ .elv-cat-list-active { font-weight: 600; } diff --git a/docs/_includes/index.css b/docs/_includes/index.css index ca1c2df..f1d4c57 100644 --- a/docs/_includes/index.css +++ b/docs/_includes/index.css @@ -533,3 +533,18 @@ main .elv-toc + h1 .direct-link { .header-anchor:hover::after { content: " 🔗"; } + +.mdi { + display: inline-block; + width: 1em; + height: 1em; + background-color: currentColor; + -webkit-mask: no-repeat center / 100%; + mask: no-repeat center / 100%; + -webkit-mask-image: var(--svg); + mask-image: var(--svg); +} + +.mdi.mdi-content-copy { + --svg: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' width='24' height='24'%3E%3Cpath fill='black' d='M19 21H8V7h11m0-2H8a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h11a2 2 0 0 0 2-2V7a2 2 0 0 0-2-2m-3-4H4a2 2 0 0 0-2 2v14h2V3h12V1Z'/%3E%3C/svg%3E"); +} diff --git a/docs/_includes/layouts/base.njk b/docs/_includes/layouts/base.njk index 361967c..dbb19d9 100644 --- a/docs/_includes/layouts/base.njk +++ b/docs/_includes/layouts/base.njk @@ -3,6 +3,7 @@ + {{ subtitle + ' - ' if subtitle}}{{ title }} {%- set css %} {% include 'index.css' %} @@ -22,6 +23,6 @@ {{ content | safe }} - + {% initClipboardJS %} diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index c0bcfb8..426e1a8 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -66,6 +66,9 @@ variables you can use here: * `configuration_filename`: borgmatic configuration filename in which the hook was defined + * `log_file` + New in version 1.7.12: + path of the borgmatic log file, only set when the `--log-file` flag is used * `repository`: path of the current repository as configured in the current borgmatic configuration file diff --git a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md index 0f57855..04ccbf7 100644 --- a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md +++ b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md @@ -49,9 +49,12 @@ location: - /home repositories: - - /mnt/removable/backup.borg + - path: /mnt/removable/backup.borg ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + Then, write a `before_backup` hook in that same configuration file that uses the external `findmnt` utility to see whether the drive is mounted before proceeding. @@ -79,13 +82,16 @@ location: - /home repositories: - - ssh://me@buddys-server.org/./backup.borg + - path: ssh://me@buddys-server.org/./backup.borg hooks: before_backup: - ping -q -c 1 buddys-server.org > /dev/null || exit 75 ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + Or to only run backups if the battery level is high enough: ```yaml @@ -110,8 +116,8 @@ There are some caveats you should be aware of with this feature. * You'll generally want to put a soft failure command in the `before_backup` hook, so as to gate whether the backup action occurs. While a soft failure is also supported in the `after_backup` hook, returning a soft failure there - won't prevent any actions from occuring, because they've already occurred! - Similiarly, you can return a soft failure from an `on_error` hook, but at + won't prevent any actions from occurring, because they've already occurred! + Similarly, you can return a soft failure from an `on_error` hook, but at that point it's too late to prevent the error. * Returning a soft failure does prevent further commands in the same hook from executing. So, like a standard error, it is an "early out". Unlike a standard diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index bc21b65..91dba18 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -136,6 +136,53 @@ hooks: format: sql ``` +### Containers + +If your database is running within a Docker container and borgmatic is too, no +problem—simply configure borgmatic to connect to the container's name on its +exposed port. For instance: + +```yaml +hooks: + postgresql_databases: + - name: users + hostname: your-database-container-name + port: 5433 + username: postgres + password: trustsome1 +``` + +But what if borgmatic is running on the host? You can still connect to a +database container if its ports are properly exposed to the host. For +instance, when running the database container with Docker, you can specify +`--publish 127.0.0.1:5433:5432` so that it exposes the container's port 5432 +to port 5433 on the host (only reachable on localhost, in this case). Or the +same thing with Docker Compose: + +```yaml +services: + your-database-container-name: + image: postgres + ports: + - 127.0.0.1:5433:5432 +``` + +And then you can connect to the database from borgmatic running on the host: + +```yaml +hooks: + postgresql_databases: + - name: users + hostname: 127.0.0.1 + port: 5433 + username: postgres + password: trustsome1 +``` + +Of course, alter the ports in these examples to suit your particular database +system. + + ### No source directories New in version 1.7.1 If you @@ -154,7 +201,6 @@ hooks: ``` - ### External passwords If you don't want to keep your database passwords in your borgmatic @@ -231,7 +277,8 @@ If you have a single repository in your borgmatic configuration file(s), no problem: the `restore` action figures out which repository to use. But if you have multiple repositories configured, then you'll need to specify -the repository path containing the archive to restore. Here's an example: +the repository to use via the `--repository` flag. This can be done either +with the repository's path or its label as configured in your borgmatic configuration file. ```bash borgmatic restore --repository repo.borg --archive host-2023-... @@ -277,6 +324,17 @@ includes any combined dump file named "all" and any other individual database dumps found in the archive. +### Restore particular schemas + +New in version 1.7.13 With +PostgreSQL and MongoDB, you can limit the restore to a single schema found +within the database dump: + +```bash +borgmatic restore --archive latest --database users --schema tentant1 +``` + + ### Limitations There are a few important limitations with borgmatic's current database @@ -334,6 +392,23 @@ dumps with any database system. ## Troubleshooting +### PostgreSQL/MySQL authentication errors + +With PostgreSQL and MySQL/MariaDB, if you're getting authentication errors +when borgmatic tries to connect to your database, a natural reaction is to +increase your borgmatic verbosity with `--verbosity 2` and go looking in the +logs. You'll notice however that your database password does not show up in +the logs. This is likely not the cause of the authentication problem unless +you mistyped your password, however; borgmatic passes your password to the +database via an environment variable that does not appear in the logs. + +The cause of an authentication error is often on the database side—in the +configuration of which users are allowed to connect and how they are +authenticated. For instance, with PostgreSQL, check your +[pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) +file for that configuration. + + ### MySQL table lock errors If you encounter table lock errors during a database dump with MySQL/MariaDB, diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index fbc1d24..027a814 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -25,7 +25,7 @@ so that you can run borgmatic commands while you're hacking on them to make sure your changes work. ```bash -cd borgmatic/ +cd borgmatic pip3 install --user --editable . ``` @@ -51,7 +51,6 @@ pip3 install --user tox Finally, to actually run tests, run: ```bash -cd borgmatic tox ``` @@ -74,6 +73,15 @@ can ask isort to order your imports for you: tox -e isort ``` +Similarly, if you get errors about spelling mistakes in source code, you can +ask [codespell](https://github.com/codespell-project/codespell) to correct +them: + +```bash +tox -e codespell +``` + + ### End-to-end tests borgmatic additionally includes some end-to-end tests that integration test @@ -87,12 +95,36 @@ If you would like to run the full test suite, first install Docker and [Docker Compose](https://docs.docker.com/compose/install/). Then run: ```bash -scripts/run-full-dev-tests +scripts/run-end-to-end-dev-tests ``` Note that this scripts assumes you have permission to run Docker. If you don't, then you may need to run with `sudo`. + +#### Podman + +New in version 1.7.12 +borgmatic's end-to-end tests optionally support using +[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md) +[Podman](https://podman.io/) instead of Docker. + +Setting up Podman is outside the scope of this documentation, but here are +some key points to double-check: + + * Install Podman along with `podman-docker` and your desired networking + support. + * Configure `/etc/subuid` and `/etc/subgid` to map users/groups for the + non-root user who will run tests. + * Create a non-root Podman socket for that user: + ```bash + systemctl --user enable --now podman.socket + ``` + +Then you'll be able to run end-to-end tests as per normal, and the test script +will automatically use your non-root Podman socket instead of a Docker socket. + + ## Code style Start with [PEP 8](https://www.python.org/dev/peps/pep-0008/). But then, apply @@ -101,10 +133,10 @@ the following deviations from it: * For strings, prefer single quotes over double quotes. * Limit all lines to a maximum of 100 characters. * Use trailing commas within multiline values or argument lists. - * For multiline constructs, put opening and closing delimeters on lines + * For multiline constructs, put opening and closing delimiters on lines separate from their contents. * Within multiline constructs, use standard four-space indentation. Don't align - indentation with an opening delimeter. + indentation with an opening delimiter. borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code formatter, the [Flake8](http://flake8.pycqa.org/en/latest/) code checker, and @@ -141,3 +173,15 @@ http://localhost:8080 to view the documentation with your changes. To close the documentation server, ctrl-C the script. Note that it does not currently auto-reload, so you'll need to stop it and re-run it for any additional documentation changes to take effect. + + +#### Podman + +New in version 1.7.12 +borgmatic's developer build for documentation optionally supports using +[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md) +[Podman](https://podman.io/) instead of Docker. + +Setting up Podman is outside the scope of this documentation. But once you +install `podman-docker`, then `scripts/dev-docs` should automatically use +Podman instead of Docker. diff --git a/docs/how-to/extract-a-backup.md b/docs/how-to/extract-a-backup.md index 4285c78..164fc13 100644 --- a/docs/how-to/extract-a-backup.md +++ b/docs/how-to/extract-a-backup.md @@ -51,7 +51,8 @@ If you have a single repository in your borgmatic configuration file(s), no problem: the `extract` action figures out which repository to use. But if you have multiple repositories configured, then you'll need to specify -the repository path containing the archive to extract. Here's an example: +the repository to use via the `--repository` flag. This can be done either +with the repository's path or its label as configured in your borgmatic configuration file. ```bash borgmatic extract --repository repo.borg --archive host-2023-... diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index 57a2381..73020ed 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -111,7 +111,7 @@ By default, borgmatic logs to a local syslog-compatible daemon if one is present and borgmatic is running in a non-interactive console. Where those logs show up depends on your particular system. If you're using systemd, try running `journalctl -xe`. Otherwise, try viewing `/var/log/syslog` or -similiar. +similar. You can customize the log level used for syslog logging with the `--syslog-verbosity` flag, and this is independent from the console logging @@ -154,5 +154,39 @@ borgmatic --log-file /path/to/file.log Note that if you use the `--log-file` flag, you are responsible for rotating the log file so it doesn't grow too large, for example with -[logrotate](https://wiki.archlinux.org/index.php/Logrotate). Also, there is a -`--log-file-verbosity` flag to customize the log file's log level. +[logrotate](https://wiki.archlinux.org/index.php/Logrotate). + +You can the `--log-file-verbosity` flag to customize the log file's log level: + +```bash +borgmatic --log-file /path/to/file.log --log-file-verbosity 2 +``` + +New in version 1.7.11 Use the +`--log-file-format` flag to override the default log message format. This +format string can contain a series of named placeholders wrapped in curly +brackets. For instance, the default log format is: `[{asctime}] {levelname}: +{message}`. This means each log message is recorded as the log time (in square +brackets), a logging level name, a colon, and the actual log message. + +So if you just want each log message to get logged *without* a timestamp or a +logging level name: + +```bash +borgmatic --log-file /path/to/file.log --log-file-format "{message}" +``` + +Here is a list of available placeholders: + + * `{asctime}`: time the log message was created + * `{levelname}`: level of the log message (`INFO`, `DEBUG`, etc.) + * `{lineno}`: line number in the source file where the log message originated + * `{message}`: actual log message + * `{pathname}`: path of the source file where the log message originated + +See the [Python logging +documentation](https://docs.python.org/3/library/logging.html#logrecord-attributes) +for additional placeholders. + +Note that this `--log-file-format` flg only applies to the specified +`--log-file` and not to syslog or other logging. diff --git a/docs/how-to/make-backups-redundant.md b/docs/how-to/make-backups-redundant.md index f77780f..2a4b812 100644 --- a/docs/how-to/make-backups-redundant.md +++ b/docs/how-to/make-backups-redundant.md @@ -20,11 +20,13 @@ location: # Paths of local or remote repositories to backup to. repositories: - - ssh://1234@usw-s001.rsync.net/./backups.borg - - ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo - - /var/lib/backups/local.borg + - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo + - path: /var/lib/backups/local.borg ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + When you run borgmatic with this configuration, it invokes Borg once for each configured repository in sequence. (So, not in parallel.) That means—in each repository—borgmatic creates a single new backup archive containing all of @@ -32,9 +34,8 @@ your source directories. Here's a way of visualizing what borgmatic does with the above configuration: -1. Backup `/home` and `/etc` to `1234@usw-s001.rsync.net:backups.borg` -2. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo` -3. Backup `/home` and `/etc` to `/var/lib/backups/local.borg` +1. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo` +2. Backup `/home` and `/etc` to `/var/lib/backups/local.borg` This gives you redundancy of your data across repositories and even potentially across providers. diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 2bf099e..9ee93f4 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -54,6 +54,93 @@ choice](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot), each entry using borgmatic's `--config` flag instead of relying on `/etc/borgmatic.d`. + +## Archive naming + +If you've got multiple borgmatic configuration files, you might want to create +archives with different naming schemes for each one. This is especially handy +if each configuration file is backing up to the same Borg repository but you +still want to be able to distinguish backup archives for one application from +another. + +borgmatic supports this use case with an `archive_name_format` option. The +idea is that you define a string format containing a number of [Borg +placeholders](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-placeholders), +and borgmatic uses that format to name any new archive it creates. For +instance: + +```yaml +storage: + ... + archive_name_format: home-directories-{now} +``` + +This means that when borgmatic creates an archive, its name will start with +the string `home-directories-` and end with a timestamp for its creation time. +If `archive_name_format` is unspecified, the default is +`{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a +timestamp in a particular format. + +New in version 1.7.11 borgmatic +uses the `archive_name_format` option to automatically limit which archives +get used for actions operating on multiple archives. This prevents, for +instance, duplicate archives from showing up in `rlist` or `info` results—even +if the same repository appears in multiple borgmatic configuration files. To +take advantage of this feature, simply use a different `archive_name_format` +in each configuration file. + +Under the hood, borgmatic accomplishes this by substituting globs for certain +ephemeral data placeholders in your `archive_name_format`—and using the result +to filter archives when running supported actions. + +For instance, let's say that you have this in your configuration: + +```yaml +storage: + ... + archive_name_format: {hostname}-user-data-{now} +``` + +borgmatic considers `{now}` an emphemeral data placeholder that will probably +change per archive, while `{hostname}` won't. So it turns the example value +into `{hostname}-user-data-*` and applies it to filter down the set of +archives used for actions like `rlist`, `info`, `prune`, `check`, etc. + +The end result is that when borgmatic runs the actions for a particular +application-specific configuration file, it only operates on the archives +created for that application. Of course, this doesn't apply to actions like +`compact` that operate on an entire repository. + +If this behavior isn't quite smart enough for your needs, you can use the +`match_archives` option to override the pattern that borgmatic uses for +filtering archives. For example: + +```yaml +storage: + ... + archive_name_format: {hostname}-user-data-{now} + match_archives: sh:myhost-user-data-* +``` + +For Borg 1.x, use a shell pattern for the `match_archives` value and see the +[Borg patterns +documentation](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns) +for more information. For Borg 2.x, see the [match archives +documentation](https://borgbackup.readthedocs.io/en/2.0.0b5/usage/help.html#borg-help-match-archives). + +Some borgmatic command-line actions also have a `--match-archives` flag that +overrides both the auto-matching behavior and the `match_archives` +configuration option. + +Prior to 1.7.11 The way to +limit the archives used for the `prune` action was a `prefix` option in the +`retention` section for matching against the start of archive names. And the +option for limiting the archives used for the `check` action was a separate +`prefix` in the `consistency` section. Both of these options are deprecated in +favor of the auto-matching behavior (or `match_archives`/`--match-archives`) +in newer versions of borgmatic. + + ## Configuration includes Once you have multiple different configuration files, you might want to share @@ -185,9 +272,140 @@ Once this include gets merged in, the resulting configuration would have a When there's an option collision between the local file and the merged include, the local file's option takes precedence. + +#### List merge + New in version 1.6.1 Colliding list values are appended together. +New in version 1.7.12 If there +is a list value from an include that you *don't* want in your local +configuration file, you can omit it with an `!omit` tag. For instance: + +```yaml +<<: !include /etc/borgmatic/common.yaml + +location: + source_directories: + - !omit /home + - /var +``` + +And `common.yaml` like this: + +```yaml +location: + source_directories: + - /home + - /etc +``` + +Once this include gets merged in, the resulting configuration will have a +`source_directories` value of `/etc` and `/var`—with `/home` omitted. + +This feature currently only works on scalar (e.g. string or number) list items +and will not work elsewhere in a configuration file. Be sure to put the +`!omit` tag *before* the list item (after the dash). Putting `!omit` after the +list item will not work, as it gets interpreted as part of the string. Here's +an example of some things not to do: + +```yaml +<<: !include /etc/borgmatic/common.yaml + +location: + source_directories: + # Do not do this! It will not work. "!omit" belongs before "/home". + - /home !omit + + # Do not do this either! "!omit" only works on scalar list items. + repositories: !omit + # Also do not do this for the same reason! This is a list item, but it's + # not a scalar. + - !omit path: repo.borg +``` + +Additionally, the `!omit` tag only works in a configuration file that also +performs a merge include with `<<: !include`. It doesn't make sense within, +for instance, an included configuration file itself (unless it in turn +performs its own merge include). That's because `!omit` only applies to the +file doing the include; it doesn't work in reverse or propagate through +includes. + + +### Shallow merge + +Even though deep merging is generally pretty handy for included files, +sometimes you want specific sections in the local file to take precedence over +included sections—without any merging occurring for them. + +New in version 1.7.12 That's +where the `!retain` tag comes in. Whenever you're merging an included file +into your configuration file, you can optionally add the `!retain` tag to +particular local mappings or lists to retain the local values and ignore +included values. + +For instance, start with this configuration file containing the `!retain` tag +on the `retention` mapping: + +```yaml +<<: !include /etc/borgmatic/common.yaml + +location: + repositories: + - path: repo.borg + +retention: !retain + keep_daily: 5 +``` + +And `common.yaml` like this: + +```yaml +location: + repositories: + - path: common.borg + +retention: + keep_hourly: 24 + keep_daily: 7 +``` + +Once this include gets merged in, the resulting configuration will have a +`keep_daily` value of `5` and nothing else in the `retention` section. That's +because the `!retain` tag says to retain the local version of `retention` and +ignore any values coming in from the include. But because the `repositories` +list doesn't have a `!retain` tag, it still gets merged together to contain +both `common.borg` and `repo.borg`. + +The `!retain` tag can only be placed on mappings and lists, and it goes right +after the name of the option (and its colon) on the same line. The effects of +`!retain` are recursive, meaning that if you place a `!retain` tag on a +top-level mapping, even deeply nested values within it will not be merged. + +Additionally, the `!retain` tag only works in a configuration file that also +performs a merge include with `<<: !include`. It doesn't make sense within, +for instance, an included configuration file itself (unless it in turn +performs its own merge include). That's because `!retain` only applies to the +file doing the include; it doesn't work in reverse or propagate through +includes. + + +## Debugging includes + +New in version 1.7.12 If you'd +like to see what the loaded configuration looks like after includes get merged +in, run `validate-borgmatic-config` on your configuration file: + +```bash +sudo validate-borgmatic-config --show +``` + +You'll need to specify your configuration file with `--config` if it's not in +a default location. + +This will output the merged configuration as borgmatic sees it, which can be +helpful for understanding how your includes work in practice. + ## Configuration overrides @@ -255,3 +473,51 @@ Be sure to quote your overrides if they contain spaces or other characters that your shell may interpret. An alternate to command-line overrides is passing in your values via [environment variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). + + +## Constant interpolation + +New in version 1.7.10 Another +tool is borgmatic's support for defining custom constants. This is similar to +the [variable interpolation +feature](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation) +for command hooks, but the constants feature lets you substitute your own +custom values into anywhere in the entire configuration file. (Constants don't +work across includes or separate configuration files though.) + +Here's an example usage: + +```yaml +constants: + user: foo + archive_prefix: bar + +location: + source_directories: + - /home/{user}/.config + - /home/{user}/.ssh + ... + +storage: + archive_name_format: '{archive_prefix}-{now}' +``` + +In this example, when borgmatic runs, all instances of `{user}` get replaced +with `foo` and all instances of `{archive-prefix}` get replaced with `bar-`. +(And in this particular example, `{now}` doesn't get replaced with anything, +but gets passed directly to Borg.) After substitution, the logical result +looks something like this: + +```yaml +location: + source_directories: + - /home/foo/.config + - /home/foo/.ssh + ... + +storage: + archive_name_format: 'bar-{now}' +``` + +An alternate to constants is passing in your values via [environment +variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index 3d119b4..0777ebb 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -53,7 +53,8 @@ This runs Borg's `rlist` command once on each configured borgmatic repository. (The native `borgmatic rlist` action should be preferred for most use.) What if you only want to run Borg on a single configured borgmatic repository -when you've got several configured? Not a problem. +when you've got several configured? Not a problem. The `--repository` argument +lets you specify the repository to use, either by its path or its label: ```bash borgmatic borg --repository repo.borg break-lock diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 52962c3..917eb43 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -90,7 +90,7 @@ installing borgmatic: * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/) * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/) * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic) - * [OpenBSD](http://ports.su/sysutils/borgmatic) + * [OpenBSD](https://openports.pl/path/sysutils/borgmatic) * [openSUSE](https://software.opensuse.org/package/borgmatic) * [macOS (via Homebrew)](https://formulae.brew.sh/formula/borgmatic) * [macOS (via MacPorts)](https://ports.macports.org/port/borgmatic/) @@ -157,7 +157,7 @@ variable or set the `BORG_PASSPHRASE` environment variable. See the section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption) of the Borg Quick Start for more info. -Alternatively, you can specify the passphrase programatically by setting +Alternatively, you can specify the passphrase programmatically by setting either the borgmatic `encryption_passcommand` configuration variable or the `BORG_PASSCOMMAND` environment variable. See the [Borg Security FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically) @@ -180,6 +180,9 @@ following command is available for that: sudo validate-borgmatic-config ``` +You'll need to specify your configuration file with `--config` if it's not in +a default location. + This command's exit status (`$?` in Bash) is zero when configuration is valid and non-zero otherwise. diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md index 296b3f8..69b5f5b 100644 --- a/docs/how-to/upgrade.md +++ b/docs/how-to/upgrade.md @@ -145,15 +145,18 @@ like this: ```yaml location: repositories: - - original.borg + - path: original.borg ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + Change it to a new (not yet created) repository path: ```yaml location: repositories: - - upgraded.borg + - path: upgraded.borg ``` Then, run the `rcreate` action (formerly `init`) to create that new Borg 2 diff --git a/docs/reference/command-line.md b/docs/reference/command-line.md index 8997cd9..9cfdd7e 100644 --- a/docs/reference/command-line.md +++ b/docs/reference/command-line.md @@ -7,8 +7,10 @@ eleventyNavigation: --- ## borgmatic options -Here are all of the available borgmatic command-line options. This includes the separate options for -each action sub-command: +Here are all of the available borgmatic command-line options, including the +separate options for each action sub-command. Note that most of the +flags listed here do not have equivalents in borgmatic's [configuration +file](https://torsion.org/borgmatic/docs/reference/configuration/). ``` {% include borgmatic/command-line.txt %} diff --git a/scripts/run-end-to-end-dev-tests b/scripts/run-end-to-end-dev-tests new file mode 100755 index 0000000..032967f --- /dev/null +++ b/scripts/run-end-to-end-dev-tests @@ -0,0 +1,20 @@ +#!/bin/sh + +# This script is for running end-to-end tests on a developer machine. It sets up database containers +# to run tests against, runs the tests, and then tears down the containers. +# +# Run this script from the root directory of the borgmatic source. +# +# For more information, see: +# https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/ + +set -e + +USER_PODMAN_SOCKET_PATH=/run/user/$UID/podman/podman.sock + +if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then + export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH" +fi + +docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \ + --renew-anon-volumes --abort-on-container-exit diff --git a/scripts/run-full-dev-tests b/scripts/run-full-dev-tests deleted file mode 100755 index 3ee37ac..0000000 --- a/scripts/run-full-dev-tests +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -# This script is for running all tests, including end-to-end tests, on a developer machine. It sets -# up database containers to run tests against, runs the tests, and then tears down the containers. -# -# Run this script from the root directory of the borgmatic source. -# -# For more information, see: -# https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/ - -set -e - -docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \ - --renew-anon-volumes --abort-on-container-exit diff --git a/scripts/run-full-tests b/scripts/run-full-tests index 58e5ac6..bf26c21 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -3,13 +3,20 @@ # This script installs test dependencies and runs all tests, including end-to-end tests. It # is designed to run inside a test container, and presumes that other test infrastructure like # databases are already running. Therefore, on a developer machine, you should not run this script -# directly. Instead, run scripts/run-full-dev-tests +# directly. Instead, run scripts/run-end-to-end-dev-tests # # For more information, see: # https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/ set -e +if [ -z "$TEST_CONTAINER" ] ; then + echo "This script is designed to work inside a test container and is not intended to" + echo "be run manually. If you're trying to run borgmatic's end-to-end tests, execute" + echo "scripts/run-end-to-end-dev-tests instead." + exit 1 +fi + apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \ py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite # If certain dependencies of black are available in this version of Alpine, install them. @@ -17,5 +24,9 @@ apk add --no-cache py3-typed-ast py3-regex || true python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1 pip3 install --ignore-installed tox==3.25.1 export COVERAGE_FILE=/tmp/.coverage -tox --workdir /tmp/.tox --sitepackages + +if [ "$1" != "--end-to-end-only" ] ; then + tox --workdir /tmp/.tox --sitepackages +fi + tox --workdir /tmp/.tox --sitepackages -e end-to-end diff --git a/setup.cfg b/setup.cfg index d8d28f8..a5ba3a6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,19 +4,23 @@ description_file=README.md [tool:pytest] testpaths = tests addopts = --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end -filterwarnings = - ignore:Coverage disabled.*:pytest.PytestWarning [flake8] -ignore = E501,W503 +max-line-length = 100 +extend-ignore = E203,E501,W503 exclude = *.*/* multiline-quotes = ''' docstring-quotes = ''' [tool:isort] -force_single_line = False -include_trailing_comma = True +profile=black known_first_party = borgmatic line_length = 100 -multi_line_output = 3 skip = .tox + +[codespell] +skip = .git,.tox,build + +[pycodestyle] +ignore = E203 +max_line_length = 100 diff --git a/setup.py b/setup.py index 5ea3c2e..ce6b78e 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.10.dev0' +VERSION = '1.7.13.dev0' setup( @@ -32,6 +32,7 @@ setup( install_requires=( 'colorama>=0.4.1,<0.5', 'jsonschema', + 'packaging', 'requests', 'ruamel.yaml>0.15.0,<0.18.0', 'setuptools', diff --git a/test_requirements.txt b/test_requirements.txt index 9cae8fb..6516a50 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,24 +1,33 @@ appdirs==1.4.4; python_version >= '3.8' -attrs==20.3.0; python_version >= '3.8' -black==19.10b0; python_version >= '3.8' -click==7.1.2; python_version >= '3.8' -colorama==0.4.4 -coverage==5.3 -flake8==4.0.1 +attrs==22.2.0; python_version >= '3.8' +black==23.3.0; python_version >= '3.8' +chardet==5.1.0 +click==8.1.3; python_version >= '3.8' +codespell==2.2.4 +colorama==0.4.6 +coverage==7.2.3 +flake8==6.0.0 flake8-quotes==3.3.2 -flexmock==0.10.4 -isort==5.9.1 -mccabe==0.6.1 -pluggy==0.13.1 -pathspec==0.8.1; python_version >= '3.8' -py==1.10.0 -pycodestyle==2.8.0 -pyflakes==2.4.0 -jsonschema==3.2.0 -pytest==7.2.0 +flake8-use-fstring==1.4 +flake8-variables-names==0.0.5 +flexmock==0.11.3 +idna==3.4 +importlib_metadata==6.3.0; python_version < '3.8' +isort==5.12.0 +mccabe==0.7.0 +packaging==23.1 +pluggy==1.0.0 +pathspec==0.11.1; python_version >= '3.8' +py==1.11.0 +pycodestyle==2.10.0 +pyflakes==3.0.1 +jsonschema==4.17.3 +pytest==7.3.0 pytest-cov==4.0.0 regex; python_version >= '3.8' -requests==2.25.0 +requests==2.28.2 ruamel.yaml>0.15.0,<0.18.0 toml==0.10.2; python_version >= '3.8' typed-ast; python_version >= '3.8' +typing-extensions==4.5.0; python_version < '3.8' +zipp==3.15.0; python_version < '3.8' diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 094ac8d..0bbec8c 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -1,30 +1,34 @@ version: '3' services: postgresql: - image: postgres:13.1-alpine + image: docker.io/postgres:13.1-alpine environment: POSTGRES_PASSWORD: test POSTGRES_DB: test mysql: - image: mariadb:10.5 + image: docker.io/mariadb:10.5 environment: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test mongodb: - image: mongo:5.0.5 + image: docker.io/mongo:5.0.5 environment: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: test tests: - image: alpine:3.13 + image: docker.io/alpine:3.13 + environment: + TEST_CONTAINER: true volumes: - "../..:/app:ro" tmpfs: - "/app/borgmatic.egg-info" + - "/app/build" tty: true working_dir: /app - command: - - /app/scripts/run-full-tests + entrypoint: /app/scripts/run-full-tests + command: --end-to-end-only depends_on: - postgresql - mysql + - mongodb diff --git a/tests/end-to-end/test_borgmatic.py b/tests/end-to-end/test_borgmatic.py index c2d1029..5e915f0 100644 --- a/tests/end-to-end/test_borgmatic.py +++ b/tests/end-to-end/test_borgmatic.py @@ -12,17 +12,14 @@ def generate_configuration(config_path, repository_path): to work for testing (including injecting the given repository path and tacking on an encryption passphrase). ''' - subprocess.check_call( - 'generate-borgmatic-config --destination {}'.format(config_path).split(' ') - ) + subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) config = ( open(config_path) .read() .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) - .replace('- ssh://user@backupserver/./{fqdn}', '') - .replace('- /var/local/backups/local.borg', '') - .replace('- /home/user/path with spaces', '') - .replace('- /home', '- {}'.format(config_path)) + .replace('- path: /mnt/backup', '') + .replace('label: local', '') + .replace('- /home', f'- {config_path}') .replace('- /etc', '') .replace('- /var/log/syslog*', '') + 'storage:\n encryption_passphrase: "test"' @@ -47,13 +44,13 @@ def test_borgmatic_command(): generate_configuration(config_path, repository_path) subprocess.check_call( - 'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ') + f'borgmatic -v 2 --config {config_path} init --encryption repokey'.split(' ') ) # Run borgmatic to generate a backup archive, and then list it to make sure it exists. - subprocess.check_call('borgmatic --config {}'.format(config_path).split(' ')) + subprocess.check_call(f'borgmatic --config {config_path}'.split(' ')) output = subprocess.check_output( - 'borgmatic --config {} list --json'.format(config_path).split(' ') + f'borgmatic --config {config_path} list --json'.split(' ') ).decode(sys.stdout.encoding) parsed_output = json.loads(output) @@ -64,16 +61,14 @@ def test_borgmatic_command(): # Extract the created archive into the current (temporary) directory, and confirm that the # extracted file looks right. output = subprocess.check_output( - 'borgmatic --config {} extract --archive {}'.format(config_path, archive_name).split( - ' ' - ) + f'borgmatic --config {config_path} extract --archive {archive_name}'.split(' '), ).decode(sys.stdout.encoding) extracted_config_path = os.path.join(extract_path, config_path) assert open(extracted_config_path).read() == open(config_path).read() # Exercise the info action. output = subprocess.check_output( - 'borgmatic --config {} info --json'.format(config_path).split(' ') + f'borgmatic --config {config_path} info --json'.split(' '), ).decode(sys.stdout.encoding) parsed_output = json.loads(output) diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 8849b3c..30aea4a 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -189,7 +189,7 @@ def test_database_dump_with_error_causes_borgmatic_to_exit(): '-v', '2', '--override', - "hooks.postgresql_databases=[{'name': 'nope'}]", + "hooks.postgresql_databases=[{'name': 'nope'}]", # noqa: FS003 ] ) finally: diff --git a/tests/end-to-end/test_override.py b/tests/end-to-end/test_override.py index 0a42018..e86186d 100644 --- a/tests/end-to-end/test_override.py +++ b/tests/end-to-end/test_override.py @@ -10,17 +10,15 @@ def generate_configuration(config_path, repository_path): to work for testing (including injecting the given repository path and tacking on an encryption passphrase). ''' - subprocess.check_call( - 'generate-borgmatic-config --destination {}'.format(config_path).split(' ') - ) + subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) config = ( open(config_path) .read() .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) - .replace('- ssh://user@backupserver/./{fqdn}', '') + .replace('- ssh://user@backupserver/./{fqdn}', '') # noqa: FS003 .replace('- /var/local/backups/local.borg', '') .replace('- /home/user/path with spaces', '') - .replace('- /home', '- {}'.format(config_path)) + .replace('- /home', f'- {config_path}') .replace('- /etc', '') .replace('- /var/log/syslog*', '') + 'storage:\n encryption_passphrase: "test"' diff --git a/tests/end-to-end/test_validate_config.py b/tests/end-to-end/test_validate_config.py index 5de83a3..5446503 100644 --- a/tests/end-to-end/test_validate_config.py +++ b/tests/end-to-end/test_validate_config.py @@ -1,5 +1,6 @@ import os import subprocess +import sys import tempfile @@ -7,12 +8,8 @@ def test_validate_config_command_with_valid_configuration_succeeds(): with tempfile.TemporaryDirectory() as temporary_directory: config_path = os.path.join(temporary_directory, 'test.yaml') - subprocess.check_call( - 'generate-borgmatic-config --destination {}'.format(config_path).split(' ') - ) - exit_code = subprocess.call( - 'validate-borgmatic-config --config {}'.format(config_path).split(' ') - ) + subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' ')) assert exit_code == 0 @@ -21,16 +18,25 @@ def test_validate_config_command_with_invalid_configuration_fails(): with tempfile.TemporaryDirectory() as temporary_directory: config_path = os.path.join(temporary_directory, 'test.yaml') - subprocess.check_call( - 'generate-borgmatic-config --destination {}'.format(config_path).split(' ') - ) + subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) config = open(config_path).read().replace('keep_daily: 7', 'keep_daily: "7"') config_file = open(config_path, 'w') config_file.write(config) config_file.close() - exit_code = subprocess.call( - 'validate-borgmatic-config --config {}'.format(config_path).split(' ') - ) + exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' ')) assert exit_code == 1 + + +def test_validate_config_command_with_show_flag_displays_configuration(): + with tempfile.TemporaryDirectory() as temporary_directory: + config_path = os.path.join(temporary_directory, 'test.yaml') + + subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + output = subprocess.check_output( + f'validate-borgmatic-config --config {config_path} --show'.split(' ') + ).decode(sys.stdout.encoding) + + assert 'location:' in output + assert 'repositories:' in output diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py new file mode 100644 index 0000000..1afb0e0 --- /dev/null +++ b/tests/integration/borg/test_commands.py @@ -0,0 +1,108 @@ +import copy + +from flexmock import flexmock + +import borgmatic.borg.info +import borgmatic.borg.list +import borgmatic.borg.rlist +import borgmatic.borg.transfer +import borgmatic.commands.arguments + + +def assert_command_does_not_duplicate_flags(command, *args, **kwargs): + ''' + Assert that the given Borg command sequence does not contain any duplicated flags, e.g. + "--match-archives" twice anywhere in the command. + ''' + flag_counts = {} + + for flag_name in command: + if not flag_name.startswith('--'): + continue + + if flag_name in flag_counts: + flag_counts[flag_name] += 1 + else: + flag_counts[flag_name] = 1 + + assert flag_counts == { + flag_name: 1 for flag_name in flag_counts + }, f"Duplicate flags found in: {' '.join(command)}" + + +def fuzz_argument(arguments, argument_name): + ''' + Given an argparse.Namespace instance of arguments and an argument name in it, copy the arguments + namespace and set the argument name in the copy with a fake value. Return the copied arguments. + + This is useful for "fuzzing" a unit under test by passing it each possible argument in turn, + making sure it doesn't blow up or duplicate Borg arguments. + ''' + arguments_copy = copy.copy(arguments) + value = getattr(arguments_copy, argument_name) + setattr(arguments_copy, argument_name, not value if isinstance(value, bool) else 'value') + + return arguments_copy + + +def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments( + 'transfer', '--source-repository', 'foo' + )['transfer'] + flexmock(borgmatic.borg.transfer).should_receive('execute_command').replace_with( + assert_command_does_not_duplicate_flags + ) + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.transfer.transfer_archives( + False, 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) + + +def test_make_list_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('list')['list'] + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + command = borgmatic.borg.list.make_list_command( + 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) + + assert_command_does_not_duplicate_flags(command) + + +def test_make_rlist_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('rlist')['rlist'] + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + command = borgmatic.borg.rlist.make_rlist_command( + 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) + + assert_command_does_not_duplicate_flags(command) + + +def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('info')['info'] + flexmock(borgmatic.borg.info).should_receive('execute_command_and_capture_output').replace_with( + assert_command_does_not_duplicate_flags + ) + flexmock(borgmatic.borg.info).should_receive('execute_command').replace_with( + assert_command_does_not_duplicate_flags + ) + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.info.display_archives_info( + 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 754564b..1fc8f8c 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -465,6 +465,20 @@ def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives ) +def test_parse_arguments_disallows_list_with_both_prefix_and_match_archives(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('list', '--prefix', 'foo', '--match-archives', 'sh:*bar') + + +def test_parse_arguments_disallows_rlist_with_both_prefix_and_match_archives(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('rlist', '--prefix', 'foo', '--match-archives', 'sh:*bar') + + def test_parse_arguments_disallows_info_with_both_archive_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff --git a/tests/integration/commands/test_validate_config.py b/tests/integration/commands/test_validate_config.py index cb0144d..78887e7 100644 --- a/tests/integration/commands/test_validate_config.py +++ b/tests/integration/commands/test_validate_config.py @@ -18,3 +18,12 @@ def test_parse_arguments_with_multiple_config_paths_parses_as_list(): parser = module.parse_arguments('--config', 'myconfig', 'otherconfig') assert parser.config_paths == ['myconfig', 'otherconfig'] + + +def test_parse_arguments_supports_show_flag(): + config_paths = ['default'] + flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) + + parser = module.parse_arguments('--config', 'myconfig', '--show') + + assert parser.show diff --git a/tests/integration/config/test_legacy.py b/tests/integration/config/test_legacy.py index 870da88..c73e7ee 100644 --- a/tests/integration/config/test_legacy.py +++ b/tests/integration/config/test_legacy.py @@ -7,7 +7,7 @@ from borgmatic.config import legacy as module def test_parse_section_options_with_punctuation_should_return_section_options(): parser = module.RawConfigParser() - parser.read_file(StringIO('[section]\nfoo: {}\n'.format(string.punctuation))) + parser.read_file(StringIO(f'[section]\nfoo: {string.punctuation}\n')) section_format = module.Section_format( 'section', (module.Config_option('foo', str, required=True),) diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py index e1ecc8a..028a652 100644 --- a/tests/integration/config/test_load.py +++ b/tests/integration/config/test_load.py @@ -2,7 +2,6 @@ import io import sys import pytest -import ruamel.yaml from flexmock import flexmock from borgmatic.config import load as module @@ -10,11 +9,41 @@ from borgmatic.config import load as module def test_load_configuration_parses_contents(): builtins = flexmock(sys.modules['builtins']) - builtins.should_receive('open').with_args('config.yaml').and_return('key: value') - + config_file = io.StringIO('key: value') + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) assert module.load_configuration('config.yaml') == {'key': 'value'} +def test_load_configuration_replaces_constants(): + builtins = flexmock(sys.modules['builtins']) + config_file = io.StringIO( + ''' + constants: + key: value + key: {key} + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + assert module.load_configuration('config.yaml') == {'key': 'value'} + + +def test_load_configuration_replaces_complex_constants(): + builtins = flexmock(sys.modules['builtins']) + config_file = io.StringIO( + ''' + constants: + key: + subkey: value + key: {key} + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + assert module.load_configuration('config.yaml') == {'key': {'subkey': 'value'}} + + def test_load_configuration_inlines_include_relative_to_current_directory(): builtins = flexmock(sys.modules['builtins']) flexmock(module.os).should_receive('getcwd').and_return('/tmp') @@ -120,6 +149,248 @@ def test_load_configuration_merges_include(): assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'} +def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + foo: bar + baz: quux + + other: + a: b + c: d + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: !retain + foo: override + + other: + a: override + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + assert module.load_configuration('config.yaml') == { + 'stuff': {'foo': 'override'}, + 'other': {'a': 'override', 'c': 'd'}, + } + + +def test_load_configuration_with_retain_tag_but_without_merge_include_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: !retain + foo: bar + baz: quux + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + foo: override + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + module.load_configuration('config.yaml') + + +def test_load_configuration_with_retain_tag_on_scalar_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + foo: bar + baz: quux + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + foo: !retain override + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + module.load_configuration('config.yaml') + + +def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_values(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - x + - !omit b + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + assert module.load_configuration('config.yaml') == {'stuff': ['a', 'c', 'x', 'y']} + + +def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_does_not_raise(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - x + - !omit q + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + assert module.load_configuration('config.yaml') == {'stuff': ['a', 'b', 'c', 'x', 'y']} + + +def test_load_configuration_with_omit_tag_on_non_list_item_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: !omit + - x + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + module.load_configuration('config.yaml') + + +def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - foo: bar + baz: quux + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - !omit foo: bar + baz: quux + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + module.load_configuration('config.yaml') + + +def test_load_configuration_with_omit_tag_but_without_merge_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - !omit b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - x + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + module.load_configuration('config.yaml') + + def test_load_configuration_does_not_merge_include_list(): builtins = flexmock(sys.modules['builtins']) flexmock(module.os).should_receive('getcwd').and_return('/tmp') @@ -143,42 +414,79 @@ def test_load_configuration_does_not_merge_include_list(): config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) - with pytest.raises(ruamel.yaml.error.YAMLError): + with pytest.raises(module.ruamel.yaml.error.YAMLError): assert module.load_configuration('config.yaml') +@pytest.mark.parametrize( + 'node_class', + ( + module.ruamel.yaml.nodes.MappingNode, + module.ruamel.yaml.nodes.SequenceNode, + module.ruamel.yaml.nodes.ScalarNode, + ), +) +def test_raise_retain_node_error_raises(node_class): + with pytest.raises(ValueError): + module.raise_retain_node_error( + loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock()) + ) + + +def test_raise_omit_node_error_raises(): + with pytest.raises(ValueError): + module.raise_omit_node_error(loader=flexmock(), node=flexmock()) + + +def test_filter_omitted_nodes(): + nodes = [ + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'), + module.ruamel.yaml.nodes.ScalarNode(tag='!omit', value='b'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'), + ] + + result = module.filter_omitted_nodes(nodes) + + assert [item.value for item in result] == ['a', 'c', 'a', 'c'] + + def test_deep_merge_nodes_replaces_colliding_scalar_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_hourly' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='24' + ), ), ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_daily' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), ), ], ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_daily' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), ), ], ), @@ -200,35 +508,39 @@ def test_deep_merge_nodes_replaces_colliding_scalar_values(): def test_deep_merge_nodes_keeps_non_colliding_scalar_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_hourly' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='24' + ), ), ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_daily' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), ), ], ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_minutely' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='10'), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='10' + ), ), ], ), @@ -252,28 +564,28 @@ def test_deep_merge_nodes_keeps_non_colliding_scalar_values(): def test_deep_merge_nodes_keeps_deeply_nested_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='lock_wait' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), ), ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='extra_borg_options' ), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='init' ), - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='--init-option' ), ), @@ -284,22 +596,22 @@ def test_deep_merge_nodes_keeps_deeply_nested_values(): ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='extra_borg_options' ), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='prune' ), - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='--prune-option' ), ), @@ -331,32 +643,48 @@ def test_deep_merge_nodes_keeps_deeply_nested_values(): def test_deep_merge_nodes_appends_colliding_sequence_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='before_backup' ), - ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:int', value=['echo 1', 'echo 2'] + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 1' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 2' + ), + ], ), ), ], ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='before_backup' ), - ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:int', value=['echo 3', 'echo 4'] + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 3' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 4' + ), + ], ), ), ], @@ -371,4 +699,178 @@ def test_deep_merge_nodes_appends_colliding_sequence_values(): options = section_value.value assert len(options) == 1 assert options[0][0].value == 'before_backup' - assert options[0][1].value == ['echo 1', 'echo 2', 'echo 3', 'echo 4'] + assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 2', 'echo 3', 'echo 4'] + + +def test_deep_merge_nodes_only_keeps_mapping_values_tagged_with_retain(): + node_values = [ + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='keep_hourly' + ), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='24' + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='keep_daily' + ), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), + ), + ], + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( + tag='!retain', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='keep_daily' + ), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), + ), + ], + ), + ), + ] + + result = module.deep_merge_nodes(node_values) + assert len(result) == 1 + (section_key, section_value) = result[0] + assert section_key.value == 'retention' + assert section_value.tag == 'tag:yaml.org,2002:map' + options = section_value.value + assert len(options) == 1 + assert options[0][0].value == 'keep_daily' + assert options[0][1].value == '5' + + +def test_deep_merge_nodes_only_keeps_sequence_values_tagged_with_retain(): + node_values = [ + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 1' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 2' + ), + ], + ), + ), + ], + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='!retain', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 3' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 4' + ), + ], + ), + ), + ], + ), + ), + ] + + result = module.deep_merge_nodes(node_values) + assert len(result) == 1 + (section_key, section_value) = result[0] + assert section_key.value == 'hooks' + options = section_value.value + assert len(options) == 1 + assert options[0][0].value == 'before_backup' + assert options[0][1].tag == 'tag:yaml.org,2002:seq' + assert [item.value for item in options[0][1].value] == ['echo 3', 'echo 4'] + + +def test_deep_merge_nodes_skips_sequence_values_tagged_with_omit(): + node_values = [ + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 1' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 2' + ), + ], + ), + ), + ], + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode(tag='!omit', value='echo 2'), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 3' + ), + ], + ), + ), + ], + ), + ), + ] + + result = module.deep_merge_nodes(node_values) + assert len(result) == 1 + (section_key, section_value) = result[0] + assert section_key.value == 'hooks' + options = section_value.value + assert len(options) == 1 + assert options[0][0].value == 'before_backup' + assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 3'] diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 5d948ae..87428dd 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -8,7 +8,7 @@ from flexmock import flexmock from borgmatic.config import validate as module -def test_schema_filename_returns_plausable_path(): +def test_schema_filename_returns_plausible_path(): schema_path = module.schema_filename() assert schema_path.endswith('/schema.yaml') @@ -63,7 +63,10 @@ def test_parse_configuration_transforms_file_into_mapping(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']}, + 'location': { + 'source_directories': ['/home', '/etc'], + 'repositories': [{'path': 'hostname.borg'}], + }, 'retention': {'keep_daily': 7, 'keep_hourly': 24, 'keep_minutely': 60}, 'consistency': {'checks': [{'name': 'repository'}, {'name': 'archives'}]}, } @@ -89,7 +92,7 @@ def test_parse_configuration_passes_through_quoted_punctuation(): assert config == { 'location': { 'source_directories': [f'/home/{string.punctuation}'], - 'repositories': ['test.borg'], + 'repositories': [{'path': 'test.borg'}], } } assert logs == [] @@ -151,7 +154,7 @@ def test_parse_configuration_inlines_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']}, + 'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]}, 'retention': {'keep_daily': 7, 'keep_hourly': 24}, } assert logs == [] @@ -185,7 +188,7 @@ def test_parse_configuration_merges_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']}, + 'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]}, 'retention': {'keep_daily': 1, 'keep_hourly': 24}, } assert logs == [] @@ -247,7 +250,7 @@ def test_parse_configuration_applies_overrides(): assert config == { 'location': { 'source_directories': ['/home'], - 'repositories': ['hostname.borg'], + 'repositories': [{'path': 'hostname.borg'}], 'local_path': 'borg2', } } @@ -273,7 +276,7 @@ def test_parse_configuration_applies_normalization(): assert config == { 'location': { 'source_directories': ['/home'], - 'repositories': ['hostname.borg'], + 'repositories': [{'path': 'hostname.borg'}], 'exclude_if_present': ['.nobackup'], } } diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index bbdb977..9c62941 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -138,16 +138,16 @@ def test_log_outputs_kills_other_processes_when_one_errors(): process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) flexmock(module).should_receive('exit_code_indicates_error').with_args( - process, None, 'borg' + ['grep'], None, 'borg' ).and_return(False) flexmock(module).should_receive('exit_code_indicates_error').with_args( - process, 2, 'borg' + ['grep'], 2, 'borg' ).and_return(True) other_process = subprocess.Popen( ['sleep', '2'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) flexmock(module).should_receive('exit_code_indicates_error').with_args( - other_process, None, 'borg' + ['sleep', '2'], None, 'borg' ).and_return(False) flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return( process.stdout @@ -239,21 +239,20 @@ def test_log_outputs_does_not_error_when_one_process_exits(): def test_log_outputs_truncates_long_error_output(): - flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0 flexmock(module.logger).should_receive('log') flexmock(module).should_receive('command_for_process').and_return('grep') process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) flexmock(module).should_receive('exit_code_indicates_error').with_args( - process, None, 'borg' + ['grep'], None, 'borg' ).and_return(False) flexmock(module).should_receive('exit_code_indicates_error').with_args( - process, 2, 'borg' + ['grep'], 2, 'borg' ).and_return(True) flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout) with pytest.raises(subprocess.CalledProcessError) as error: - module.log_outputs( + flexmock(module, ERROR_OUTPUT_MAX_LINE_COUNT=0).log_outputs( (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg' ) diff --git a/tests/unit/actions/test_borg.py b/tests/unit/actions/test_borg.py index 7b22c7b..f597acb 100644 --- a/tests/unit/actions/test_borg.py +++ b/tests/unit/actions/test_borg.py @@ -13,7 +13,7 @@ def test_run_borg_does_not_raise(): borg_arguments = flexmock(repository=flexmock(), archive=flexmock(), options=flexmock()) module.run_borg( - repository='repo', + repository={'path': 'repos'}, storage={}, local_borg_version=None, borg_arguments=borg_arguments, diff --git a/tests/unit/actions/test_break_lock.py b/tests/unit/actions/test_break_lock.py index c7db00b..6dc2470 100644 --- a/tests/unit/actions/test_break_lock.py +++ b/tests/unit/actions/test_break_lock.py @@ -10,7 +10,7 @@ def test_run_break_lock_does_not_raise(): break_lock_arguments = flexmock(repository=flexmock()) module.run_break_lock( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, break_lock_arguments=break_lock_arguments, diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 3e1a9c2..05f63b6 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -12,13 +12,17 @@ def test_run_check_calls_hooks_for_configured_repository(): flexmock(module.borgmatic.borg.check).should_receive('check_archives').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( - repository=None, progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock(), + repository=None, + progress=flexmock(), + repair=flexmock(), + only=flexmock(), + force=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_check( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, consistency={}, @@ -49,7 +53,7 @@ def test_run_check_runs_with_selected_repository(): module.run_check( config_filename='test.yaml', - repository=flexmock(), + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, consistency={}, @@ -80,7 +84,7 @@ def test_run_check_bails_if_repository_does_not_match(): module.run_check( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, consistency={}, diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index 4dae903..fbd4f90 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -16,7 +16,7 @@ def test_compact_actions_calls_hooks_for_configured_repository(): module.run_compact( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, @@ -44,7 +44,7 @@ def test_compact_runs_with_selected_repository(): module.run_compact( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, @@ -72,7 +72,7 @@ def test_compact_bails_if_repository_does_not_match(): module.run_compact( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 8a9d0b4..2b72408 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -24,7 +24,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): list( module.run_create( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={}, storage={}, hooks={}, @@ -57,7 +57,7 @@ def test_run_create_runs_with_selected_repository(): list( module.run_create( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={}, storage={}, hooks={}, diff --git a/tests/unit/actions/test_export_tar.py b/tests/unit/actions/test_export_tar.py index 41b680a..6741d42 100644 --- a/tests/unit/actions/test_export_tar.py +++ b/tests/unit/actions/test_export_tar.py @@ -19,7 +19,7 @@ def test_run_export_tar_does_not_raise(): global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_export_tar( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, export_tar_arguments=export_tar_arguments, diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py index 4222b8a..32b93b4 100644 --- a/tests/unit/actions/test_extract.py +++ b/tests/unit/actions/test_extract.py @@ -20,7 +20,7 @@ def test_run_extract_calls_hooks(): module.run_extract( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, hooks={}, diff --git a/tests/unit/actions/test_info.py b/tests/unit/actions/test_info.py index 8cde178..a4f1d54 100644 --- a/tests/unit/actions/test_info.py +++ b/tests/unit/actions/test_info.py @@ -14,7 +14,7 @@ def test_run_info_does_not_raise(): list( module.run_info( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, info_arguments=info_arguments, diff --git a/tests/unit/actions/test_list.py b/tests/unit/actions/test_list.py index f4a603d..bfdfd01 100644 --- a/tests/unit/actions/test_list.py +++ b/tests/unit/actions/test_list.py @@ -14,7 +14,7 @@ def test_run_list_does_not_raise(): list( module.run_list( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, list_arguments=list_arguments, diff --git a/tests/unit/actions/test_mount.py b/tests/unit/actions/test_mount.py index 0624a89..7eadfca 100644 --- a/tests/unit/actions/test_mount.py +++ b/tests/unit/actions/test_mount.py @@ -17,7 +17,7 @@ def test_run_mount_does_not_raise(): ) module.run_mount( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, mount_arguments=mount_arguments, diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index db9c124..7af7ea7 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -13,7 +13,7 @@ def test_run_prune_calls_hooks_for_configured_repository(): module.run_prune( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, @@ -38,7 +38,7 @@ def test_run_prune_runs_with_selected_repository(): module.run_prune( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, diff --git a/tests/unit/actions/test_rcreate.py b/tests/unit/actions/test_rcreate.py index 78d4af1..b77fa75 100644 --- a/tests/unit/actions/test_rcreate.py +++ b/tests/unit/actions/test_rcreate.py @@ -18,7 +18,7 @@ def test_run_rcreate_does_not_raise(): ) module.run_rcreate( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rcreate_arguments=arguments, @@ -45,7 +45,7 @@ def test_run_rcreate_bails_if_repository_does_not_match(): ) module.run_rcreate( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rcreate_arguments=arguments, diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index eaad6bf..16fe292 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -67,7 +67,7 @@ def test_collect_archive_database_names_parses_archive_paths(): ) archive_database_names = module.collect_archive_database_names( - repository='repo', + repository={'path': 'repo'}, archive='archive', location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), @@ -92,7 +92,7 @@ def test_collect_archive_database_names_parses_directory_format_archive_paths(): ) archive_database_names = module.collect_archive_database_names( - repository='repo', + repository={'path': 'repo'}, archive='archive', location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), @@ -113,7 +113,7 @@ def test_collect_archive_database_names_skips_bad_archive_paths(): ) archive_database_names = module.collect_archive_database_names( - repository='repo', + repository={'path': 'repo'}, archive='archive', location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), @@ -148,7 +148,8 @@ def test_find_databases_to_restore_without_requested_names_finds_all_archive_dat archive_database_names = {'postresql_databases': ['foo', 'bar']} restore_names = module.find_databases_to_restore( - requested_database_names=[], archive_database_names=archive_database_names, + requested_database_names=[], + archive_database_names=archive_database_names, ) assert restore_names == archive_database_names @@ -158,7 +159,8 @@ def test_find_databases_to_restore_with_all_in_requested_names_finds_all_archive archive_database_names = {'postresql_databases': ['foo', 'bar']} restore_names = module.find_databases_to_restore( - requested_database_names=['all'], archive_database_names=archive_database_names, + requested_database_names=['all'], + archive_database_names=archive_database_names, ) assert restore_names == archive_database_names @@ -194,7 +196,9 @@ def test_ensure_databases_found_with_all_databases_found_does_not_raise(): def test_ensure_databases_found_with_no_databases_raises(): with pytest.raises(ValueError): module.ensure_databases_found( - restore_names={'postgresql_databases': []}, remaining_restore_names={}, found_names=[], + restore_names={'postgresql_databases': []}, + remaining_restore_names={}, + found_names=[], ) @@ -233,7 +237,7 @@ def test_run_restore_restores_each_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -246,17 +250,19 @@ def test_run_restore_restores_each_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).once() flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), @@ -273,7 +279,7 @@ def test_run_restore_bails_for_non_matching_repository(): flexmock(module).should_receive('restore_single_database').never() module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), @@ -327,7 +333,7 @@ def test_run_restore_restores_database_configured_with_all_name(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -340,17 +346,19 @@ def test_run_restore_restores_database_configured_with_all_name(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).once() flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), @@ -399,7 +407,7 @@ def test_run_restore_skips_missing_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -412,17 +420,19 @@ def test_run_restore_skips_missing_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).never() flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), @@ -465,7 +475,7 @@ def test_run_restore_restores_databases_from_different_hooks(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -478,17 +488,19 @@ def test_run_restore_restores_databases_from_different_hooks(): remote_path=object, archive_name=object, hook_name='mysql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).once() flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), diff --git a/tests/unit/actions/test_rinfo.py b/tests/unit/actions/test_rinfo.py index d789ef1..133e61a 100644 --- a/tests/unit/actions/test_rinfo.py +++ b/tests/unit/actions/test_rinfo.py @@ -11,7 +11,7 @@ def test_run_rinfo_does_not_raise(): list( module.run_rinfo( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rinfo_arguments=rinfo_arguments, diff --git a/tests/unit/actions/test_rlist.py b/tests/unit/actions/test_rlist.py index 3da0f90..7f8b58a 100644 --- a/tests/unit/actions/test_rlist.py +++ b/tests/unit/actions/test_rlist.py @@ -11,7 +11,7 @@ def test_run_rlist_does_not_raise(): list( module.run_rlist( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rlist_arguments=rlist_arguments, diff --git a/tests/unit/actions/test_transfer.py b/tests/unit/actions/test_transfer.py index cc9f138..58d8a16 100644 --- a/tests/unit/actions/test_transfer.py +++ b/tests/unit/actions/test_transfer.py @@ -10,7 +10,7 @@ def test_run_transfer_does_not_raise(): global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_transfer( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, transfer_arguments=transfer_arguments, diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 6eae5fb..5b73596 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -21,7 +21,10 @@ def test_run_arbitrary_borg_calls_borg_with_parameters(): ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['break-lock'], ) @@ -40,7 +43,10 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['break-lock'], ) @@ -59,7 +65,10 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['break-lock'], ) @@ -80,7 +89,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters( ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', options=['break-lock'], @@ -103,7 +112,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], @@ -125,7 +134,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], @@ -149,7 +158,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], @@ -171,7 +180,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['list', '--progress'], @@ -192,7 +201,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['--', 'break-lock'], @@ -213,7 +222,10 @@ def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise(): ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=[], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=[], ) @@ -231,7 +243,10 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository(): ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['key', 'export'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['key', 'export'], ) @@ -249,7 +264,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository() ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['debug', 'dump-manifest', 'path'], @@ -270,7 +285,10 @@ def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repositor ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['debug', 'info'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['debug', 'info'], ) @@ -288,7 +306,7 @@ def test_run_arbitrary_borg_with_debug_convert_profile_command_does_not_pass_bor ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['debug', 'convert-profile', 'in', 'out'], diff --git a/tests/unit/borg/test_break_lock.py b/tests/unit/borg/test_break_lock.py index 0663f93..509fc1b 100644 --- a/tests/unit/borg/test_break_lock.py +++ b/tests/unit/borg/test_break_lock.py @@ -10,7 +10,9 @@ from ..test_verbosity import insert_logging_mock def insert_execute_command_mock(command): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - command, borg_local_path='borg', extra_environment=None, + command, + borg_local_path='borg', + extra_environment=None, ).once() @@ -19,7 +21,9 @@ def test_break_lock_calls_borg_with_required_flags(): insert_execute_command_mock(('borg', 'break-lock', 'repo')) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', ) @@ -28,7 +32,10 @@ def test_break_lock_calls_borg_with_remote_path_flags(): insert_execute_command_mock(('borg', 'break-lock', '--remote-path', 'borg1', 'repo')) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', remote_path='borg1', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + remote_path='borg1', ) @@ -37,7 +44,9 @@ def test_break_lock_calls_borg_with_umask_flags(): insert_execute_command_mock(('borg', 'break-lock', '--umask', '0770', 'repo')) module.break_lock( - repository='repo', storage_config={'umask': '0770'}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={'umask': '0770'}, + local_borg_version='1.2.3', ) @@ -46,7 +55,9 @@ def test_break_lock_calls_borg_with_lock_wait_flags(): insert_execute_command_mock(('borg', 'break-lock', '--lock-wait', '5', 'repo')) module.break_lock( - repository='repo', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={'lock_wait': '5'}, + local_borg_version='1.2.3', ) @@ -56,7 +67,9 @@ def test_break_lock_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', ) @@ -66,5 +79,7 @@ def test_break_lock_with_log_debug_calls_borg_with_debug_flags(): insert_logging_mock(logging.DEBUG) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index ba82225..1f992d3 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -79,7 +79,12 @@ def test_parse_frequency_parses_into_timedeltas(frequency, expected_result): @pytest.mark.parametrize( - 'frequency', ('sometime', 'x days', '3 decades',), + 'frequency', + ( + 'sometime', + 'x days', + '3 decades', + ), ) def test_parse_frequency_raises_on_parse_error(frequency): with pytest.raises(ValueError): @@ -189,150 +194,191 @@ def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_ def test_make_check_flags_with_repository_check_returns_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',)) + flags = module.make_check_flags('1.2.3', {}, ('repository',)) assert flags == ('--repository-only',) def test_make_check_flags_with_archives_check_returns_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',)) + flags = module.make_check_flags('1.2.3', {}, ('archives',)) assert flags == ('--archives-only',) def test_make_check_flags_with_data_check_returns_flag_and_implies_archives(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',)) + flags = module.make_check_flags('1.2.3', {}, ('data',)) - assert flags == ('--archives-only', '--verify-data',) + assert flags == ( + '--archives-only', + '--verify-data', + ) def test_make_check_flags_with_extract_omits_extract_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('extract',)) + flags = module.make_check_flags('1.2.3', {}, ('extract',)) assert flags == () def test_make_check_flags_with_repository_and_data_checks_does_not_return_repository_only(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'data',)) + flags = module.make_check_flags( + '1.2.3', + {}, + ( + 'repository', + 'data', + ), + ) assert flags == ('--verify-data',) -def test_make_check_flags_with_default_checks_and_default_prefix_returns_default_flags(): +def test_make_check_flags_with_default_checks_and_prefix_returns_default_flags(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', ('repository', 'archives'), prefix=module.DEFAULT_PREFIX + '1.2.3', + {}, + ('repository', 'archives'), + prefix='foo', ) - assert flags == ('--match-archives', f'sh:{module.DEFAULT_PREFIX}*') + assert flags == ('--match-archives', 'sh:foo*') -def test_make_check_flags_with_all_checks_and_default_prefix_returns_default_flags(): +def test_make_check_flags_with_all_checks_and_prefix_returns_default_flags(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', ('repository', 'archives', 'extract'), prefix=module.DEFAULT_PREFIX + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', ) - assert flags == ('--match-archives', f'sh:{module.DEFAULT_PREFIX}*') + assert flags == ('--match-archives', 'sh:foo*') -def test_make_check_flags_with_all_checks_and_default_prefix_without_borg_features_returns_glob_archives_flags(): +def test_make_check_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags(): flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', ('repository', 'archives', 'extract'), prefix=module.DEFAULT_PREFIX + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', ) - assert flags == ('--glob-archives', f'{module.DEFAULT_PREFIX}*') + assert flags == ('--glob-archives', 'foo*') def test_make_check_flags_with_archives_check_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('archives',), check_last=3) assert flags == ('--archives-only', '--last', '3') def test_make_check_flags_with_data_check_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('data',), check_last=3) assert flags == ('--archives-only', '--last', '3', '--verify-data') def test_make_check_flags_with_repository_check_and_last_omits_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('repository',), check_last=3) assert flags == ('--repository-only',) def test_make_check_flags_with_default_checks_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'archives'), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), check_last=3) assert flags == ('--last', '3') def test_make_check_flags_with_archives_check_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix='foo-') assert flags == ('--archives-only', '--match-archives', 'sh:foo-*') def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('data',), prefix='foo-') assert flags == ('--archives-only', '--match-archives', 'sh:foo-*', '--verify-data') -def test_make_check_flags_with_archives_check_and_empty_prefix_omits_match_archives_flag(): +def test_make_check_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) - flags = module.make_check_flags('1.2.3', ('archives',), prefix='') + flags = module.make_check_flags( + '1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003 + ) - assert flags == ('--archives-only',) + assert flags == ('--archives-only', '--match-archives', 'sh:bar-*') def test_make_check_flags_with_archives_check_and_none_prefix_omits_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), prefix=None) + flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix=None) assert flags == ('--archives-only',) def test_make_check_flags_with_repository_check_and_prefix_omits_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('repository',), prefix='foo-') assert flags == ('--repository-only',) def test_make_check_flags_with_default_checks_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'archives'), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-') assert flags == ('--match-archives', 'sh:foo-*') @@ -370,7 +416,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -400,7 +446,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -427,7 +473,11 @@ def test_check_archives_calls_borg_with_parameters(checks): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', + {}, + checks, + check_last, + prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) @@ -435,7 +485,7 @@ def test_check_archives_calls_borg_with_parameters(checks): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -455,7 +505,7 @@ def test_check_archives_with_json_error_raises(): with pytest.raises(ValueError): module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -473,7 +523,7 @@ def test_check_archives_with_missing_json_keys_raises(): with pytest.raises(ValueError): module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -497,7 +547,7 @@ def test_check_archives_with_extract_check_calls_extract_only(): insert_execute_command_never() module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -521,7 +571,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -545,7 +595,7 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -563,7 +613,7 @@ def test_check_archives_without_any_checks_bails(): insert_execute_command_never() module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -581,7 +631,11 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', + {}, + checks, + check_last, + prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg1', 'check', 'repo')) @@ -589,7 +643,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -608,7 +662,11 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', + {}, + checks, + check_last, + prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo')) @@ -616,7 +674,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -628,6 +686,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): checks = ('repository',) check_last = flexmock() + storage_config = {'lock_wait': 5} consistency_config = {'check_last': check_last} flexmock(module).should_receive('parse_checks') flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) @@ -635,7 +694,11 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', + storage_config, + checks, + check_last, + None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--lock-wait', '5', 'repo')) @@ -643,9 +706,9 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, - storage_config={'lock_wait': 5}, + storage_config=storage_config, consistency_config=consistency_config, local_borg_version='1.2.3', ) @@ -662,7 +725,7 @@ def test_check_archives_with_retention_prefix(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, prefix + '1.2.3', {}, checks, check_last, prefix ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) @@ -670,7 +733,7 @@ def test_check_archives_with_retention_prefix(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -693,7 +756,7 @@ def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={'extra_borg_options': {'check': '--extra --options'}}, consistency_config=consistency_config, diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index 36760f3..60447db 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -25,7 +25,7 @@ def test_compact_segments_calls_borg_with_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.INFO) module.compact_segments( - dry_run=False, repository='repo', storage_config={}, local_borg_version='1.2.3' + dry_run=False, repository_path='repo', storage_config={}, local_borg_version='1.2.3' ) @@ -35,7 +35,7 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.compact_segments( - repository='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False ) @@ -45,7 +45,7 @@ def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.compact_segments( - repository='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False ) @@ -53,7 +53,7 @@ def test_compact_segments_with_dry_run_skips_borg_call(): flexmock(module).should_receive('execute_command').never() module.compact_segments( - repository='repo', storage_config={}, local_borg_version='1.2.3', dry_run=True + repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=True ) @@ -63,7 +63,7 @@ def test_compact_segments_with_local_path_calls_borg_via_local_path(): module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', local_path='borg1', @@ -76,7 +76,7 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', remote_path='borg1', @@ -89,7 +89,7 @@ def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', progress=True, @@ -102,7 +102,7 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', cleanup_commits=True, @@ -115,7 +115,7 @@ def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', threshold=20, @@ -128,7 +128,10 @@ def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('--umask', '077', 'repo'), logging.INFO) module.compact_segments( - dry_run=False, repository='repo', storage_config=storage_config, local_borg_version='1.2.3' + dry_run=False, + repository_path='repo', + storage_config=storage_config, + local_borg_version='1.2.3', ) @@ -138,7 +141,10 @@ def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) module.compact_segments( - dry_run=False, repository='repo', storage_config=storage_config, local_borg_version='1.2.3' + dry_run=False, + repository_path='repo', + storage_config=storage_config, + local_borg_version='1.2.3', ) @@ -148,7 +154,7 @@ def test_compact_segments_with_extra_borg_options_calls_borg_with_extra_options( module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={'extra_borg_options': {'compact': '--extra --options'}}, local_borg_version='1.2.3', ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 69a3ede..818ce27 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -449,7 +449,7 @@ def test_collect_special_file_paths_excludes_non_special_files(): ) == ('/foo', '/baz') -DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' +DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' # noqa: FS003 REPO_ARCHIVE_WITH_PATHS = (f'repo::{DEFAULT_ARCHIVE_NAME}', 'foo', 'bar') @@ -484,7 +484,7 @@ def test_create_archive_calls_borg_with_parameters(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -527,7 +527,7 @@ def test_create_archive_calls_borg_with_environment(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -572,7 +572,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -617,7 +617,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -660,7 +660,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -700,7 +700,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -744,7 +744,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -784,7 +784,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -827,7 +827,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): module.create_archive( dry_run=True, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -872,7 +872,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete module.create_archive( dry_run=True, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -915,7 +915,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -957,7 +957,7 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -999,7 +999,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1041,7 +1041,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1053,7 +1053,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), + 'feature_available,option_flag', + ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), ) def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_parameters( feature_available, option_flag @@ -1088,7 +1089,7 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_ module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1132,7 +1133,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1175,7 +1176,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1188,7 +1189,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--numeric-ids'), (False, '--numeric-owner')), + 'feature_available,option_flag', + ((True, '--numeric-ids'), (False, '--numeric-owner')), ) def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( feature_available, option_flag @@ -1223,7 +1225,7 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1276,7 +1278,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1290,7 +1292,12 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter @pytest.mark.parametrize( 'option_name,option_value', - (('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),), + ( + ('ctime', True), + ('ctime', False), + ('birthtime', True), + ('birthtime', False), + ), ) def test_create_archive_with_basic_option_calls_borg_with_corresponding_parameter( option_name, option_value @@ -1326,7 +1333,7 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1380,7 +1387,7 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1434,7 +1441,7 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1477,7 +1484,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1520,7 +1527,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1563,7 +1570,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1606,7 +1613,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1648,7 +1655,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1690,7 +1697,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1733,7 +1740,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1766,7 +1773,12 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--info', '--progress',), + ('borg', 'create') + + REPO_ARCHIVE_WITH_PATHS + + ( + '--info', + '--progress', + ), output_log_level=logging.INFO, output_file=module.DO_NOT_CAPTURE, borg_local_path='borg', @@ -1777,7 +1789,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1820,7 +1832,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1880,7 +1892,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1943,7 +1955,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2011,7 +2023,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2074,7 +2086,7 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2115,7 +2127,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): json_output = module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2157,7 +2169,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() json_output = module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2193,7 +2205,7 @@ def test_create_archive_with_source_directories_glob_expands(): ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'), + ('borg', 'create', f'repo::{DEFAULT_ARCHIVE_NAME}', 'foo', 'food'), output_log_level=logging.INFO, output_file=None, borg_local_path='borg', @@ -2204,7 +2216,7 @@ def test_create_archive_with_source_directories_glob_expands(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo*'], 'repositories': ['repo'], @@ -2236,7 +2248,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'), + ('borg', 'create', f'repo::{DEFAULT_ARCHIVE_NAME}', 'foo*'), output_log_level=logging.INFO, output_file=None, borg_local_path='borg', @@ -2247,7 +2259,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo*'], 'repositories': ['repo'], @@ -2279,7 +2291,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'), + ('borg', 'create', f'repo::{DEFAULT_ARCHIVE_NAME}', 'foo', 'food'), output_log_level=logging.INFO, output_file=None, borg_local_path='borg', @@ -2289,7 +2301,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo*'], 'repositories': ['repo'], @@ -2331,7 +2343,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2345,7 +2357,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - repository_archive_pattern = 'repo::Documents_{hostname}-{now}' + repository_archive_pattern = 'repo::Documents_{hostname}-{now}' # noqa: FS003 flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('map_directories_to_devices').and_return({}) @@ -2374,13 +2386,13 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, + storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', ) @@ -2388,7 +2400,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): def test_create_archive_with_repository_accepts_borg_placeholders(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - repository_archive_pattern = '{fqdn}::Documents_{hostname}-{now}' + repository_archive_pattern = '{fqdn}::Documents_{hostname}-{now}' # noqa: FS003 flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('map_directories_to_devices').and_return({}) @@ -2417,13 +2429,13 @@ def test_create_archive_with_repository_accepts_borg_placeholders(): module.create_archive( dry_run=False, - repository='{fqdn}', + repository_path='{fqdn}', # noqa: FS003 location_config={ 'source_directories': ['foo', 'bar'], - 'repositories': ['{fqdn}'], + 'repositories': ['{fqdn}'], # noqa: FS003 'exclude_patterns': None, }, - storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, + storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', ) @@ -2459,7 +2471,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2519,7 +2531,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2543,7 +2555,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ with pytest.raises(ValueError): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index a3c0352..92776dd 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -32,7 +32,7 @@ def test_export_tar_archive_calls_borg_with_path_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=['path1', 'path2'], destination_path='test.tar', @@ -53,7 +53,7 @@ def test_export_tar_archive_calls_borg_with_local_path_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -75,7 +75,7 @@ def test_export_tar_archive_calls_borg_with_remote_path_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -97,7 +97,7 @@ def test_export_tar_archive_calls_borg_with_umask_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -118,7 +118,7 @@ def test_export_tar_archive_calls_borg_with_lock_wait_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -138,7 +138,7 @@ def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -160,7 +160,7 @@ def test_export_tar_archive_with_log_debug_calls_borg_with_debug_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -179,7 +179,7 @@ def test_export_tar_archive_calls_borg_with_dry_run_parameter(): module.export_tar_archive( dry_run=True, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -200,7 +200,7 @@ def test_export_tar_archive_calls_borg_with_tar_filter_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -223,7 +223,7 @@ def test_export_tar_archive_calls_borg_with_list_parameter(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -245,7 +245,7 @@ def test_export_tar_archive_calls_borg_with_strip_components_parameter(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -265,7 +265,7 @@ def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): module.export_tar_archive( dry_run=False, - repository='server:repo', + repository_path='server:repo', archive='archive', paths=None, destination_path='test.tar', @@ -284,7 +284,7 @@ def test_export_tar_archive_calls_borg_with_stdout_destination_path(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='-', diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index d27026e..26fd738 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -11,7 +11,9 @@ from ..test_verbosity import insert_logging_mock def insert_execute_command_mock(command, working_directory=None): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - command, working_directory=working_directory, extra_environment=None, + command, + working_directory=working_directory, + extra_environment=None, ).once() @@ -23,7 +25,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_last_archive(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -32,7 +34,7 @@ def test_extract_last_archive_dry_run_without_any_archives_should_not_raise(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(('repo',)) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -45,7 +47,7 @@ def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_paramet ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -60,7 +62,7 @@ def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_param ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -74,7 +76,7 @@ def test_extract_last_archive_dry_run_calls_borg_via_local_path(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', - repository='repo', + repository_path='repo', lock_wait=None, local_path='borg1', ) @@ -92,7 +94,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', - repository='repo', + repository_path='repo', lock_wait=None, remote_path='borg1', ) @@ -108,7 +110,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=5 + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=5 ) @@ -152,7 +154,11 @@ def test_extract_archive_calls_borg_with_remote_path_parameters(): @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--numeric-ids'), (False, '--numeric-owner'),), + 'feature_available,option_flag', + ( + (True, '--numeric-ids'), + (False, '--numeric-owner'), + ), ) def test_extract_archive_calls_borg_with_numeric_ids_parameter(feature_available, option_flag): flexmock(module.os.path).should_receive('abspath').and_return('repo') @@ -441,7 +447,9 @@ def test_extract_archive_skips_abspath_for_remote_repository(): flexmock(module.os.path).should_receive('abspath').never() flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'extract', 'server:repo::archive'), working_directory=None, extra_environment=None, + ('borg', 'extract', 'server:repo::archive'), + working_directory=None, + extra_environment=None, ).once() flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index 7bf621d..2eaff0a 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -1,3 +1,4 @@ +import pytest from flexmock import flexmock from borgmatic.borg import flags as module @@ -50,7 +51,7 @@ def test_make_flags_from_arguments_omits_excludes(): def test_make_repository_flags_with_borg_features_includes_repo_flag(): flexmock(module.feature).should_receive('available').and_return(True) - assert module.make_repository_flags(repository='repo', local_borg_version='1.2.3') == ( + assert module.make_repository_flags(repository_path='repo', local_borg_version='1.2.3') == ( '--repo', 'repo', ) @@ -59,20 +60,90 @@ def test_make_repository_flags_with_borg_features_includes_repo_flag(): def test_make_repository_flags_without_borg_features_includes_omits_flag(): flexmock(module.feature).should_receive('available').and_return(False) - assert module.make_repository_flags(repository='repo', local_borg_version='1.2.3') == ('repo',) + assert module.make_repository_flags(repository_path='repo', local_borg_version='1.2.3') == ( + 'repo', + ) def test_make_repository_archive_flags_with_borg_features_separates_repository_and_archive(): flexmock(module.feature).should_receive('available').and_return(True) assert module.make_repository_archive_flags( - repository='repo', archive='archive', local_borg_version='1.2.3' - ) == ('--repo', 'repo', 'archive',) + repository_path='repo', archive='archive', local_borg_version='1.2.3' + ) == ( + '--repo', + 'repo', + 'archive', + ) def test_make_repository_archive_flags_with_borg_features_joins_repository_and_archive(): flexmock(module.feature).should_receive('available').and_return(False) assert module.make_repository_archive_flags( - repository='repo', archive='archive', local_borg_version='1.2.3' + repository_path='repo', archive='archive', local_borg_version='1.2.3' ) == ('repo::archive',) + + +@pytest.mark.parametrize( + 'match_archives, archive_name_format,feature_available,expected_result', + ( + (None, None, True, ()), + (None, '', True, ()), + ( + 're:foo-.*', + '{hostname}-{now}', + True, + ('--match-archives', 're:foo-.*'), + ), # noqa: FS003 + ( + 'sh:foo-*', + '{hostname}-{now}', + False, + ('--glob-archives', 'foo-*'), + ), # noqa: FS003 + ( + 'foo-*', + '{hostname}-{now}', + False, + ('--glob-archives', 'foo-*'), + ), # noqa: FS003 + ( + None, + '{hostname}-docs-{now}', # noqa: FS003 + True, + ('--match-archives', 'sh:{hostname}-docs-*'), # noqa: FS003 + ), + ( + None, + '{utcnow}-docs-{user}', # noqa: FS003 + True, + ('--match-archives', 'sh:*-docs-{user}'), # noqa: FS003 + ), + (None, '{fqdn}-{pid}', True, ('--match-archives', 'sh:{fqdn}-*')), # noqa: FS003 + ( + None, + 'stuff-{now:%Y-%m-%dT%H:%M:%S.%f}', # noqa: FS003 + True, + ('--match-archives', 'sh:stuff-*'), + ), + ( + None, + '{hostname}-docs-{now}', # noqa: FS003 + False, + ('--glob-archives', '{hostname}-docs-*'), # noqa: FS003 + ), + (None, '{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')), # noqa: FS003 + ), +) +def test_make_match_archives_flags_makes_flags_with_globs( + match_archives, archive_name_format, feature_available, expected_result +): + flexmock(module.feature).should_receive('available').and_return(feature_available) + + assert ( + module.make_match_archives_flags( + match_archives, archive_name_format, local_borg_version=flexmock() + ) + == expected_result + ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index ab92065..2eed4fe 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -12,6 +12,9 @@ def test_display_archives_info_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -23,10 +26,10 @@ def test_display_archives_info_calls_borg_with_parameters(): ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -34,6 +37,9 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -45,10 +51,10 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): ) insert_logging_mock(logging.INFO) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -56,19 +62,23 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'info', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.INFO) json_output = module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=True, prefix=None), + info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) assert json_output == '[]' @@ -78,6 +88,9 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -90,10 +103,10 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -101,19 +114,23 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'info', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.DEBUG) json_output = module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=True, prefix=None), + info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) assert json_output == '[]' @@ -123,18 +140,22 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'info', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') json_output = module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=True, prefix=None), + info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) assert json_output == '[]' @@ -144,24 +165,24 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags').with_args( - 'match-archives', 'archive' + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'archive', None, '2.3.4' ).and_return(('--match-archives', 'archive')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'info', '--repo', 'repo', '--match-archives', 'archive'), + ('borg', 'info', '--match-archives', 'archive', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, borg_local_path='borg', extra_environment=None, ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive='archive', json=False, prefix=None), + info_arguments=flexmock(archive='archive', json=False, prefix=None, match_archives=None), ) @@ -169,6 +190,9 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -180,10 +204,10 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), local_path='borg1', ) @@ -195,6 +219,9 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para flexmock(module.flags).should_receive('make_flags').with_args( 'remote-path', 'borg1' ).and_return(('--remote-path', 'borg1')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -206,10 +233,10 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), remote_path='borg1', ) @@ -221,6 +248,9 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete flexmock(module.flags).should_receive('make_flags').with_args('lock-wait', 5).and_return( ('--lock-wait', '5') ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) storage_config = {'lock_wait': 5} @@ -233,20 +263,23 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) -def test_display_archives_info_with_prefix_calls_borg_with_match_archives_parameters(): +def test_display_archives_info_transforms_prefix_into_match_archives_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'sh:foo*' ).and_return(('--match-archives', 'sh:foo*')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -258,19 +291,128 @@ def test_display_archives_info_with_prefix_calls_borg_with_match_archives_parame ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix='foo'), ) -@pytest.mark.parametrize('argument_name', ('match_archives', 'sort_by', 'first', 'last')) +def test_display_archives_info_prefers_prefix_over_archive_name_format(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args( + 'match-archives', 'sh:foo*' + ).and_return(('--match-archives', 'sh:foo*')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:foo*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix='foo'), + ) + + +def test_display_archives_info_transforms_archive_name_format_into_match_archives_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:bar-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), + ) + + +def test_display_archives_with_match_archives_option_calls_borg_with_match_archives_parameter(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'sh:foo-*', 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:foo-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:foo-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={ + 'archive_name_format': 'bar-{now}', # noqa: FS003 + 'match_archives': 'sh:foo-*', + }, + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), + ) + + +def test_display_archives_with_match_archives_flag_calls_borg_with_match_archives_parameter(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'sh:foo-*', 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:foo-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:foo-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'), + ) + + +@pytest.mark.parametrize('argument_name', ('sort_by', 'first', 'last')) def test_display_archives_info_passes_through_arguments_to_borg(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flag_name = f"--{argument_name.replace('_', ' ')}" flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (flag_name, 'value') ) @@ -284,8 +426,10 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None, **{argument_name: 'value'}), + info_arguments=flexmock( + archive=None, json=False, prefix=None, match_archives=None, **{argument_name: 'value'} + ), ) diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index cedcec8..0a7db4c 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -16,7 +16,7 @@ def test_make_list_command_includes_log_info(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -32,7 +32,7 @@ def test_make_list_command_includes_json_but_not_info(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), @@ -48,7 +48,7 @@ def test_make_list_command_includes_log_debug(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -64,7 +64,7 @@ def test_make_list_command_includes_json_but_not_debug(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), @@ -79,7 +79,7 @@ def test_make_list_command_includes_json(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), @@ -96,7 +96,7 @@ def test_make_list_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={'lock_wait': 5}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -113,7 +113,7 @@ def test_make_list_command_includes_archive(): ) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=None, json=False), @@ -130,7 +130,7 @@ def test_make_list_command_includes_archive_and_path(): ) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False), @@ -145,7 +145,7 @@ def test_make_list_command_includes_local_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -163,7 +163,7 @@ def test_make_list_command_includes_remote_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -179,7 +179,7 @@ def test_make_list_command_includes_short(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False, short=True), @@ -210,7 +210,7 @@ def test_make_list_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock( @@ -259,7 +259,7 @@ def test_capture_archive_listing_does_not_raise(): flexmock(module).should_receive('make_list_command') module.capture_archive_listing( - repository='repo', + repository_path='repo', archive='archive', storage_config=flexmock(), local_borg_version=flexmock(), @@ -284,7 +284,7 @@ def test_list_archive_calls_borg_with_parameters(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -301,7 +301,7 @@ def test_list_archive_calls_borg_with_parameters(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -318,7 +318,7 @@ def test_list_archive_with_archive_and_json_errors(): with pytest.raises(ValueError): module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -343,7 +343,7 @@ def test_list_archive_calls_borg_with_local_path(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -360,7 +360,7 @@ def test_list_archive_calls_borg_with_local_path(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -387,7 +387,8 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module.rlist).should_receive('make_rlist_command').and_return(('borg', 'list', 'repo')) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list', 'repo'), extra_environment=None, + ('borg', 'list', 'repo'), + extra_environment=None, ).and_return('archive1\narchive2').once() flexmock(module).should_receive('make_list_command').and_return( ('borg', 'list', 'repo::archive1') @@ -408,7 +409,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -433,7 +434,7 @@ def test_list_archive_calls_borg_with_archive(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -450,7 +451,7 @@ def test_list_archive_calls_borg_with_archive(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -480,7 +481,7 @@ def test_list_archive_without_archive_delegates_to_list_repository(): flexmock(module).should_receive('execute_command').never() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -510,7 +511,7 @@ def test_list_archive_with_borg_features_without_archive_delegates_to_list_repos flexmock(module).should_receive('execute_command').never() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -518,9 +519,18 @@ def test_list_archive_with_borg_features_without_archive_delegates_to_list_repos @pytest.mark.parametrize( - 'archive_filter_flag', ('prefix', 'match_archives', 'sort_by', 'first', 'last',), + 'archive_filter_flag', + ( + 'prefix', + 'match_archives', + 'sort_by', + 'first', + 'last', + ), ) -def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_flag,): +def test_list_archive_with_archive_ignores_archive_filter_flag( + archive_filter_flag, +): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.logger).answer = lambda message: None @@ -537,7 +547,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_fl module.feature.Feature.RLIST, '1.2.3' ).and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -556,7 +566,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_fl ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -566,7 +576,14 @@ def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_fl @pytest.mark.parametrize( - 'archive_filter_flag', ('prefix', 'match_archives', 'sort_by', 'first', 'last',), + 'archive_filter_flag', + ( + 'prefix', + 'match_archives', + 'sort_by', + 'first', + 'last', + ), ) def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes_it_to_rlist( archive_filter_flag, @@ -586,7 +603,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.rlist).should_receive('make_rlist_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=argparse.Namespace( @@ -597,11 +614,12 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes ).and_return(('borg', 'rlist', '--repo', 'repo')) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rlist', '--repo', 'repo'), extra_environment=None, + ('borg', 'rlist', '--repo', 'repo'), + extra_environment=None, ).and_return('archive1\narchive2').once() flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -619,7 +637,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes ).and_return(('borg', 'list', '--repo', 'repo', 'archive1')) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -652,7 +670,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 7f2060f..658b2e5 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -10,7 +10,9 @@ from ..test_verbosity import insert_logging_mock def insert_execute_command_mock(command): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - command, borg_local_path='borg', extra_environment=None, + command, + borg_local_path='borg', + extra_environment=None, ).once() @@ -20,7 +22,7 @@ def test_mount_archive_calls_borg_with_required_flags(): insert_execute_command_mock(('borg', 'mount', 'repo', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive=None, mount_point='/mnt', paths=None, @@ -33,13 +35,18 @@ def test_mount_archive_calls_borg_with_required_flags(): def test_mount_archive_with_borg_features_calls_borg_with_repository_and_match_archives_flags(): flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) insert_execute_command_mock( ('borg', 'mount', '--repo', 'repo', '--match-archives', 'archive', '/mnt') ) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -58,7 +65,7 @@ def test_mount_archive_without_archive_calls_borg_with_repository_flags_only(): insert_execute_command_mock(('borg', 'mount', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -77,7 +84,7 @@ def test_mount_archive_calls_borg_with_path_flags(): insert_execute_command_mock(('borg', 'mount', 'repo::archive', '/mnt', 'path1', 'path2')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=['path1', 'path2'], @@ -98,7 +105,7 @@ def test_mount_archive_calls_borg_with_remote_path_flags(): ) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -118,7 +125,7 @@ def test_mount_archive_calls_borg_with_umask_flags(): insert_execute_command_mock(('borg', 'mount', '--umask', '0770', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -137,7 +144,7 @@ def test_mount_archive_calls_borg_with_lock_wait_flags(): insert_execute_command_mock(('borg', 'mount', '--lock-wait', '5', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -157,7 +164,7 @@ def test_mount_archive_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -177,7 +184,7 @@ def test_mount_archive_with_log_debug_calls_borg_with_debug_flags(): insert_logging_mock(logging.DEBUG) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -202,7 +209,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter(): ).once() module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -221,7 +228,7 @@ def test_mount_archive_calls_borg_with_options_flags(): insert_execute_command_mock(('borg', 'mount', '-o', 'super_mount', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index ed4101e..128bdc0 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -18,60 +18,93 @@ def insert_execute_command_mock(prune_command, output_log_level): ).once() -BASE_PRUNE_FLAGS = (('--keep-daily', '1'), ('--keep-weekly', '2'), ('--keep-monthly', '3')) +BASE_PRUNE_FLAGS = ('--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3') -def test_make_prune_flags_returns_flags_from_config_plus_default_prefix_glob(): +def test_make_prune_flags_returns_flags_from_config(): retention_config = OrderedDict((('keep_daily', 1), ('keep_weekly', 2), ('keep_monthly', 3))) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') - assert tuple(result) == BASE_PRUNE_FLAGS + (('--match-archives', 'sh:{hostname}-*'),) + assert result == BASE_PRUNE_FLAGS def test_make_prune_flags_accepts_prefix_with_placeholders(): - retention_config = OrderedDict((('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}'))) + retention_config = OrderedDict( + (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 + ) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') - expected = (('--keep-daily', '1'), ('--match-archives', 'sh:Documents_{hostname}-{now}*')) + expected = ( + '--keep-daily', + '1', + '--match-archives', + 'sh:Documents_{hostname}-{now}*', # noqa: FS003 + ) - assert tuple(result) == expected + assert result == expected def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives(): - retention_config = OrderedDict((('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}'))) + retention_config = OrderedDict( + (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 + ) flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') - expected = (('--keep-daily', '1'), ('--glob-archives', 'Documents_{hostname}-{now}*')) + expected = ( + '--keep-daily', + '1', + '--glob-archives', + 'Documents_{hostname}-{now}*', # noqa: FS003 + ) - assert tuple(result) == expected + assert result == expected -def test_make_prune_flags_treats_empty_prefix_as_no_prefix(): - retention_config = OrderedDict((('keep_daily', 1), ('prefix', ''))) +def test_make_prune_flags_prefers_prefix_to_archive_name_format(): + storage_config = {'archive_name_format': 'bar-{now}'} # noqa: FS003 + retention_config = OrderedDict((('keep_daily', 1), ('prefix', 'bar-'))) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').never() - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3') - expected = (('--keep-daily', '1'),) + expected = ( + '--keep-daily', + '1', + '--match-archives', + 'sh:bar-*', # noqa: FS003 + ) - assert tuple(result) == expected + assert result == expected -def test_make_prune_flags_treats_none_prefix_as_no_prefix(): +def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): + storage_config = {'archive_name_format': 'bar-{now}'} # noqa: FS003 retention_config = OrderedDict((('keep_daily', 1), ('prefix', None))) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3') - expected = (('--keep-daily', '1'),) + expected = ( + '--keep-daily', + '1', + '--match-archives', + 'sh:bar-*', # noqa: FS003 + ) - assert tuple(result) == expected + assert result == expected PRUNE_COMMAND = ('borg', 'prune', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3') @@ -86,7 +119,7 @@ def test_prune_archives_calls_borg_with_parameters(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -102,7 +135,7 @@ def test_prune_archives_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.prune_archives( - repository='repo', + repository_path='repo', storage_config={}, dry_run=False, retention_config=flexmock(), @@ -119,7 +152,7 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.prune_archives( - repository='repo', + repository_path='repo', storage_config={}, dry_run=False, retention_config=flexmock(), @@ -135,7 +168,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO) module.prune_archives( - repository='repo', + repository_path='repo', storage_config={}, dry_run=True, retention_config=flexmock(), @@ -152,7 +185,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -169,7 +202,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters( module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -186,7 +219,7 @@ def test_prune_archives_with_stats_calls_borg_with_stats_parameter_and_answer_ou module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -203,7 +236,7 @@ def test_prune_archives_with_files_calls_borg_with_list_parameter_and_answer_out module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -221,7 +254,7 @@ def test_prune_archives_with_umask_calls_borg_with_umask_parameters(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', @@ -238,7 +271,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', @@ -254,7 +287,7 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={'extra_borg_options': {'prune': '--extra --options'}}, retention_config=flexmock(), local_borg_version='1.2.3', diff --git a/tests/unit/borg/test_rcreate.py b/tests/unit/borg/test_rcreate.py index 612ec11..4da04df 100644 --- a/tests/unit/borg/test_rcreate.py +++ b/tests/unit/borg/test_rcreate.py @@ -36,11 +36,16 @@ def test_create_repository_calls_borg_with_flags(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -51,11 +56,16 @@ def test_create_repository_with_dry_run_skips_borg_call(): insert_rinfo_command_not_found_mock() flexmock(module).should_receive('execute_command').never() flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=True, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -65,7 +75,12 @@ def test_create_repository_with_dry_run_skips_borg_call(): def test_create_repository_raises_for_borg_rcreate_error(): insert_rinfo_command_not_found_mock() flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').and_raise( module.subprocess.CalledProcessError(2, 'borg rcreate') @@ -74,7 +89,7 @@ def test_create_repository_raises_for_borg_rcreate_error(): with pytest.raises(subprocess.CalledProcessError): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -84,11 +99,16 @@ def test_create_repository_raises_for_borg_rcreate_error(): def test_create_repository_skips_creation_when_repository_already_exists(): insert_rinfo_command_found_mock() flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -103,7 +123,7 @@ def test_create_repository_raises_for_unknown_rinfo_command_error(): with pytest.raises(subprocess.CalledProcessError): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -114,11 +134,16 @@ def test_create_repository_with_source_repository_calls_borg_with_other_repo_fla insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--other-repo', 'other.borg', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -130,11 +155,16 @@ def test_create_repository_with_copy_crypt_key_calls_borg_with_copy_crypt_key_fl insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--copy-crypt-key', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -146,11 +176,16 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--append-only', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -162,11 +197,16 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--storage-quota', '5G', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -178,11 +218,16 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--make-parent-dirs', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -195,11 +240,16 @@ def test_create_repository_with_log_info_calls_borg_with_info_flag(): insert_rcreate_command_mock(RCREATE_COMMAND + ('--info', '--repo', 'repo')) insert_logging_mock(logging.INFO) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -211,11 +261,16 @@ def test_create_repository_with_log_debug_calls_borg_with_debug_flag(): insert_rcreate_command_mock(RCREATE_COMMAND + ('--debug', '--repo', 'repo')) insert_logging_mock(logging.DEBUG) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -226,11 +281,16 @@ def test_create_repository_with_local_path_calls_borg_via_local_path(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(('borg1',) + RCREATE_COMMAND[1:] + ('--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -242,11 +302,16 @@ def test_create_repository_with_remote_path_calls_borg_with_remote_path_flag(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--remote-path', 'borg1', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -258,11 +323,16 @@ def test_create_repository_with_extra_borg_options_calls_borg_with_extra_options insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--extra', '--options', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={'extra_borg_options': {'rcreate': '--extra --options'}}, local_borg_version='2.3.4', encryption_mode='repokey', diff --git a/tests/unit/borg/test_rinfo.py b/tests/unit/borg/test_rinfo.py index d9cbf9b..979b253 100644 --- a/tests/unit/borg/test_rinfo.py +++ b/tests/unit/borg/test_rinfo.py @@ -11,7 +11,12 @@ def test_display_repository_info_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--repo', 'repo'), @@ -21,7 +26,7 @@ def test_display_repository_info_calls_borg_with_parameters(): ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -42,7 +47,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_ ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -53,7 +58,12 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--info', '--repo', 'repo'), @@ -63,7 +73,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): ) insert_logging_mock(logging.INFO) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -74,15 +84,21 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'rinfo', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.INFO) json_output = module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), @@ -95,7 +111,12 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter( flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--debug', '--show-rc', '--repo', 'repo'), @@ -106,7 +127,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter( insert_logging_mock(logging.DEBUG) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -117,15 +138,21 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'rinfo', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.DEBUG) json_output = module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), @@ -138,14 +165,20 @@ def test_display_repository_info_with_json_calls_borg_with_json_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'rinfo', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') json_output = module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), @@ -158,7 +191,12 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg1', 'rinfo', '--repo', 'repo'), @@ -168,7 +206,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -180,7 +218,12 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_pa flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--remote-path', 'borg1', '--repo', 'repo'), @@ -190,7 +233,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_pa ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -203,7 +246,12 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parame flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--lock-wait', '5', '--repo', 'repo'), @@ -213,7 +261,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parame ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index 178a820..b83ba61 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -29,7 +29,8 @@ def test_resolve_archive_name_calls_borg_with_parameters(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') assert ( @@ -42,7 +43,8 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_parameter(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') insert_logging_mock(logging.INFO) @@ -56,7 +58,8 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_parameter( expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') insert_logging_mock(logging.DEBUG) @@ -70,7 +73,8 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') assert ( @@ -100,7 +104,8 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param def test_resolve_archive_name_without_archives_raises(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return('') with pytest.raises(ValueError): @@ -127,14 +132,19 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter def test_make_rlist_command_includes_log_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--info', 'repo') @@ -143,14 +153,19 @@ def test_make_rlist_command_includes_log_info(): def test_make_rlist_command_includes_json_but_not_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=True, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--json', 'repo') @@ -159,14 +174,19 @@ def test_make_rlist_command_includes_json_but_not_info(): def test_make_rlist_command_includes_log_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--debug', '--show-rc', 'repo') @@ -175,14 +195,19 @@ def test_make_rlist_command_includes_log_debug(): def test_make_rlist_command_includes_json_but_not_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=True, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--json', 'repo') @@ -190,14 +215,19 @@ def test_make_rlist_command_includes_json_but_not_debug(): def test_make_rlist_command_includes_json(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=True, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--json', 'repo') @@ -207,14 +237,19 @@ def test_make_rlist_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') ).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={'lock_wait': 5}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--lock-wait', '5', 'repo') @@ -222,14 +257,19 @@ def test_make_rlist_command_includes_lock_wait(): def test_make_rlist_command_includes_local_path(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), local_path='borg2', ) @@ -240,14 +280,19 @@ def test_make_rlist_command_includes_remote_path(): flexmock(module.flags).should_receive('make_flags').and_return( ('--remote-path', 'borg2') ).and_return(()).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), remote_path='borg2', ) @@ -258,11 +303,14 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( ('--match-archives', 'sh:foo*') ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), @@ -271,16 +319,59 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') +def test_make_rlist_command_prefers_prefix_over_archive_name_format(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( + ('--match-archives', 'sh:foo*') + ) + flexmock(module.flags).should_receive('make_match_archives_flags').never() + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + ) + + assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') + + +def test_make_rlist_command_transforms_archive_name_format_into_match_archives(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='1.2.3', + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), + ) + + assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo') + + def test_make_rlist_command_includes_short(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--short',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None, short=True), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None, short=True + ), ) assert command == ('borg', 'list', '--short', 'repo') @@ -289,7 +380,6 @@ def test_make_rlist_command_includes_short(): @pytest.mark.parametrize( 'argument_name', ( - 'match_archives', 'sort_by', 'first', 'last', @@ -301,13 +391,16 @@ def test_make_rlist_command_includes_short(): ) def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (f"--{argument_name.replace('_', '-')}", 'value') ) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( @@ -315,6 +408,7 @@ def test_make_rlist_command_includes_additional_flags(argument_name): paths=None, json=False, prefix=None, + match_archives=None, find_paths=None, format=None, **{argument_name: 'value'}, @@ -324,6 +418,37 @@ def test_make_rlist_command_includes_additional_flags(argument_name): assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo') +def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_parameters(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'foo-*', + None, + '1.2.3', + ).and_return(('--match-archives', 'foo-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock( + archive=None, + paths=None, + json=False, + prefix=None, + match_archives='foo-*', + find_paths=None, + format=None, + ), + ) + + assert command == ('borg', 'list', '--match-archives', 'foo-*', 'repo') + + def test_list_repository_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -331,7 +456,7 @@ def test_list_repository_calls_borg_with_parameters(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_rlist_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, @@ -347,7 +472,7 @@ def test_list_repository_calls_borg_with_parameters(): ).once() module.list_repository( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, @@ -362,7 +487,7 @@ def test_list_repository_with_json_returns_borg_output(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_rlist_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, @@ -374,7 +499,7 @@ def test_list_repository_with_json_returns_borg_output(): assert ( module.list_repository( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 27b0347..8f41bf5 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -12,6 +12,7 @@ def test_transfer_archives_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -25,7 +26,7 @@ def test_transfer_archives_calls_borg_with_flags(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -41,6 +42,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): flexmock(module.flags).should_receive('make_flags').with_args('dry-run', True).and_return( ('--dry-run',) ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -54,7 +56,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): module.transfer_archives( dry_run=True, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -67,6 +69,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -80,7 +83,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): insert_logging_mock(logging.INFO) module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -93,6 +96,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -107,7 +111,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -120,8 +124,8 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags').with_args( - 'match-archives', 'archive' + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'archive', 'bar-{now}', '2.3.4' # noqa: FS003 ).and_return(('--match-archives', 'archive')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -136,8 +140,8 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): module.transfer_archives( dry_run=False, - repository='repo', - storage_config={}, + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive='archive', progress=None, match_archives=None, source_repository=None @@ -149,8 +153,8 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags').with_args( - 'match-archives', 'sh:foo*' + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'sh:foo*', 'bar-{now}', '2.3.4' # noqa: FS003 ).and_return(('--match-archives', 'sh:foo*')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -165,8 +169,8 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl module.transfer_archives( dry_run=False, - repository='repo', - storage_config={}, + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None @@ -174,10 +178,40 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl ) +def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archives_flag(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'transfer', '--match-archives', 'sh:bar-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, + borg_local_path='borg', + extra_environment=None, + ) + + module.transfer_archives( + dry_run=False, + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), + ) + + def test_transfer_archives_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -191,7 +225,7 @@ def test_transfer_archives_with_local_path_calls_borg_via_local_path(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -208,6 +242,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.flags).should_receive('make_flags').with_args( 'remote-path', 'borg2' ).and_return(('--remote-path', 'borg2')) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -221,7 +256,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -238,6 +273,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.flags).should_receive('make_flags').with_args('lock-wait', 5).and_return( ('--lock-wait', '5') ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) storage_config = {'lock_wait': 5} @@ -252,7 +288,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -265,7 +301,8 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( @@ -278,7 +315,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -293,6 +330,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flag_name = f"--{argument_name.replace('_', ' ')}" flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (flag_name, 'value') ) @@ -308,7 +346,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -327,6 +365,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla flexmock(module.flags).should_receive('make_flags').with_args('other-repo', 'other').and_return( ('--other-repo', 'other') ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -340,7 +379,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( diff --git a/tests/unit/borg/test_version.py b/tests/unit/borg/test_version.py index 66789a8..a051f69 100644 --- a/tests/unit/borg/test_version.py +++ b/tests/unit/borg/test_version.py @@ -15,7 +15,8 @@ def insert_execute_command_and_capture_output_mock( ): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - command, extra_environment=None, + command, + extra_environment=None, ).once().and_return(version_output) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 19ac00d..bd98c01 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -15,7 +15,7 @@ def test_run_configuration_runs_actions_for_each_repository(): flexmock(module).should_receive('run_actions').and_return(expected_results[:1]).and_return( expected_results[1:] ) - config = {'location': {'repositories': ['foo', 'bar']}} + config = {'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}} arguments = {'global': flexmock(monitoring_verbosity=1)} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -75,7 +75,7 @@ def test_run_configuration_logs_actions_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -91,7 +91,7 @@ def test_run_configuration_bails_for_actions_soft_failure(): flexmock(module).should_receive('run_actions').and_raise(error) flexmock(module).should_receive('log_error_records').never() flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -108,7 +108,7 @@ def test_run_configuration_logs_monitor_log_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_return([]) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -126,7 +126,7 @@ def test_run_configuration_bails_for_monitor_log_soft_failure(): flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -143,7 +143,7 @@ def test_run_configuration_logs_monitor_finish_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_return([]) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -161,7 +161,7 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure(): flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -178,7 +178,7 @@ def test_run_configuration_logs_on_error_hook_error(): expected_results[:1] ).and_return(expected_results[1:]) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -194,7 +194,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -209,7 +209,7 @@ def test_run_configuration_retries_soft_error(): flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([]) flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} + config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == [] @@ -229,9 +229,10 @@ def test_run_configuration_retries_hard_error(): ).and_return([flexmock()]) error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', OSError, + 'foo: Error running actions for repository', + OSError, ).and_return(error_logs) - config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} + config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -249,13 +250,13 @@ def test_run_configuration_repos_ordered(): flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError ).and_return(expected_results[1:]).ordered() - config = {'location': {'repositories': ['foo', 'bar']}} + config = {'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results -def test_run_configuration_retries_round_robbin(): +def test_run_configuration_retries_round_robin(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.command).should_receive('execute_hook') @@ -280,7 +281,10 @@ def test_run_configuration_retries_round_robbin(): flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError ).and_return(bar_error_logs).ordered() - config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} + config = { + 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, + 'storage': {'retries': 1}, + } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == foo_error_logs + bar_error_logs @@ -309,7 +313,10 @@ def test_run_configuration_retries_one_passes(): flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError ).and_return(error_logs).ordered() - config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} + config = { + 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, + 'storage': {'retries': 1}, + } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -348,7 +355,10 @@ def test_run_configuration_retry_wait(): flexmock(module).should_receive('log_error_records').with_args( 'foo: Error running actions for repository', OSError ).and_return(error_logs).ordered() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_wait': 10}} + config = { + 'location': {'repositories': [{'path': 'foo'}]}, + 'storage': {'retries': 3, 'retry_wait': 10}, + } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -384,7 +394,7 @@ def test_run_configuration_retries_timeout_multiple_repos(): 'bar: Error running actions for repository', OSError ).and_return(error_logs).ordered() config = { - 'location': {'repositories': ['foo', 'bar']}, + 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, 'storage': {'retries': 1, 'retry_wait': 10}, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -399,7 +409,7 @@ def test_run_actions_runs_rcreate(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'rcreate': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rcreate': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -409,11 +419,48 @@ def test_run_actions_runs_rcreate(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) +def test_run_actions_adds_log_file_to_hook_context(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + expected = flexmock() + flexmock(borgmatic.actions.create).should_receive('run_create').with_args( + config_filename=object, + repository={'path': 'repo'}, + location={'repositories': []}, + storage=object, + hooks={}, + hook_context={'repository': 'repo', 'repositories': '', 'log_file': 'foo'}, + local_borg_version=object, + create_arguments=object, + global_arguments=object, + dry_run_label='', + local_path=object, + remote_path=object, + ).once().and_return(expected) + + result = tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), + hooks={}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) + assert result == (expected,) + + def test_run_actions_runs_transfer(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.command).should_receive('execute_hook') @@ -421,7 +468,7 @@ def test_run_actions_runs_transfer(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'transfer': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'transfer': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -431,7 +478,7 @@ def test_run_actions_runs_transfer(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) @@ -444,7 +491,7 @@ def test_run_actions_runs_create(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'create': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -454,7 +501,7 @@ def test_run_actions_runs_create(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -467,7 +514,7 @@ def test_run_actions_runs_prune(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'prune': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'prune': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -477,7 +524,7 @@ def test_run_actions_runs_prune(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) @@ -489,7 +536,7 @@ def test_run_actions_runs_compact(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'compact': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'compact': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -499,7 +546,7 @@ def test_run_actions_runs_compact(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) @@ -512,7 +559,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'check': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -522,7 +569,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) @@ -535,7 +582,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'check': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -545,7 +592,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) @@ -557,7 +604,7 @@ def test_run_actions_runs_extract(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'extract': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'extract': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -567,7 +614,7 @@ def test_run_actions_runs_extract(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) @@ -579,7 +626,7 @@ def test_run_actions_runs_export_tar(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'export-tar': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'export-tar': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -589,7 +636,7 @@ def test_run_actions_runs_export_tar(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) @@ -601,7 +648,7 @@ def test_run_actions_runs_mount(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'mount': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'mount': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -611,7 +658,7 @@ def test_run_actions_runs_mount(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) @@ -623,7 +670,7 @@ def test_run_actions_runs_restore(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'restore': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'restore': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -633,7 +680,7 @@ def test_run_actions_runs_restore(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) @@ -646,7 +693,7 @@ def test_run_actions_runs_rlist(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'rlist': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rlist': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -656,7 +703,7 @@ def test_run_actions_runs_rlist(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -670,7 +717,7 @@ def test_run_actions_runs_list(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'list': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'list': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -680,7 +727,7 @@ def test_run_actions_runs_list(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -694,7 +741,7 @@ def test_run_actions_runs_rinfo(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'rinfo': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rinfo': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -704,7 +751,7 @@ def test_run_actions_runs_rinfo(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -718,7 +765,7 @@ def test_run_actions_runs_info(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'info': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'info': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -728,7 +775,7 @@ def test_run_actions_runs_info(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -741,7 +788,7 @@ def test_run_actions_runs_break_lock(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'break-lock': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'break-lock': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -751,7 +798,7 @@ def test_run_actions_runs_break_lock(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) @@ -763,7 +810,7 @@ def test_run_actions_runs_borg(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'borg': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'borg': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -773,7 +820,7 @@ def test_run_actions_runs_borg(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) @@ -787,7 +834,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): tuple( module.run_actions( arguments={ - 'global': flexmock(dry_run=False), + 'global': flexmock(dry_run=False, log_file='foo'), 'borg': flexmock(), 'restore': flexmock(), }, @@ -800,7 +847,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository={'path': 'repo'}, ) ) diff --git a/tests/unit/config/test_environment.py b/tests/unit/config/test_environment.py index b7b56dd..3e342fa 100644 --- a/tests/unit/config/test_environment.py +++ b/tests/unit/config/test_environment.py @@ -12,7 +12,7 @@ def test_env(monkeypatch): def test_env_braces(monkeypatch): monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo') - config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} + config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} # noqa: FS003 module.resolve_env_variables(config) assert config == {'key': 'Hello foo'} @@ -20,7 +20,7 @@ def test_env_braces(monkeypatch): def test_env_multi(monkeypatch): monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo') monkeypatch.setenv('MY_CUSTOM_VALUE2', 'bar') - config = {'key': 'Hello ${MY_CUSTOM_VALUE}${MY_CUSTOM_VALUE2}'} + config = {'key': 'Hello ${MY_CUSTOM_VALUE}${MY_CUSTOM_VALUE2}'} # noqa: FS003 module.resolve_env_variables(config) assert config == {'key': 'Hello foobar'} @@ -28,21 +28,21 @@ def test_env_multi(monkeypatch): def test_env_escape(monkeypatch): monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo') monkeypatch.setenv('MY_CUSTOM_VALUE2', 'bar') - config = {'key': r'Hello ${MY_CUSTOM_VALUE} \${MY_CUSTOM_VALUE}'} + config = {'key': r'Hello ${MY_CUSTOM_VALUE} \${MY_CUSTOM_VALUE}'} # noqa: FS003 module.resolve_env_variables(config) - assert config == {'key': r'Hello foo ${MY_CUSTOM_VALUE}'} + assert config == {'key': r'Hello foo ${MY_CUSTOM_VALUE}'} # noqa: FS003 def test_env_default_value(monkeypatch): monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False) - config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'} + config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'} # noqa: FS003 module.resolve_env_variables(config) assert config == {'key': 'Hello bar'} def test_env_unknown(monkeypatch): monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False) - config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} + config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} # noqa: FS003 with pytest.raises(ValueError): module.resolve_env_variables(config) @@ -55,20 +55,20 @@ def test_env_full(monkeypatch): 'dict': { 'key': 'value', 'anotherdict': { - 'key': 'My ${MY_CUSTOM_VALUE} here', - 'other': '${MY_CUSTOM_VALUE}', - 'escaped': r'\${MY_CUSTOM_VALUE}', + 'key': 'My ${MY_CUSTOM_VALUE} here', # noqa: FS003 + 'other': '${MY_CUSTOM_VALUE}', # noqa: FS003 + 'escaped': r'\${MY_CUSTOM_VALUE}', # noqa: FS003 'list': [ - '/home/${MY_CUSTOM_VALUE}/.local', + '/home/${MY_CUSTOM_VALUE}/.local', # noqa: FS003 '/var/log/', - '/home/${MY_CUSTOM_VALUE2:-bar}/.config', + '/home/${MY_CUSTOM_VALUE2:-bar}/.config', # noqa: FS003 ], }, }, 'list': [ - '/home/${MY_CUSTOM_VALUE}/.local', + '/home/${MY_CUSTOM_VALUE}/.local', # noqa: FS003 '/var/log/', - '/home/${MY_CUSTOM_VALUE2-bar}/.config', + '/home/${MY_CUSTOM_VALUE2-bar}/.config', # noqa: FS003 ], } module.resolve_env_variables(config) @@ -79,7 +79,7 @@ def test_env_full(monkeypatch): 'anotherdict': { 'key': 'My foo here', 'other': 'foo', - 'escaped': '${MY_CUSTOM_VALUE}', + 'escaped': '${MY_CUSTOM_VALUE}', # noqa: FS003 'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'], }, }, diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index 821c320..63e3187 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -21,13 +21,21 @@ from borgmatic.config import normalize as module {'location': {'source_directories': ['foo', 'bar']}}, False, ), - ({'location': None}, {'location': None}, False,), + ( + {'location': None}, + {'location': None}, + False, + ), ( {'storage': {'compression': 'yes_please'}}, {'storage': {'compression': 'yes_please'}}, False, ), - ({'storage': None}, {'storage': None}, False,), + ( + {'storage': None}, + {'storage': None}, + False, + ), ( {'hooks': {'healthchecks': 'https://example.com'}}, {'hooks': {'healthchecks': {'ping_url': 'https://example.com'}}}, @@ -48,10 +56,9 @@ from borgmatic.config import normalize as module {'hooks': {'cronhub': {'ping_url': 'https://example.com'}}}, False, ), - ({'hooks': None}, {'hooks': None}, False,), ( - {'consistency': {'checks': ['archives']}}, - {'consistency': {'checks': [{'name': 'archives'}]}}, + {'hooks': None}, + {'hooks': None}, False, ), ( @@ -59,9 +66,26 @@ from borgmatic.config import normalize as module {'consistency': {'checks': [{'name': 'archives'}]}}, False, ), - ({'consistency': None}, {'consistency': None}, False,), - ({'location': {'numeric_owner': False}}, {'location': {'numeric_ids': False}}, False,), - ({'location': {'bsd_flags': False}}, {'location': {'flags': False}}, False,), + ( + {'consistency': {'checks': ['archives']}}, + {'consistency': {'checks': [{'name': 'archives'}]}}, + False, + ), + ( + {'consistency': None}, + {'consistency': None}, + False, + ), + ( + {'location': {'numeric_owner': False}}, + {'location': {'numeric_ids': False}}, + False, + ), + ( + {'location': {'bsd_flags': False}}, + {'location': {'flags': False}}, + False, + ), ( {'storage': {'remote_rate_limit': False}}, {'storage': {'upload_rate_limit': False}}, @@ -69,27 +93,42 @@ from borgmatic.config import normalize as module ), ( {'location': {'repositories': ['foo@bar:/repo']}}, - {'location': {'repositories': ['ssh://foo@bar/repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/repo'}]}}, True, ), ( {'location': {'repositories': ['foo@bar:repo']}}, - {'location': {'repositories': ['ssh://foo@bar/./repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/./repo'}]}}, True, ), ( {'location': {'repositories': ['foo@bar:~/repo']}}, - {'location': {'repositories': ['ssh://foo@bar/~/repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/~/repo'}]}}, True, ), ( {'location': {'repositories': ['ssh://foo@bar:1234/repo']}}, - {'location': {'repositories': ['ssh://foo@bar:1234/repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar:1234/repo'}]}}, False, ), ( {'location': {'repositories': ['file:///repo']}}, - {'location': {'repositories': ['/repo']}}, + {'location': {'repositories': [{'path': '/repo'}]}}, + False, + ), + ( + {'location': {'repositories': [{'path': 'foo@bar:/repo', 'label': 'foo'}]}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/repo', 'label': 'foo'}]}}, + True, + ), + ( + {'location': {'repositories': [{'path': 'file:///repo', 'label': 'foo'}]}}, + {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, + False, + ), + ( + {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, + {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, False, ), ), @@ -105,3 +144,15 @@ def test_normalize_applies_hard_coded_normalization_to_config( assert logs else: assert logs == [] + + +def test_normalize_raises_error_if_repository_data_is_not_consistent(): + with pytest.raises(TypeError): + module.normalize( + 'test.yaml', + { + 'location': { + 'repositories': [{'path': 'foo@bar:/repo', 'label': 'foo'}, 'file:///repo'] + } + }, + ) diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 6a9f4a4..e2b9f98 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -4,6 +4,28 @@ from flexmock import flexmock from borgmatic.config import validate as module +def test_schema_filename_finds_schema_path(): + schema_path = '/var/borgmatic/config/schema.yaml' + + flexmock(module.importlib_metadata).should_receive('files').and_return( + flexmock(match=lambda path: False, locate=lambda: None), + flexmock(match=lambda path: True, locate=lambda: schema_path), + flexmock(match=lambda path: False, locate=lambda: None), + ) + + assert module.schema_filename() == schema_path + + +def test_schema_filename_with_missing_schema_path_raises(): + flexmock(module.importlib_metadata).should_receive('files').and_return( + flexmock(match=lambda path: False, locate=lambda: None), + flexmock(match=lambda path: False, locate=lambda: None), + ) + + with pytest.raises(FileNotFoundError): + assert module.schema_filename() + + def test_format_json_error_path_element_formats_array_index(): module.format_json_error_path_element(3) == '[3]' @@ -13,7 +35,7 @@ def test_format_json_error_path_element_formats_property(): def test_format_json_error_formats_error_including_path(): - flexmock(module).format_json_error_path_element = lambda element: '.{}'.format(element) + flexmock(module).format_json_error_path_element = lambda element: f'.{element}' error = flexmock(message='oops', path=['foo', 'bar']) assert module.format_json_error(error) == "At 'foo.bar': oops" @@ -37,7 +59,7 @@ def test_validation_error_string_contains_errors(): assert 'uh oh' in result -def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories(): +def test_apply_logical_validation_raises_if_unknown_repository_in_check_repositories(): flexmock(module).format_json_error = lambda error: error.message with pytest.raises(module.Validation_error): @@ -51,24 +73,40 @@ def test_apply_locical_validation_raises_if_unknown_repository_in_check_reposito ) -def test_apply_locical_validation_does_not_raise_if_known_repository_in_check_repositories(): +def test_apply_logical_validation_does_not_raise_if_known_repository_path_in_check_repositories(): module.apply_logical_validation( 'config.yaml', { - 'location': {'repositories': ['repo.borg', 'other.borg']}, + 'location': {'repositories': [{'path': 'repo.borg'}, {'path': 'other.borg'}]}, 'retention': {'keep_secondly': 1000}, 'consistency': {'check_repositories': ['repo.borg']}, }, ) +def test_apply_logical_validation_does_not_raise_if_known_repository_label_in_check_repositories(): + module.apply_logical_validation( + 'config.yaml', + { + 'location': { + 'repositories': [ + {'path': 'repo.borg', 'label': 'my_repo'}, + {'path': 'other.borg', 'label': 'other_repo'}, + ] + }, + 'retention': {'keep_secondly': 1000}, + 'consistency': {'check_repositories': ['my_repo']}, + }, + ) + + def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present(): module.apply_logical_validation( 'config.yaml', { - 'storage': {'archive_name_format': '{hostname}-{now}'}, - 'retention': {'prefix': '{hostname}-'}, - 'consistency': {'prefix': '{hostname}-'}, + 'storage': {'archive_name_format': '{hostname}-{now}'}, # noqa: FS003 + 'retention': {'prefix': '{hostname}-'}, # noqa: FS003 + 'consistency': {'prefix': '{hostname}-'}, # noqa: FS003 }, ) @@ -121,6 +159,15 @@ def test_guard_configuration_contains_repository_does_not_raise_when_repository_ ) +def test_guard_configuration_contains_repository_does_not_raise_when_repository_label_in_config(): + module.guard_configuration_contains_repository( + repository='repo', + configurations={ + 'config.yaml': {'location': {'repositories': [{'path': 'foo/bar', 'label': 'repo'}]}} + }, + ) + + def test_guard_configuration_contains_repository_does_not_raise_when_repository_not_given(): module.guard_configuration_contains_repository( repository=None, configurations={'config.yaml': {'location': {'repositories': ['repo']}}} @@ -164,13 +211,15 @@ def test_guard_single_repository_selected_raises_when_multiple_repositories_conf def test_guard_single_repository_selected_does_not_raise_when_single_repository_configured_and_none_selected(): module.guard_single_repository_selected( - repository=None, configurations={'config.yaml': {'location': {'repositories': ['repo']}}}, + repository=None, + configurations={'config.yaml': {'location': {'repositories': ['repo']}}}, ) def test_guard_single_repository_selected_does_not_raise_when_no_repositories_configured_and_one_selected(): module.guard_single_repository_selected( - repository='repo', configurations={'config.yaml': {'location': {'repositories': []}}}, + repository='repo', + configurations={'config.yaml': {'location': {'repositories': []}}}, ) diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index 3d1686d..3a657eb 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -11,27 +11,20 @@ def test_interpolate_context_passes_through_command_without_variable(): def test_interpolate_context_passes_through_command_with_unknown_variable(): - assert ( - module.interpolate_context('test.yaml', 'pre-backup', 'ls {baz}', {'foo': 'bar'}) - == 'ls {baz}' - ) + command = 'ls {baz}' # noqa: FS003 + + assert module.interpolate_context('test.yaml', 'pre-backup', command, {'foo': 'bar'}) == command def test_interpolate_context_interpolates_variables(): + command = 'ls {foo}{baz} {baz}' # noqa: FS003 context = {'foo': 'bar', 'baz': 'quux'} assert ( - module.interpolate_context('test.yaml', 'pre-backup', 'ls {foo}{baz} {baz}', context) - == 'ls barquux quux' + module.interpolate_context('test.yaml', 'pre-backup', command, context) == 'ls barquux quux' ) -def test_interpolate_context_does_not_touch_unknown_variables(): - context = {'foo': 'bar', 'baz': 'quux'} - - assert module.interpolate_context('test.yaml', 'pre-backup', 'ls {wtf}', context) == 'ls {wtf}' - - def test_execute_hook_invokes_each_command(): flexmock(module).should_receive('interpolate_context').replace_with( lambda config_file, hook_description, command, context: command diff --git a/tests/unit/hooks/test_cronhub.py b/tests/unit/hooks/test_cronhub.py index f470b88..2941592 100644 --- a/tests/unit/hooks/test_cronhub.py +++ b/tests/unit/hooks/test_cronhub.py @@ -108,5 +108,9 @@ def test_ping_monitor_with_unsupported_monitoring_state(): hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + hook_config, + 'config.yaml', + module.monitor.State.LOG, + monitoring_log_level=1, + dry_run=False, ) diff --git a/tests/unit/hooks/test_cronitor.py b/tests/unit/hooks/test_cronitor.py index 7ec1e2e..12b9685 100644 --- a/tests/unit/hooks/test_cronitor.py +++ b/tests/unit/hooks/test_cronitor.py @@ -93,5 +93,9 @@ def test_ping_monitor_with_unsupported_monitoring_state(): hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + hook_config, + 'config.yaml', + module.monitor.State.LOG, + monitoring_log_level=1, + dry_run=False, ) diff --git a/tests/unit/hooks/test_healthchecks.py b/tests/unit/hooks/test_healthchecks.py index d577953..5c6977d 100644 --- a/tests/unit/hooks/test_healthchecks.py +++ b/tests/unit/hooks/test_healthchecks.py @@ -206,7 +206,7 @@ def test_ping_monitor_with_ping_uuid_hits_corresponding_url(): payload = 'data' flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload) flexmock(module.requests).should_receive('post').with_args( - 'https://hc-ping.com/{}'.format(hook_config['ping_url']), + f"https://hc-ping.com/{hook_config['ping_url']}", data=payload.encode('utf-8'), verify=True, ).and_return(flexmock(ok=True)) diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index f61f3c7..f038a88 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -17,7 +17,7 @@ def test_dump_databases_runs_mongodump_for_each_database(): for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_command').with_args( - ['mongodump', '--db', name, '--archive', '>', 'databases/localhost/{}'.format(name)], + ['mongodump', '--db', name, '--archive', '>', f'databases/localhost/{name}'], shell=True, run_to_completion=False, ).and_return(process).once() @@ -114,7 +114,8 @@ def test_dump_databases_runs_mongodump_with_directory_format(): flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').with_args( - ['mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'], shell=True, + ['mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'], + shell=True, ).and_return(flexmock()).once() assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [] @@ -157,7 +158,7 @@ def test_dump_databases_runs_mongodumpall_for_all_databases(): def test_restore_database_dump_runs_mongorestore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -189,7 +190,9 @@ def test_restore_database_dump_errors_on_multiple_database_config(): def test_restore_database_dump_runs_mongorestore_with_hostname_and_port(): - database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] + database_config = [ + {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None} + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -223,6 +226,7 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): 'username': 'mongo', 'password': 'trustsome1', 'authentication_database': 'admin', + 'schemas': None, } ] extract_process = flexmock(stdout=flexmock()) @@ -254,7 +258,7 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): def test_restore_database_dump_runs_mongorestore_with_options(): - database_config = [{'name': 'foo', 'restore_options': '--harder'}] + database_config = [{'name': 'foo', 'restore_options': '--harder', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -271,8 +275,36 @@ def test_restore_database_dump_runs_mongorestore_with_options(): ) +def test_restore_databases_dump_runs_mongorestore_with_schemas(): + database_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + [ + 'mongorestore', + '--archive', + '--drop', + '--db', + 'foo', + '--nsInclude', + 'bar', + '--nsInclude', + 'baz', + ], + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + module.restore_database_dump( + database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + ) + + def test_restore_database_dump_runs_psql_for_all_database_dump(): - database_config = [{'name': 'all'}] + database_config = [{'name': 'all', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -290,7 +322,7 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): def test_restore_database_dump_with_dry_run_skips_restore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') @@ -302,7 +334,7 @@ def test_restore_database_dump_with_dry_run_skips_restore(): def test_restore_database_dump_without_extract_process_restores_from_disk(): - database_config = [{'name': 'foo', 'format': 'directory'}] + database_config = [{'name': 'foo', 'format': 'directory', 'schemas': None}] flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path') diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 1e8df69..da5da16 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -149,8 +149,14 @@ def test_execute_dump_command_runs_mysqldump(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--add-drop-database', '--databases', 'foo', '>', 'dump',), - shell=True, + ( + 'mysqldump', + '--add-drop-database', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment=None, run_to_completion=False, ).and_return(process).once() @@ -176,8 +182,13 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--databases', 'foo', '>', 'dump',), - shell=True, + ( + 'mysqldump', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment=None, run_to_completion=False, ).and_return(process).once() @@ -214,10 +225,9 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): 'tcp', '--databases', 'foo', - '>', + '--result-file', 'dump', ), - shell=True, extra_environment=None, run_to_completion=False, ).and_return(process).once() @@ -243,8 +253,16 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--add-drop-database', '--user', 'root', '--databases', 'foo', '>', 'dump',), - shell=True, + ( + 'mysqldump', + '--add-drop-database', + '--user', + 'root', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment={'MYSQL_PWD': 'trustsome1'}, run_to_completion=False, ).and_return(process).once() @@ -270,8 +288,15 @@ def test_execute_dump_command_runs_mysqldump_with_options(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--stuff=such', '--add-drop-database', '--databases', 'foo', '>', 'dump',), - shell=True, + ( + 'mysqldump', + '--stuff=such', + '--add-drop-database', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment=None, run_to_completion=False, ).and_return(process).once() diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 9cb4c0f..70cff92 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -134,7 +134,7 @@ def test_dump_databases_runs_pg_dump_for_each_database(): 'custom', name, '>', - 'databases/localhost/{}'.format(name), + f'databases/localhost/{name}', ), shell=True, extra_environment={'PGSSLMODE': 'disable'}, @@ -411,7 +411,7 @@ def test_dump_databases_runs_non_default_pg_dump(): def test_restore_database_dump_runs_pg_restore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -458,7 +458,9 @@ def test_restore_database_dump_errors_on_multiple_database_config(): def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): - database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] + database_config = [ + {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None} + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -506,7 +508,9 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): def test_restore_database_dump_runs_pg_restore_with_username_and_password(): - database_config = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}] + database_config = [ + {'name': 'foo', 'username': 'postgres', 'password': 'trustsome1', 'schemas': None} + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return( @@ -553,7 +557,12 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): def test_restore_database_dump_runs_pg_restore_with_options(): database_config = [ - {'name': 'foo', 'restore_options': '--harder', 'analyze_options': '--smarter'} + { + 'name': 'foo', + 'restore_options': '--harder', + 'analyze_options': '--smarter', + 'schemas': None, + } ] extract_process = flexmock(stdout=flexmock()) @@ -596,7 +605,7 @@ def test_restore_database_dump_runs_pg_restore_with_options(): def test_restore_database_dump_runs_psql_for_all_database_dump(): - database_config = [{'name': 'all'}] + database_config = [{'name': 'all', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -621,7 +630,12 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): database_config = [ - {'name': 'foo', 'pg_restore_command': 'special_pg_restore', 'psql_command': 'special_psql'} + { + 'name': 'foo', + 'pg_restore_command': 'special_pg_restore', + 'psql_command': 'special_psql', + 'schemas': None, + } ] extract_process = flexmock(stdout=flexmock()) @@ -654,7 +668,7 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): def test_restore_database_dump_with_dry_run_skips_restore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') @@ -667,7 +681,7 @@ def test_restore_database_dump_with_dry_run_skips_restore(): def test_restore_database_dump_without_extract_process_restores_from_disk(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') @@ -696,3 +710,39 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): module.restore_database_dump( database_config, 'test.yaml', {}, dry_run=False, extract_process=None ) + + +def test_restore_database_dump_with_schemas_restores_schemas(): + database_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] + + flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path') + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'pg_restore', + '--no-password', + '--if-exists', + '--exit-on-error', + '--clean', + '--dbname', + 'foo', + '/dump/path', + '--schema', + 'bar', + '--schema', + 'baz', + ), + processes=[], + output_log_level=logging.DEBUG, + input_file=None, + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + flexmock(module).should_receive('execute_command').with_args( + ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + + module.restore_database_dump( + database_config, 'test.yaml', {}, dry_run=False, extract_process=None + ) diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 0441e9d..a6dd9d8 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -7,32 +7,32 @@ from borgmatic import execute as module @pytest.mark.parametrize( - 'process,exit_code,borg_local_path,expected_result', + 'command,exit_code,borg_local_path,expected_result', ( - (flexmock(args=['grep']), 2, None, True), - (flexmock(args=['grep']), 2, 'borg', True), - (flexmock(args=['borg']), 2, 'borg', True), - (flexmock(args=['borg1']), 2, 'borg1', True), - (flexmock(args=['grep']), 1, None, True), - (flexmock(args=['grep']), 1, 'borg', True), - (flexmock(args=['borg']), 1, 'borg', False), - (flexmock(args=['borg1']), 1, 'borg1', False), - (flexmock(args=['grep']), 0, None, False), - (flexmock(args=['grep']), 0, 'borg', False), - (flexmock(args=['borg']), 0, 'borg', False), - (flexmock(args=['borg1']), 0, 'borg1', False), + (['grep'], 2, None, True), + (['grep'], 2, 'borg', True), + (['borg'], 2, 'borg', True), + (['borg1'], 2, 'borg1', True), + (['grep'], 1, None, True), + (['grep'], 1, 'borg', True), + (['borg'], 1, 'borg', False), + (['borg1'], 1, 'borg1', False), + (['grep'], 0, None, False), + (['grep'], 0, 'borg', False), + (['borg'], 0, 'borg', False), + (['borg1'], 0, 'borg1', False), # -9 exit code occurs when child process get SIGKILLed. - (flexmock(args=['grep']), -9, None, True), - (flexmock(args=['grep']), -9, 'borg', True), - (flexmock(args=['borg']), -9, 'borg', True), - (flexmock(args=['borg1']), -9, 'borg1', True), - (flexmock(args=['borg']), None, None, False), + (['grep'], -9, None, True), + (['grep'], -9, 'borg', True), + (['borg'], -9, 'borg', True), + (['borg1'], -9, 'borg1', True), + (['borg'], None, None, False), ), ) def test_exit_code_indicates_error_respects_exit_code_and_borg_local_path( - process, exit_code, borg_local_path, expected_result + command, exit_code, borg_local_path, expected_result ): - assert module.exit_code_indicates_error(process, exit_code, borg_local_path) is expected_result + assert module.exit_code_indicates_error(command, exit_code, borg_local_path) is expected_result def test_command_for_process_converts_sequence_command_to_string(): @@ -65,6 +65,41 @@ def test_output_buffer_for_process_returns_stdout_when_not_excluded(): ) +def test_append_last_lines_under_max_line_count_appends(): + last_lines = ['last'] + flexmock(module.logger).should_receive('log').once() + + module.append_last_lines( + last_lines, captured_output=flexmock(), line='line', output_log_level=flexmock() + ) + + assert last_lines == ['last', 'line'] + + +def test_append_last_lines_over_max_line_count_trims_and_appends(): + original_last_lines = [str(number) for number in range(0, module.ERROR_OUTPUT_MAX_LINE_COUNT)] + last_lines = list(original_last_lines) + flexmock(module.logger).should_receive('log').once() + + module.append_last_lines( + last_lines, captured_output=flexmock(), line='line', output_log_level=flexmock() + ) + + assert last_lines == original_last_lines[1:] + ['line'] + + +def test_append_last_lines_with_output_log_level_none_appends_captured_output(): + last_lines = ['last'] + captured_output = ['captured'] + flexmock(module.logger).should_receive('log').never() + + module.append_last_lines( + last_lines, captured_output=captured_output, line='line', output_log_level=None + ) + + assert captured_output == ['captured', 'line'] + + def test_execute_command_calls_full_command(): full_command = ['foo', 'bar'] flexmock(module.os, environ={'a': 'b'}) @@ -239,6 +274,34 @@ def test_execute_command_and_capture_output_with_capture_stderr_returns_stderr() assert output == expected_output +def test_execute_command_and_capture_output_returns_output_when_process_error_is_not_considered_an_error(): + full_command = ['foo', 'bar'] + expected_output = '[]' + err_output = b'[]' + flexmock(module.os, environ={'a': 'b'}) + flexmock(module.subprocess).should_receive('check_output').with_args( + full_command, stderr=None, shell=False, env=None, cwd=None + ).and_raise(subprocess.CalledProcessError(1, full_command, err_output)).once() + flexmock(module).should_receive('exit_code_indicates_error').and_return(False).once() + + output = module.execute_command_and_capture_output(full_command) + + assert output == expected_output + + +def test_execute_command_and_capture_output_raises_when_command_errors(): + full_command = ['foo', 'bar'] + expected_output = '[]' + flexmock(module.os, environ={'a': 'b'}) + flexmock(module.subprocess).should_receive('check_output').with_args( + full_command, stderr=None, shell=False, env=None, cwd=None + ).and_raise(subprocess.CalledProcessError(2, full_command, expected_output)).once() + flexmock(module).should_receive('exit_code_indicates_error').and_return(True).once() + + with pytest.raises(subprocess.CalledProcessError): + module.execute_command_and_capture_output(full_command) + + def test_execute_command_and_capture_output_returns_output_with_shell(): full_command = ['foo', 'bar'] expected_output = '[]' @@ -257,7 +320,11 @@ def test_execute_command_and_capture_output_returns_output_with_extra_environmen expected_output = '[]' flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( - full_command, stderr=None, shell=False, env={'a': 'b', 'c': 'd'}, cwd=None, + full_command, + stderr=None, + shell=False, + env={'a': 'b', 'c': 'd'}, + cwd=None, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output( diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 0fb284a..dc9c748 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -285,7 +285,7 @@ def test_configure_logging_skips_syslog_if_interactive_console(): module.configure_logging(console_log_level=logging.INFO) -def test_configure_logging_to_logfile_instead_of_syslog(): +def test_configure_logging_to_log_file_instead_of_syslog(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER flexmock(module).should_receive('Multi_stream_handler').and_return( @@ -309,7 +309,36 @@ def test_configure_logging_to_logfile_instead_of_syslog(): ) -def test_configure_logging_skips_logfile_if_argument_is_none(): +def test_configure_logging_to_log_file_formats_with_custom_log_format(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.ANSWER + flexmock(module.logging).should_receive('Formatter').with_args( + '{message}', style='{' # noqa: FS003 + ).once() + flexmock(module).should_receive('Multi_stream_handler').and_return( + flexmock(setFormatter=lambda formatter: None, setLevel=lambda level: None) + ) + + flexmock(module).should_receive('interactive_console').and_return(False) + flexmock(module.logging).should_receive('basicConfig').with_args( + level=logging.DEBUG, handlers=tuple + ) + flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True) + flexmock(module.logging.handlers).should_receive('SysLogHandler').never() + file_handler = logging.handlers.WatchedFileHandler('/tmp/logfile') + flexmock(module.logging.handlers).should_receive('WatchedFileHandler').with_args( + '/tmp/logfile' + ).and_return(file_handler).once() + + module.configure_logging( + console_log_level=logging.INFO, + log_file_log_level=logging.DEBUG, + log_file='/tmp/logfile', + log_file_format='{message}', # noqa: FS003 + ) + + +def test_configure_logging_skips_log_file_if_argument_is_none(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER flexmock(module).should_receive('Multi_stream_handler').and_return( diff --git a/tox.ini b/tox.ini index 17b7a9d..3a2b476 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,7 @@ commands = py38,py39,py310,py311: black --check . isort --check-only --settings-path setup.cfg . flake8 borgmatic tests + codespell [testenv:black] commands = @@ -26,7 +27,9 @@ commands = pytest {posargs} [testenv:end-to-end] +usedevelop = False deps = -rtest_requirements.txt + . passenv = COVERAGE_FILE commands = pytest {posargs} --no-cov tests/end-to-end @@ -35,3 +38,8 @@ commands = deps = {[testenv]deps} commands = isort --settings-path setup.cfg . + +[testenv:codespell] +deps = {[testenv]deps} +commands = + codespell --write-changes