From 317dc7fbce45dd275af93e91857545db4e9ca0ec Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 21 Aug 2022 21:48:37 -0700 Subject: [PATCH] Add "before_actions" and "after_actions" command hooks that run before/after all the actions for each repository, update docs to cover per-repository configurations (#463). --- NEWS | 5 ++ borgmatic/commands/borgmatic.py | 18 ++++++++ borgmatic/config/schema.yaml | 46 +++++++++++++------ docs/Dockerfile | 2 +- ...reparation-and-cleanup-steps-to-backups.md | 7 +++ docs/how-to/deal-with-very-large-backups.md | 8 ++-- docs/how-to/make-per-application-backups.md | 21 +++++++-- docs/how-to/provide-your-passwords.md | 4 +- docs/how-to/run-arbitrary-borg-commands.md | 2 +- docs/how-to/set-up-backups.md | 2 +- docs/how-to/upgrade.md | 5 +- tests/unit/commands/test_borgmatic.py | 20 ++++++-- 12 files changed, 104 insertions(+), 36 deletions(-) diff --git a/NEWS b/NEWS index 9b2bc76..54efe6a 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,9 @@ 1.7.0.dev0 + * #463: Add "before_actions" and "after_actions" command hooks that run before/after all the + actions for each repository. These new hooks are a good place to run per-repository steps like + mounting/unmounting a remote filesystem. + * #463: Update documentation to cover per-repository configurations: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/ * #557: Support for Borg 2 while still working with Borg 1. This includes new borgmatic actions like "rcreate" (replaces "init"), "rlist" (list archives in repository), "rinfo" (show repository info), and "transfer" (for upgrading Borg repositories). For the most part, borgmatic tries to diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 05c86df..710ccad 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -252,6 +252,15 @@ def run_actions( 'repositories': ','.join(location['repositories']), } + command.execute_hook( + hooks.get('before_actions'), + hooks.get('umask'), + config_filename, + 'pre-actions', + global_arguments.dry_run, + **hook_context, + ) + if 'rcreate' in arguments: logger.info('{}: Creating repository'.format(repository)) borg_rcreate.create_repository( @@ -745,6 +754,15 @@ def run_actions( remote_path=remote_path, ) + command.execute_hook( + hooks.get('after_actions'), + hooks.get('umask'), + config_filename, + 'post-actions', + global_arguments.dry_run, + **hook_context, + ) + def load_configurations(config_filenames, overrides=None, resolve_env=True): ''' diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 2434ecc..8163811 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -145,10 +145,10 @@ properties: type: string description: | Any paths matching these patterns are excluded from backups. - Globs and tildes are expanded. (Note however that a glob - pattern must either start with a glob or be an absolute - path.) Do not backslash spaces in path names. See the output - of "borg help patterns" for more details. + Globs and tildes are expanded. Note that a glob pattern must + either start with a glob or be an absolute path. Do not + backslash spaces in path names. See the output of "borg help + patterns" for more details. example: - '*.pyc' - /home/*/.cache @@ -538,13 +538,22 @@ properties: prevent potential shell injection or privilege escalation. additionalProperties: false properties: + before_actions: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute + before all the actions for each repository. + example: + - echo "Starting actions." before_backup: type: array items: type: string description: | List of one or more shell commands or scripts to execute - before creating a backup, run once per configuration file. + before creating a backup, run once per repository. example: - echo "Starting a backup." before_prune: @@ -553,7 +562,7 @@ properties: type: string description: | List of one or more shell commands or scripts to execute - before pruning, run once per configuration file. + before pruning, run once per repository. example: - echo "Starting pruning." before_compact: @@ -562,7 +571,7 @@ properties: type: string description: | List of one or more shell commands or scripts to execute - before compaction, run once per configuration file. + before compaction, run once per repository. example: - echo "Starting compaction." before_check: @@ -571,7 +580,7 @@ properties: type: string description: | List of one or more shell commands or scripts to execute - before consistency checks, run once per configuration file. + before consistency checks, run once per repository. example: - echo "Starting checks." before_extract: @@ -580,7 +589,7 @@ properties: type: string description: | List of one or more shell commands or scripts to execute - before extracting a backup, run once per configuration file. + before extracting a backup, run once per repository. example: - echo "Starting extracting." after_backup: @@ -589,7 +598,7 @@ properties: type: string description: | List of one or more shell commands or scripts to execute - after creating a backup, run once per configuration file. + after creating a backup, run once per repository. example: - echo "Finished a backup." after_compact: @@ -598,7 +607,7 @@ properties: type: string description: | List of one or more shell commands or scripts to execute - after compaction, run once per configuration file. + after compaction, run once per repository. example: - echo "Finished compaction." after_prune: @@ -607,7 +616,7 @@ properties: type: string description: | List of one or more shell commands or scripts to execute - after pruning, run once per configuration file. + after pruning, run once per repository. example: - echo "Finished pruning." after_check: @@ -616,7 +625,7 @@ properties: type: string description: | List of one or more shell commands or scripts to execute - after consistency checks, run once per configuration file. + after consistency checks, run once per repository. example: - echo "Finished checks." after_extract: @@ -625,9 +634,18 @@ properties: type: string description: | List of one or more shell commands or scripts to execute - after extracting a backup, run once per configuration file. + after extracting a backup, run once per repository. example: - echo "Finished extracting." + after_actions: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute + after all actions for each repository. + example: + - echo "Finished actions." on_error: type: array items: diff --git a/docs/Dockerfile b/docs/Dockerfile index 35d50b7..8bdde7c 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -4,7 +4,7 @@ COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml RUN borgmatic --help > /command-line.txt \ - && for action in rcreate prune compact create check extract export-tar mount umount restore rlist list rinfo info borg; do \ + && for action in rcreate transfer prune compact create check extract export-tar mount umount restore rlist list rinfo info borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ && borgmatic "$action" --help >> /command-line.txt; done diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 913f1a9..c0bcfb8 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -40,6 +40,13 @@ There are additional hooks that run before/after other actions as well. For instance, `before_prune` runs before a `prune` action for a repository, while `after_prune` runs after it. +New in version 1.7.0 The +`before_actions` and `after_actions` hooks run before/after all the actions +(like `create`, `prune`, etc.) for each repository. These hooks are a good +place to run per-repository steps like mounting/unmounting a remote +filesystem. + + ## Variable interpolation The before and after action hooks support interpolating particular runtime diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index 8318ebc..a717a03 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -96,12 +96,12 @@ Unlike a real scheduler like cron, borgmatic only makes a best effort to run checks on the configured frequency. It compares that frequency with how long it's been since the last check for a given repository (as recorded in a file within `~/.borgmatic/checks`). If it hasn't been long enough, the check is -skipped. And you still have to run `borgmatic check` (or just `borgmatic`) in -order for checks to run, even when a `frequency` is configured! +skipped. And you still have to run `borgmatic check` (or `borgmatic` without +actions) in order for checks to run, even when a `frequency` is configured! This also applies *across* configuration files that have the same repository -configured. Just make sure you have the same check frequency configured in -each—or the most frequently configured check will apply. +configured. Make sure you have the same check frequency configured in each +though—or the most frequently configured check will apply. If you want to temporarily ignore your configured frequencies, you can invoke `borgmatic check --force` to run checks unconditionally. diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 06ff8c6..2c68670 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -8,13 +8,15 @@ eleventyNavigation: ## Multiple backup configurations You may find yourself wanting to create different backup policies for -different applications on your system. For instance, you may want one backup -configuration for your database data directory, and a different configuration -for your user home directories. +different applications on your system or even for different backup +repositories. For instance, you might want one backup configuration for your +database data directory and a different configuration for your user home +directories. Or one backup configuration for your local backups with a +different configuration for your remote repository. The way to accomplish that is pretty simple: Create multiple separate configuration files and place each one in a `/etc/borgmatic.d/` directory. For -instance: +instance, for applications: ```bash sudo mkdir /etc/borgmatic.d @@ -22,6 +24,14 @@ sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml ``` +Or, for repositories: + +```bash +sudo mkdir /etc/borgmatic.d +sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo1.yaml +sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo2.yaml +``` + When you set up multiple configuration files like this, borgmatic will run each one in turn from a single borgmatic invocation. This includes, by default, the traditional `/etc/borgmatic/config.yaml` as well. @@ -29,7 +39,8 @@ default, the traditional `/etc/borgmatic/config.yaml` as well. Each configuration file is interpreted independently, as if you ran borgmatic for each configuration file one at a time. In other words, borgmatic does not perform any merging of configuration files by default. If you'd like borgmatic -to merge your configuration files, see below about configuration includes. +to merge your configuration files, for instance to avoid duplication of +settings, see below about configuration includes. Additionally, the `~/.config/borgmatic.d/` directory works the same way as `/etc/borgmatic.d`. diff --git a/docs/how-to/provide-your-passwords.md b/docs/how-to/provide-your-passwords.md index e187512..76f89d8 100644 --- a/docs/how-to/provide-your-passwords.md +++ b/docs/how-to/provide-your-passwords.md @@ -25,8 +25,8 @@ storage: ``` This uses the `MY_PASSPHRASE` environment variable as your encryption -passphrase. Note that the `{` `}` brackets are required. Just `$MY_PASSPHRASE` -will not work. +passphrase. Note that the `{` `}` brackets are required. `$MY_PASSPHRASE` by +itself will not work. In the case of `encryption_passphrase` in particular, an alternate approach is to use Borg's `BORG_PASSPHRASE` environment variable, which doesn't even diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index 8ffddeb..9a5bc69 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -50,7 +50,7 @@ borgmatic borg rlist --short ``` This runs Borg's `rlist` command once on each configured borgmatic repository. -However, the native `borgmatic rlist` action should be preferred for most use. +(The native `borgmatic rlist` action should be preferred for most use.) What if you only want to run Borg on a single configured borgmatic repository when you've got several configured? Not a problem. diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 1814204..1f43bd9 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -347,7 +347,7 @@ instead: sudo su -c "borgmatic --bash-completion > /usr/share/bash-completion/completions/borgmatic" ``` -Or, if you'd like to install the script for just the current user: +Or, if you'd like to install the script for only the current user: ```bash mkdir --parents ~/.local/share/bash-completion/completions diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md index 28955c4..39043c1 100644 --- a/docs/how-to/upgrade.md +++ b/docs/how-to/upgrade.md @@ -122,9 +122,8 @@ files. To upgrade to a new version of Borg, you can generally install a new version the same way you installed the previous version, paying attention to any instructions included with each Borg release changelog linked from the -[releases page](https://github.com/borgbackup/borg/releases). However, some -more major Borg releases require additional steps that borgmatic can help -with. +[releases page](https://github.com/borgbackup/borg/releases). Some more major +Borg releases require additional steps that borgmatic can help with. ### Borg 1.2 to 2.0 diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 4b9f40f..d412e87 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -397,7 +397,9 @@ def test_run_actions_does_not_raise_for_transfer_action(): def test_run_actions_calls_hooks_for_prune_action(): flexmock(module.borg_prune).should_receive('prune_archives') - flexmock(module.command).should_receive('execute_hook').twice() + flexmock(module.command).should_receive('execute_hook').times( + 4 + ) # Before/after extract and before/after actions. arguments = { 'global': flexmock(monitoring_verbosity=1, dry_run=False), 'prune': flexmock(stats=flexmock(), list_archives=flexmock()), @@ -423,7 +425,9 @@ def test_run_actions_calls_hooks_for_prune_action(): def test_run_actions_calls_hooks_for_compact_action(): flexmock(module.borg_feature).should_receive('available').and_return(True) flexmock(module.borg_compact).should_receive('compact_segments') - flexmock(module.command).should_receive('execute_hook').twice() + flexmock(module.command).should_receive('execute_hook').times( + 4 + ) # Before/after extract and before/after actions. arguments = { 'global': flexmock(monitoring_verbosity=1, dry_run=False), 'compact': flexmock(progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock()), @@ -448,7 +452,9 @@ def test_run_actions_calls_hooks_for_compact_action(): def test_run_actions_executes_and_calls_hooks_for_create_action(): flexmock(module.borg_create).should_receive('create_archive') - flexmock(module.command).should_receive('execute_hook').twice() + flexmock(module.command).should_receive('execute_hook').times( + 4 + ) # Before/after extract and before/after actions. flexmock(module.dispatch).should_receive('call_hooks').and_return({}).times(3) arguments = { 'global': flexmock(monitoring_verbosity=1, dry_run=False), @@ -477,7 +483,9 @@ def test_run_actions_executes_and_calls_hooks_for_create_action(): def test_run_actions_calls_hooks_for_check_action(): flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True) flexmock(module.borg_check).should_receive('check_archives') - flexmock(module.command).should_receive('execute_hook').twice() + flexmock(module.command).should_receive('execute_hook').times( + 4 + ) # Before/after extract and before/after actions. arguments = { 'global': flexmock(monitoring_verbosity=1, dry_run=False), 'check': flexmock( @@ -505,7 +513,9 @@ def test_run_actions_calls_hooks_for_check_action(): def test_run_actions_calls_hooks_for_extract_action(): flexmock(module.validate).should_receive('repositories_match').and_return(True) flexmock(module.borg_extract).should_receive('extract_archive') - flexmock(module.command).should_receive('execute_hook').twice() + flexmock(module.command).should_receive('execute_hook').times( + 4 + ) # Before/after extract and before/after actions. arguments = { 'global': flexmock(monitoring_verbosity=1, dry_run=False), 'extract': flexmock(