User-defined hooks for global setup or cleanup that run before/after all actions. (#192).
This commit is contained in:
parent
a897ffd514
commit
e14ebee4e0
9 changed files with 219 additions and 64 deletions
5
NEWS
5
NEWS
|
@ -1,3 +1,8 @@
|
|||
1.3.21
|
||||
* #192: User-defined hooks for global setup or cleanup that run before/after all actions. See the
|
||||
documentation for more information:
|
||||
https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
|
||||
|
||||
1.3.20
|
||||
* #205: More robust sample systemd service: boot delay, network dependency, lowered CPU/IO
|
||||
priority, etc.
|
||||
|
|
|
@ -231,6 +231,32 @@ def load_configurations(config_filenames):
|
|||
return (configs, logs)
|
||||
|
||||
|
||||
def make_error_log_records(error, message):
|
||||
'''
|
||||
Given an exception object and error message text, yield a series of logging.LogRecord instances
|
||||
with error summary information.
|
||||
'''
|
||||
try:
|
||||
raise error
|
||||
except CalledProcessError as error:
|
||||
yield logging.makeLogRecord(
|
||||
dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
|
||||
)
|
||||
yield logging.makeLogRecord(
|
||||
dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output)
|
||||
)
|
||||
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error))
|
||||
except (ValueError, OSError) as error:
|
||||
yield logging.makeLogRecord(
|
||||
dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
|
||||
)
|
||||
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error))
|
||||
except: # noqa: E722
|
||||
# Raising above only as a means of determining the error type. Swallow the exception here
|
||||
# because we don't want the exception to propagate out of this function.
|
||||
pass
|
||||
|
||||
|
||||
def collect_configuration_run_summary_logs(configs, arguments):
|
||||
'''
|
||||
Given a dict of configuration filename to corresponding parsed configuration, and parsed
|
||||
|
@ -258,6 +284,33 @@ def collect_configuration_run_summary_logs(configs, arguments):
|
|||
)
|
||||
return
|
||||
|
||||
if not configs:
|
||||
yield logging.makeLogRecord(
|
||||
dict(
|
||||
levelno=logging.CRITICAL,
|
||||
levelname='CRITICAL',
|
||||
msg='{}: No configuration files found'.format(
|
||||
' '.join(arguments['global'].config_paths)
|
||||
),
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
if 'create' in arguments:
|
||||
for config_filename, config in configs.items():
|
||||
hooks = config.get('hooks', {})
|
||||
hook.execute_hook(
|
||||
hooks.get('before_everything'),
|
||||
hooks.get('umask'),
|
||||
config_filename,
|
||||
'pre-everything',
|
||||
arguments['global'].dry_run,
|
||||
)
|
||||
except (CalledProcessError, ValueError, OSError) as error:
|
||||
yield from make_error_log_records(error, 'Error running pre-everything hook')
|
||||
return
|
||||
|
||||
# Execute the actions corresponding to each configuration file.
|
||||
json_results = []
|
||||
for config_filename, config in configs.items():
|
||||
|
@ -270,45 +323,27 @@ def collect_configuration_run_summary_logs(configs, arguments):
|
|||
msg='{}: Successfully ran configuration file'.format(config_filename),
|
||||
)
|
||||
)
|
||||
except CalledProcessError as error:
|
||||
yield logging.makeLogRecord(
|
||||
dict(
|
||||
levelno=logging.CRITICAL,
|
||||
levelname='CRITICAL',
|
||||
msg='{}: Error running configuration file'.format(config_filename),
|
||||
)
|
||||
)
|
||||
yield logging.makeLogRecord(
|
||||
dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output)
|
||||
)
|
||||
yield logging.makeLogRecord(
|
||||
dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
|
||||
)
|
||||
except (ValueError, OSError) as error:
|
||||
yield logging.makeLogRecord(
|
||||
dict(
|
||||
levelno=logging.CRITICAL,
|
||||
levelname='CRITICAL',
|
||||
msg='{}: Error running configuration file'.format(config_filename),
|
||||
)
|
||||
)
|
||||
yield logging.makeLogRecord(
|
||||
dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
|
||||
except (CalledProcessError, ValueError, OSError) as error:
|
||||
yield from make_error_log_records(
|
||||
error, '{}: Error running configuration file'.format(config_filename)
|
||||
)
|
||||
|
||||
if json_results:
|
||||
sys.stdout.write(json.dumps(json_results))
|
||||
|
||||
if not configs:
|
||||
yield logging.makeLogRecord(
|
||||
dict(
|
||||
levelno=logging.CRITICAL,
|
||||
levelname='CRITICAL',
|
||||
msg='{}: No configuration files found'.format(
|
||||
' '.join(arguments['global'].config_paths)
|
||||
),
|
||||
)
|
||||
)
|
||||
try:
|
||||
if 'create' in arguments:
|
||||
for config_filename, config in configs.items():
|
||||
hooks = config.get('hooks', {})
|
||||
hook.execute_hook(
|
||||
hooks.get('after_everything'),
|
||||
hooks.get('umask'),
|
||||
config_filename,
|
||||
'post-everything',
|
||||
arguments['global'].dry_run,
|
||||
)
|
||||
except (CalledProcessError, ValueError, OSError) as error:
|
||||
yield from make_error_log_records(error, 'Error running post-everything hook')
|
||||
|
||||
|
||||
def exit_with_help_link(): # pragma: no cover
|
||||
|
|
|
@ -337,31 +337,52 @@ map:
|
|||
example: false
|
||||
hooks:
|
||||
desc: |
|
||||
Shell commands or scripts to execute before and after a backup or if an error has occurred.
|
||||
IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic.
|
||||
Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to
|
||||
prevent potential shell injection or privilege escalation.
|
||||
Shell commands or scripts to execute at various points during a borgmatic run.
|
||||
IMPORTANT: All provided commands and scripts are executed with user permissions of
|
||||
borgmatic. Do not forget to set secure permissions on this configuration file (chmod
|
||||
0600) as well as on any script called from a hook (chmod 0700) to prevent potential
|
||||
shell injection or privilege escalation.
|
||||
map:
|
||||
before_backup:
|
||||
seq:
|
||||
- type: str
|
||||
desc: List of one or more shell commands or scripts to execute before creating a backup.
|
||||
desc: |
|
||||
List of one or more shell commands or scripts to execute before creating a
|
||||
backup, run once per configuration file.
|
||||
example:
|
||||
- echo "Starting a backup job."
|
||||
- echo "Starting a backup."
|
||||
after_backup:
|
||||
seq:
|
||||
- type: str
|
||||
desc: List of one or more shell commands or scripts to execute after creating a backup.
|
||||
desc: |
|
||||
List of one or more shell commands or scripts to execute after creating a
|
||||
backup, run once per configuration file.
|
||||
example:
|
||||
- echo "Backup created."
|
||||
- echo "Created a backup."
|
||||
on_error:
|
||||
seq:
|
||||
- type: str
|
||||
desc: |
|
||||
List of one or more shell commands or scripts to execute when an exception occurs
|
||||
during a backup or when running a hook.
|
||||
during a backup or when running a before_backup or after_backup hook.
|
||||
example:
|
||||
- echo "Error while creating a backup or running a hook."
|
||||
- echo "Error while creating a backup or running a backup hook."
|
||||
before_everything:
|
||||
seq:
|
||||
- type: str
|
||||
desc: |
|
||||
List of one or more shell commands or scripts to execute before running all
|
||||
actions (if one of them is "create"), run once before all configuration files.
|
||||
example:
|
||||
- echo "Starting actions."
|
||||
after_everything:
|
||||
seq:
|
||||
- type: str
|
||||
desc: |
|
||||
List of one or more shell commands or scripts to execute after running all
|
||||
actions (if one of them is "create"), run once after all configuration files.
|
||||
example:
|
||||
- echo "Completed actions."
|
||||
umask:
|
||||
type: scalar
|
||||
desc: Umask used when executing hooks. Defaults to the umask that borgmatic is run with.
|
||||
|
|
|
@ -53,6 +53,8 @@ def execute_command(full_command, output_log_level=logging.INFO, shell=False):
|
|||
Execute the given command (a sequence of command/argument strings) and log its output at the
|
||||
given log level. If output log level is None, instead capture and return the output. If
|
||||
shell is True, execute the command within a shell.
|
||||
|
||||
Raise subprocesses.CalledProcessError if an error occurs while running the command.
|
||||
'''
|
||||
logger.debug(' '.join(full_command))
|
||||
|
||||
|
|
|
@ -13,6 +13,7 @@ def execute_hook(commands, umask, config_filename, description, dry_run):
|
|||
if this is a dry run.
|
||||
|
||||
Raise ValueError if the umask cannot be parsed.
|
||||
Raise subprocesses.CalledProcessError if an error occurs in a hook.
|
||||
'''
|
||||
if not commands:
|
||||
logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
|
||||
|
|
|
@ -20,22 +20,47 @@ hooks:
|
|||
- rm /to/file.sql
|
||||
```
|
||||
|
||||
borgmatic hooks run once per configuration file. `before_backup` hooks run
|
||||
prior to backups of all repositories. `after_backup` hooks run afterwards, but
|
||||
not if an error occurs in a previous hook or in the backups themselves.
|
||||
The `before_backup` and `after_backup` hooks each run once per configuration
|
||||
file. `before_backup` hooks run prior to backups of all repositories in a
|
||||
configuration file, right before the `create` action. `after_backup` hooks run
|
||||
afterwards, but not if an error occurs in a previous hook or in the backups
|
||||
themselves.
|
||||
|
||||
You can also use `before_everything` and `after_everything` hooks to perform
|
||||
global setup or cleanup:
|
||||
|
||||
```yaml
|
||||
hooks:
|
||||
before_everything:
|
||||
- set-up-stuff-globally
|
||||
after_everything:
|
||||
- clean-up-stuff-globally
|
||||
```
|
||||
|
||||
`before_everything` hooks collected from all borgmatic configuration files run
|
||||
once before all configuration files (prior to all actions), but only if there
|
||||
is a `create` action. An error encountered during a `before_everything` hook
|
||||
causes borgmatic to exit without creating backups.
|
||||
|
||||
`after_everything` hooks run once after all configuration files and actions,
|
||||
but only if there is a `create` action. It runs even if an error occurs during
|
||||
a backup or a backup hook, but not if an error occurs during a
|
||||
`before_everything` hook.
|
||||
|
||||
## Error hooks
|
||||
|
||||
borgmatic also runs `on_error` hooks if an error occurs, either when creating
|
||||
a backup or running another hook. Here's an example configuration:
|
||||
a backup or running a backup hook. Here's an example configuration:
|
||||
|
||||
```yaml
|
||||
hooks:
|
||||
on_error:
|
||||
- echo "Error while creating a backup or running a hook."
|
||||
- echo "Error while creating a backup or running a backup hook."
|
||||
```
|
||||
|
||||
Note however that borgmatic does not run `on_error` hooks if an error occurs
|
||||
within a `before_everything` or `after_everything` hook.
|
||||
|
||||
## Hook output
|
||||
|
||||
Any output produced by your hooks shows up both at the console and in syslog
|
||||
|
@ -48,7 +73,8 @@ your backups</a>.
|
|||
An important security note about hooks: borgmatic executes all hook commands
|
||||
with the user permissions of borgmatic itself. So to prevent potential shell
|
||||
injection or privilege escalation, do not forget to set secure permissions
|
||||
(`chmod 0700`) on borgmatic configuration files and scripts invoked by hooks.
|
||||
on borgmatic configuration files (`chmod 0600`) and scripts (`chmod 0700`)
|
||||
invoked by hooks.
|
||||
|
||||
|
||||
## Related documentation
|
||||
|
|
|
@ -7,11 +7,11 @@ To get up and running, first [install
|
|||
Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at
|
||||
least version 1.1.
|
||||
|
||||
Borgmatic consumes configurations in `/etc/borgmatic/` and `/etc/borgmatic.d/`
|
||||
by default. Therefore, we show how to install borgmatic for the root user which
|
||||
will have access permissions for these locations by default.
|
||||
By default, borgmatic looks for its configuration files in `/etc/borgmatic/`
|
||||
and `/etc/borgmatic.d/`, where the root user typically has read access.
|
||||
|
||||
Run the following commands to download and install borgmatic:
|
||||
So, to download and install borgmatic as the root user, run the following
|
||||
commands:
|
||||
|
||||
```bash
|
||||
sudo pip3 install --user --upgrade borgmatic
|
||||
|
@ -39,6 +39,7 @@ borgmatic:
|
|||
* [OpenBSD](http://ports.su/sysutils/borgmatic)
|
||||
* [openSUSE](https://software.opensuse.org/package/borgmatic)
|
||||
* [stand-alone binary](https://github.com/cmarquardt/borgmatic-binary)
|
||||
* [virtualenv](https://virtualenv.pypa.io/en/stable/)
|
||||
|
||||
|
||||
## Hosting providers
|
||||
|
|
2
setup.py
2
setup.py
|
@ -1,6 +1,6 @@
|
|||
from setuptools import find_packages, setup
|
||||
|
||||
VERSION = '1.3.20'
|
||||
VERSION = '1.3.21'
|
||||
|
||||
|
||||
setup(
|
||||
|
|
|
@ -24,10 +24,40 @@ def test_load_configurations_logs_critical_for_parse_error():
|
|||
configs, logs = tuple(module.load_configurations(('test.yaml',)))
|
||||
|
||||
assert configs == {}
|
||||
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
|
||||
assert {log.levelno for log in logs} == {module.logging.CRITICAL}
|
||||
|
||||
|
||||
def test_make_error_log_records_generates_output_logs_for_called_process_error():
|
||||
logs = tuple(
|
||||
module.make_error_log_records(
|
||||
subprocess.CalledProcessError(1, 'ls', 'error output'), 'Error'
|
||||
)
|
||||
)
|
||||
|
||||
assert {log.levelno for log in logs} == {module.logging.CRITICAL}
|
||||
assert any(log for log in logs if 'error output' in str(log))
|
||||
|
||||
|
||||
def test_make_error_log_records_generates_logs_for_value_error():
|
||||
logs = tuple(module.make_error_log_records(ValueError(), 'Error'))
|
||||
|
||||
assert {log.levelno for log in logs} == {module.logging.CRITICAL}
|
||||
|
||||
|
||||
def test_make_error_log_records_generates_logs_for_os_error():
|
||||
logs = tuple(module.make_error_log_records(OSError(), 'Error'))
|
||||
|
||||
assert {log.levelno for log in logs} == {module.logging.CRITICAL}
|
||||
|
||||
|
||||
def test_make_error_log_records_generates_nothing_for_other_error():
|
||||
logs = tuple(module.make_error_log_records(KeyError(), 'Error'))
|
||||
|
||||
assert logs == ()
|
||||
|
||||
|
||||
def test_collect_configuration_run_summary_logs_info_for_success():
|
||||
flexmock(module.hook).should_receive('execute_hook').never()
|
||||
flexmock(module).should_receive('run_configuration').and_return([])
|
||||
arguments = {}
|
||||
|
||||
|
@ -35,7 +65,18 @@ def test_collect_configuration_run_summary_logs_info_for_success():
|
|||
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
|
||||
)
|
||||
|
||||
assert all(log for log in logs if log.levelno == module.logging.INFO)
|
||||
assert {log.levelno for log in logs} == {module.logging.INFO}
|
||||
|
||||
|
||||
def test_collect_configuration_run_summary_executes_hooks_for_create():
|
||||
flexmock(module).should_receive('run_configuration').and_return([])
|
||||
arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
|
||||
|
||||
logs = tuple(
|
||||
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
|
||||
)
|
||||
|
||||
assert {log.levelno for log in logs} == {module.logging.INFO}
|
||||
|
||||
|
||||
def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
|
||||
|
@ -47,7 +88,7 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
|
|||
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
|
||||
)
|
||||
|
||||
assert all(log for log in logs if log.levelno == module.logging.INFO)
|
||||
assert {log.levelno for log in logs} == {module.logging.INFO}
|
||||
|
||||
|
||||
def test_collect_configuration_run_summary_logs_critical_for_extract_with_repository_error():
|
||||
|
@ -60,7 +101,30 @@ def test_collect_configuration_run_summary_logs_critical_for_extract_with_reposi
|
|||
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
|
||||
)
|
||||
|
||||
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
|
||||
assert {log.levelno for log in logs} == {module.logging.CRITICAL}
|
||||
|
||||
|
||||
def test_collect_configuration_run_summary_logs_critical_for_pre_hook_error():
|
||||
flexmock(module.hook).should_receive('execute_hook').and_raise(ValueError)
|
||||
arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
|
||||
|
||||
logs = tuple(
|
||||
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
|
||||
)
|
||||
|
||||
assert {log.levelno for log in logs} == {module.logging.CRITICAL}
|
||||
|
||||
|
||||
def test_collect_configuration_run_summary_logs_critical_for_post_hook_error():
|
||||
flexmock(module.hook).should_receive('execute_hook').and_return(None).and_raise(ValueError)
|
||||
flexmock(module).should_receive('run_configuration').and_return([])
|
||||
arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
|
||||
|
||||
logs = tuple(
|
||||
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
|
||||
)
|
||||
|
||||
assert {log.levelno for log in logs} == {module.logging.INFO, module.logging.CRITICAL}
|
||||
|
||||
|
||||
def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_and_repository_error():
|
||||
|
@ -73,7 +137,7 @@ def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_a
|
|||
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
|
||||
)
|
||||
|
||||
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
|
||||
assert {log.levelno for log in logs} == {module.logging.CRITICAL}
|
||||
|
||||
|
||||
def test_collect_configuration_run_summary_logs_info_for_success_with_list():
|
||||
|
@ -84,7 +148,7 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_list():
|
|||
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
|
||||
)
|
||||
|
||||
assert all(log for log in logs if log.levelno == module.logging.INFO)
|
||||
assert {log.levelno for log in logs} == {module.logging.INFO}
|
||||
|
||||
|
||||
def test_collect_configuration_run_summary_logs_critical_for_run_value_error():
|
||||
|
@ -96,7 +160,7 @@ def test_collect_configuration_run_summary_logs_critical_for_run_value_error():
|
|||
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
|
||||
)
|
||||
|
||||
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
|
||||
assert {log.levelno for log in logs} == {module.logging.CRITICAL}
|
||||
|
||||
|
||||
def test_collect_configuration_run_summary_logs_critical_including_output_for_run_process_error():
|
||||
|
@ -110,7 +174,7 @@ def test_collect_configuration_run_summary_logs_critical_including_output_for_ru
|
|||
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
|
||||
)
|
||||
|
||||
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
|
||||
assert {log.levelno for log in logs} == {module.logging.CRITICAL}
|
||||
assert any(log for log in logs if 'error output' in str(log))
|
||||
|
||||
|
||||
|
@ -134,4 +198,4 @@ def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
|
|||
|
||||
logs = tuple(module.collect_configuration_run_summary_logs({}, arguments=arguments))
|
||||
|
||||
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
|
||||
assert {log.levelno for log in logs} == {module.logging.CRITICAL}
|
||||
|
|
Loading…
Reference in a new issue