Monitor backups with Cronhub hook integration. Fix Healthchecks/Cronitor hooks to respect dry run.
This commit is contained in:
parent
ac777965d0
commit
17fda7281a
11 changed files with 146 additions and 16 deletions
5
NEWS
5
NEWS
|
@ -1,3 +1,8 @@
|
||||||
|
1.4.8
|
||||||
|
* Monitor backups with Cronhub hook integration. See the documentation for more information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook
|
||||||
|
* Fix Healthchecks/Cronitor hooks to skip actions when the borgmatic "--dry-run" flag is used.
|
||||||
|
|
||||||
1.4.7
|
1.4.7
|
||||||
* #238: In documentation, clarify when Healthchecks/Cronitor hooks fire in relation to other hooks.
|
* #238: In documentation, clarify when Healthchecks/Cronitor hooks fire in relation to other hooks.
|
||||||
* #239: Upgrade your borgmatic configuration to get new options and comments via
|
* #239: Upgrade your borgmatic configuration to get new options and comments via
|
||||||
|
|
|
@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list
|
||||||
from borgmatic.borg import prune as borg_prune
|
from borgmatic.borg import prune as borg_prune
|
||||||
from borgmatic.commands.arguments import parse_arguments
|
from borgmatic.commands.arguments import parse_arguments
|
||||||
from borgmatic.config import checks, collect, convert, validate
|
from borgmatic.config import checks, collect, convert, validate
|
||||||
from borgmatic.hooks import command, cronitor, healthchecks, postgresql
|
from borgmatic.hooks import command, cronhub, cronitor, healthchecks, postgresql
|
||||||
from borgmatic.logger import configure_logging, should_do_markup
|
from borgmatic.logger import configure_logging, should_do_markup
|
||||||
from borgmatic.signals import configure_signals
|
from borgmatic.signals import configure_signals
|
||||||
from borgmatic.verbosity import verbosity_to_log_level
|
from borgmatic.verbosity import verbosity_to_log_level
|
||||||
|
@ -59,6 +59,9 @@ def run_configuration(config_filename, config, arguments):
|
||||||
cronitor.ping_cronitor(
|
cronitor.ping_cronitor(
|
||||||
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run'
|
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run'
|
||||||
)
|
)
|
||||||
|
cronhub.ping_cronhub(
|
||||||
|
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'start'
|
||||||
|
)
|
||||||
command.execute_hook(
|
command.execute_hook(
|
||||||
hooks.get('before_backup'),
|
hooks.get('before_backup'),
|
||||||
hooks.get('umask'),
|
hooks.get('umask'),
|
||||||
|
@ -114,6 +117,9 @@ def run_configuration(config_filename, config, arguments):
|
||||||
cronitor.ping_cronitor(
|
cronitor.ping_cronitor(
|
||||||
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete'
|
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete'
|
||||||
)
|
)
|
||||||
|
cronhub.ping_cronhub(
|
||||||
|
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'finish'
|
||||||
|
)
|
||||||
except (OSError, CalledProcessError) as error:
|
except (OSError, CalledProcessError) as error:
|
||||||
encountered_error = error
|
encountered_error = error
|
||||||
yield from make_error_log_records(
|
yield from make_error_log_records(
|
||||||
|
@ -138,6 +144,9 @@ def run_configuration(config_filename, config, arguments):
|
||||||
cronitor.ping_cronitor(
|
cronitor.ping_cronitor(
|
||||||
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail'
|
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail'
|
||||||
)
|
)
|
||||||
|
cronhub.ping_cronhub(
|
||||||
|
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'fail'
|
||||||
|
)
|
||||||
except (OSError, CalledProcessError) as error:
|
except (OSError, CalledProcessError) as error:
|
||||||
yield from make_error_log_records(
|
yield from make_error_log_records(
|
||||||
'{}: Error running on-error hook'.format(config_filename), error
|
'{}: Error running on-error hook'.format(config_filename), error
|
||||||
|
|
|
@ -439,7 +439,8 @@ map:
|
||||||
desc: |
|
desc: |
|
||||||
Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors.
|
Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors.
|
||||||
Create an account at https://healthchecks.io if you'd like to use this service.
|
Create an account at https://healthchecks.io if you'd like to use this service.
|
||||||
See http://localhost:8080/docs/how-to/monitor-your-backups/#healthchecks-hook
|
See
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
|
||||||
for details.
|
for details.
|
||||||
example:
|
example:
|
||||||
https://hc-ping.com/your-uuid-here
|
https://hc-ping.com/your-uuid-here
|
||||||
|
@ -448,10 +449,19 @@ map:
|
||||||
desc: |
|
desc: |
|
||||||
Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
|
Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
|
||||||
account at https://cronitor.io if you'd like to use this service. See
|
account at https://cronitor.io if you'd like to use this service. See
|
||||||
http://localhost:8080/docs/how-to/monitor-your-backups/#cronitor-hook for
|
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
|
||||||
details.
|
for details.
|
||||||
example:
|
example:
|
||||||
https://cronitor.link/d3x0c1
|
https://cronitor.link/d3x0c1
|
||||||
|
cronhub:
|
||||||
|
type: str
|
||||||
|
desc: |
|
||||||
|
Cronhub ping URL to notify when a backup begins, ends, or errors. Create an
|
||||||
|
account at https://cronhub.io if you'd like to use this service. See
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook for
|
||||||
|
details.
|
||||||
|
example:
|
||||||
|
https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
|
||||||
before_everything:
|
before_everything:
|
||||||
seq:
|
seq:
|
||||||
- type: str
|
- type: str
|
||||||
|
|
26
borgmatic/hooks/cronhub.py
Normal file
26
borgmatic/hooks/cronhub.py
Normal file
|
@ -0,0 +1,26 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def ping_cronhub(ping_url, config_filename, dry_run, state):
|
||||||
|
'''
|
||||||
|
Ping the given Cronhub URL, substituting in the state string. Use the given configuration
|
||||||
|
filename in any log entries. If this is a dry run, then don't actually ping anything.
|
||||||
|
'''
|
||||||
|
if not ping_url:
|
||||||
|
logger.debug('{}: No Cronhub hook set'.format(config_filename))
|
||||||
|
return
|
||||||
|
|
||||||
|
dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
|
||||||
|
formatted_state = '/{}/'.format(state)
|
||||||
|
ping_url = ping_url.replace('/start/', formatted_state).replace('/ping/', formatted_state)
|
||||||
|
|
||||||
|
logger.info('{}: Pinging Cronhub {}{}'.format(config_filename, state, dry_run_label))
|
||||||
|
logger.debug('{}: Using Cronhub ping URL {}'.format(config_filename, ping_url))
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
logging.getLogger('urllib3').setLevel(logging.ERROR)
|
||||||
|
requests.get(ping_url)
|
|
@ -20,5 +20,6 @@ def ping_cronitor(ping_url, config_filename, dry_run, append):
|
||||||
logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label))
|
logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label))
|
||||||
logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
|
logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
|
||||||
|
|
||||||
logging.getLogger('urllib3').setLevel(logging.ERROR)
|
if not dry_run:
|
||||||
requests.get(ping_url)
|
logging.getLogger('urllib3').setLevel(logging.ERROR)
|
||||||
|
requests.get(ping_url)
|
||||||
|
|
|
@ -32,5 +32,6 @@ def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
|
||||||
)
|
)
|
||||||
logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
|
logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
|
||||||
|
|
||||||
logging.getLogger('urllib3').setLevel(logging.ERROR)
|
if not dry_run:
|
||||||
requests.get(ping_url)
|
logging.getLogger('urllib3').setLevel(logging.ERROR)
|
||||||
|
requests.get(ping_url)
|
||||||
|
|
|
@ -27,14 +27,15 @@ See [error
|
||||||
hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
|
hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
|
||||||
below for how to configure this.
|
below for how to configure this.
|
||||||
4. **borgmatic monitoring hooks**: This feature integrates with monitoring
|
4. **borgmatic monitoring hooks**: This feature integrates with monitoring
|
||||||
services like [Healthchecks](https://healthchecks.io/) and
|
services like [Healthchecks](https://healthchecks.io/),
|
||||||
[Cronitor](https://cronitor.io), and pings these services whenever borgmatic
|
[Cronitor](https://cronitor.io), and [Cronhub](https://cronhub.io), and pings
|
||||||
runs. That way, you'll receive an alert when something goes wrong or the
|
these services whenever borgmatic runs. That way, you'll receive an alert when
|
||||||
service doesn't hear from borgmatic for a configured interval. See
|
something goes wrong or the service doesn't hear from borgmatic for a
|
||||||
|
configured interval. See
|
||||||
[Healthchecks
|
[Healthchecks
|
||||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook)
|
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook), [Cronitor
|
||||||
and [Cronitor
|
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook), and [Cronhub
|
||||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook)
|
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook)
|
||||||
below for how to configure this.
|
below for how to configure this.
|
||||||
3. **Third-party monitoring software**: You can use traditional monitoring
|
3. **Third-party monitoring software**: You can use traditional monitoring
|
||||||
software to consume borgmatic JSON output and track when the last
|
software to consume borgmatic JSON output and track when the last
|
||||||
|
@ -151,6 +152,37 @@ mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
|
||||||
or it doesn't hear from borgmatic for a certain period of time.
|
or it doesn't hear from borgmatic for a certain period of time.
|
||||||
|
|
||||||
|
|
||||||
|
## Cronhub hook
|
||||||
|
|
||||||
|
[Cronhub](https://cronhub.io/) provides "instant alerts when any of your
|
||||||
|
background jobs fail silently or run longer than expected", and borgmatic has
|
||||||
|
built-in integration with it. Once you create a Cronhub account and monitor on
|
||||||
|
their site, all you need to do is configure borgmatic with the unique "Ping
|
||||||
|
URL" for your monitor. Here's an example:
|
||||||
|
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
hooks:
|
||||||
|
cronhub: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
|
||||||
|
```
|
||||||
|
|
||||||
|
With this hook in place, borgmatic pings your Cronhub monitor when a backup
|
||||||
|
begins, ends, or errors. Specifically, before the <a
|
||||||
|
href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
|
||||||
|
hooks</a> run, borgmatic lets Cronhub know that a backup has started. Then,
|
||||||
|
if the backup completes successfully, borgmatic notifies Cronhub of the
|
||||||
|
success after the `after_backup` hooks run. And if an error occurs during the
|
||||||
|
backup, borgmatic notifies Cronhub after the `on_error` hooks run.
|
||||||
|
|
||||||
|
Note that even though you configure borgmatic with the "start" variant of the
|
||||||
|
ping URL, borgmatic substitutes the correct state into the URL when pinging
|
||||||
|
Cronhub ("start", "finish", or "fail").
|
||||||
|
|
||||||
|
You can configure Cronhub to notify you by a [variety of
|
||||||
|
mechanisms](https://docs.cronhub.io/integrations.html) when backups fail
|
||||||
|
or it doesn't hear from borgmatic for a certain period of time.
|
||||||
|
|
||||||
|
|
||||||
## Scripting borgmatic
|
## Scripting borgmatic
|
||||||
|
|
||||||
To consume the output of borgmatic in other software, you can include an
|
To consume the output of borgmatic in other software, you can include an
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -1,6 +1,6 @@
|
||||||
from setuptools import find_packages, setup
|
from setuptools import find_packages, setup
|
||||||
|
|
||||||
VERSION = '1.4.7'
|
VERSION = '1.4.8'
|
||||||
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
|
|
32
tests/unit/hooks/test_cronhub.py
Normal file
32
tests/unit/hooks/test_cronhub.py
Normal file
|
@ -0,0 +1,32 @@
|
||||||
|
from flexmock import flexmock
|
||||||
|
|
||||||
|
from borgmatic.hooks import cronhub as module
|
||||||
|
|
||||||
|
|
||||||
|
def test_ping_cronhub_hits_ping_url_with_start_state():
|
||||||
|
ping_url = 'https://example.com/start/abcdef'
|
||||||
|
state = 'bork'
|
||||||
|
flexmock(module.requests).should_receive('get').with_args('https://example.com/bork/abcdef')
|
||||||
|
|
||||||
|
module.ping_cronhub(ping_url, 'config.yaml', dry_run=False, state=state)
|
||||||
|
|
||||||
|
|
||||||
|
def test_ping_cronhub_hits_ping_url_with_ping_state():
|
||||||
|
ping_url = 'https://example.com/ping/abcdef'
|
||||||
|
state = 'bork'
|
||||||
|
flexmock(module.requests).should_receive('get').with_args('https://example.com/bork/abcdef')
|
||||||
|
|
||||||
|
module.ping_cronhub(ping_url, 'config.yaml', dry_run=False, state=state)
|
||||||
|
|
||||||
|
|
||||||
|
def test_ping_cronhub_without_ping_url_does_not_raise():
|
||||||
|
flexmock(module.requests).should_receive('get').never()
|
||||||
|
|
||||||
|
module.ping_cronhub(ping_url=None, config_filename='config.yaml', dry_run=False, state='oops')
|
||||||
|
|
||||||
|
|
||||||
|
def test_ping_cronhub_dry_run_does_not_hit_ping_url():
|
||||||
|
ping_url = 'https://example.com'
|
||||||
|
flexmock(module.requests).should_receive('get').never()
|
||||||
|
|
||||||
|
module.ping_cronhub(ping_url, 'config.yaml', dry_run=True, state='yay')
|
|
@ -15,3 +15,10 @@ def test_ping_cronitor_without_ping_url_does_not_raise():
|
||||||
flexmock(module.requests).should_receive('get').never()
|
flexmock(module.requests).should_receive('get').never()
|
||||||
|
|
||||||
module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops')
|
module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops')
|
||||||
|
|
||||||
|
|
||||||
|
def test_ping_cronitor_dry_run_does_not_hit_ping_url():
|
||||||
|
ping_url = 'https://example.com'
|
||||||
|
flexmock(module.requests).should_receive('get').never()
|
||||||
|
|
||||||
|
module.ping_cronitor(ping_url, 'config.yaml', dry_run=True, append='yay')
|
||||||
|
|
|
@ -31,3 +31,10 @@ def test_ping_healthchecks_hits_ping_url_with_append():
|
||||||
flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
|
flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
|
||||||
|
|
||||||
module.ping_healthchecks(ping_url, 'config.yaml', dry_run=False, append=append)
|
module.ping_healthchecks(ping_url, 'config.yaml', dry_run=False, append=append)
|
||||||
|
|
||||||
|
|
||||||
|
def test_ping_healthchecks_dry_run_does_not_hit_ping_url():
|
||||||
|
ping_url = 'https://example.com'
|
||||||
|
flexmock(module.requests).should_receive('get').never()
|
||||||
|
|
||||||
|
module.ping_healthchecks(ping_url, 'config.yaml', dry_run=True)
|
||||||
|
|
Loading…
Reference in a new issue