Monitor backups with Cronhub hook integration. Fix Healthchecks/Cronitor hooks to respect dry run.

This commit is contained in:
Dan Helfman 2019-11-07 10:08:44 -08:00
parent ac777965d0
commit 17fda7281a
11 changed files with 146 additions and 16 deletions

5
NEWS
View file

@ -1,3 +1,8 @@
1.4.8
* Monitor backups with Cronhub hook integration. See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook
* Fix Healthchecks/Cronitor hooks to skip actions when the borgmatic "--dry-run" flag is used.
1.4.7
* #238: In documentation, clarify when Healthchecks/Cronitor hooks fire in relation to other hooks.
* #239: Upgrade your borgmatic configuration to get new options and comments via

View file

@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list
from borgmatic.borg import prune as borg_prune
from borgmatic.commands.arguments import parse_arguments
from borgmatic.config import checks, collect, convert, validate
from borgmatic.hooks import command, cronitor, healthchecks, postgresql
from borgmatic.hooks import command, cronhub, cronitor, healthchecks, postgresql
from borgmatic.logger import configure_logging, should_do_markup
from borgmatic.signals import configure_signals
from borgmatic.verbosity import verbosity_to_log_level
@ -59,6 +59,9 @@ def run_configuration(config_filename, config, arguments):
cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run'
)
cronhub.ping_cronhub(
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'start'
)
command.execute_hook(
hooks.get('before_backup'),
hooks.get('umask'),
@ -114,6 +117,9 @@ def run_configuration(config_filename, config, arguments):
cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete'
)
cronhub.ping_cronhub(
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'finish'
)
except (OSError, CalledProcessError) as error:
encountered_error = error
yield from make_error_log_records(
@ -138,6 +144,9 @@ def run_configuration(config_filename, config, arguments):
cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail'
)
cronhub.ping_cronhub(
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'fail'
)
except (OSError, CalledProcessError) as error:
yield from make_error_log_records(
'{}: Error running on-error hook'.format(config_filename), error

View file

@ -439,7 +439,8 @@ map:
desc: |
Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors.
Create an account at https://healthchecks.io if you'd like to use this service.
See http://localhost:8080/docs/how-to/monitor-your-backups/#healthchecks-hook
See
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
for details.
example:
https://hc-ping.com/your-uuid-here
@ -448,10 +449,19 @@ map:
desc: |
Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
account at https://cronitor.io if you'd like to use this service. See
http://localhost:8080/docs/how-to/monitor-your-backups/#cronitor-hook for
details.
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
for details.
example:
https://cronitor.link/d3x0c1
cronhub:
type: str
desc: |
Cronhub ping URL to notify when a backup begins, ends, or errors. Create an
account at https://cronhub.io if you'd like to use this service. See
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook for
details.
example:
https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
before_everything:
seq:
- type: str

View file

@ -0,0 +1,26 @@
import logging
import requests
logger = logging.getLogger(__name__)
def ping_cronhub(ping_url, config_filename, dry_run, state):
'''
Ping the given Cronhub URL, substituting in the state string. Use the given configuration
filename in any log entries. If this is a dry run, then don't actually ping anything.
'''
if not ping_url:
logger.debug('{}: No Cronhub hook set'.format(config_filename))
return
dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
formatted_state = '/{}/'.format(state)
ping_url = ping_url.replace('/start/', formatted_state).replace('/ping/', formatted_state)
logger.info('{}: Pinging Cronhub {}{}'.format(config_filename, state, dry_run_label))
logger.debug('{}: Using Cronhub ping URL {}'.format(config_filename, ping_url))
if not dry_run:
logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url)

View file

@ -20,5 +20,6 @@ def ping_cronitor(ping_url, config_filename, dry_run, append):
logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label))
logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url)
if not dry_run:
logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url)

View file

@ -32,5 +32,6 @@ def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
)
logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url)
if not dry_run:
logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url)

View file

@ -27,14 +27,15 @@ See [error
hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
below for how to configure this.
4. **borgmatic monitoring hooks**: This feature integrates with monitoring
services like [Healthchecks](https://healthchecks.io/) and
[Cronitor](https://cronitor.io), and pings these services whenever borgmatic
runs. That way, you'll receive an alert when something goes wrong or the
service doesn't hear from borgmatic for a configured interval. See
services like [Healthchecks](https://healthchecks.io/),
[Cronitor](https://cronitor.io), and [Cronhub](https://cronhub.io), and pings
these services whenever borgmatic runs. That way, you'll receive an alert when
something goes wrong or the service doesn't hear from borgmatic for a
configured interval. See
[Healthchecks
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook)
and [Cronitor
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook)
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook), [Cronitor
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook), and [Cronhub
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook)
below for how to configure this.
3. **Third-party monitoring software**: You can use traditional monitoring
software to consume borgmatic JSON output and track when the last
@ -151,6 +152,37 @@ mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
or it doesn't hear from borgmatic for a certain period of time.
## Cronhub hook
[Cronhub](https://cronhub.io/) provides "instant alerts when any of your
background jobs fail silently or run longer than expected", and borgmatic has
built-in integration with it. Once you create a Cronhub account and monitor on
their site, all you need to do is configure borgmatic with the unique "Ping
URL" for your monitor. Here's an example:
```yaml
hooks:
cronhub: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
```
With this hook in place, borgmatic pings your Cronhub monitor when a backup
begins, ends, or errors. Specifically, before the <a
href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
hooks</a> run, borgmatic lets Cronhub know that a backup has started. Then,
if the backup completes successfully, borgmatic notifies Cronhub of the
success after the `after_backup` hooks run. And if an error occurs during the
backup, borgmatic notifies Cronhub after the `on_error` hooks run.
Note that even though you configure borgmatic with the "start" variant of the
ping URL, borgmatic substitutes the correct state into the URL when pinging
Cronhub ("start", "finish", or "fail").
You can configure Cronhub to notify you by a [variety of
mechanisms](https://docs.cronhub.io/integrations.html) when backups fail
or it doesn't hear from borgmatic for a certain period of time.
## Scripting borgmatic
To consume the output of borgmatic in other software, you can include an

View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup
VERSION = '1.4.7'
VERSION = '1.4.8'
setup(

View file

@ -0,0 +1,32 @@
from flexmock import flexmock
from borgmatic.hooks import cronhub as module
def test_ping_cronhub_hits_ping_url_with_start_state():
ping_url = 'https://example.com/start/abcdef'
state = 'bork'
flexmock(module.requests).should_receive('get').with_args('https://example.com/bork/abcdef')
module.ping_cronhub(ping_url, 'config.yaml', dry_run=False, state=state)
def test_ping_cronhub_hits_ping_url_with_ping_state():
ping_url = 'https://example.com/ping/abcdef'
state = 'bork'
flexmock(module.requests).should_receive('get').with_args('https://example.com/bork/abcdef')
module.ping_cronhub(ping_url, 'config.yaml', dry_run=False, state=state)
def test_ping_cronhub_without_ping_url_does_not_raise():
flexmock(module.requests).should_receive('get').never()
module.ping_cronhub(ping_url=None, config_filename='config.yaml', dry_run=False, state='oops')
def test_ping_cronhub_dry_run_does_not_hit_ping_url():
ping_url = 'https://example.com'
flexmock(module.requests).should_receive('get').never()
module.ping_cronhub(ping_url, 'config.yaml', dry_run=True, state='yay')

View file

@ -15,3 +15,10 @@ def test_ping_cronitor_without_ping_url_does_not_raise():
flexmock(module.requests).should_receive('get').never()
module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops')
def test_ping_cronitor_dry_run_does_not_hit_ping_url():
ping_url = 'https://example.com'
flexmock(module.requests).should_receive('get').never()
module.ping_cronitor(ping_url, 'config.yaml', dry_run=True, append='yay')

View file

@ -31,3 +31,10 @@ def test_ping_healthchecks_hits_ping_url_with_append():
flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
module.ping_healthchecks(ping_url, 'config.yaml', dry_run=False, append=append)
def test_ping_healthchecks_dry_run_does_not_hit_ping_url():
ping_url = 'https://example.com'
flexmock(module.requests).should_receive('get').never()
module.ping_healthchecks(ping_url, 'config.yaml', dry_run=True)