Monitor backups with Cronhub hook integration. Fix Healthchecks/Cronitor hooks to respect dry run.
This commit is contained in:
parent
ac777965d0
commit
17fda7281a
11 changed files with 146 additions and 16 deletions
5
NEWS
5
NEWS
|
@ -1,3 +1,8 @@
|
|||
1.4.8
|
||||
* Monitor backups with Cronhub hook integration. See the documentation for more information:
|
||||
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook
|
||||
* Fix Healthchecks/Cronitor hooks to skip actions when the borgmatic "--dry-run" flag is used.
|
||||
|
||||
1.4.7
|
||||
* #238: In documentation, clarify when Healthchecks/Cronitor hooks fire in relation to other hooks.
|
||||
* #239: Upgrade your borgmatic configuration to get new options and comments via
|
||||
|
|
|
@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list
|
|||
from borgmatic.borg import prune as borg_prune
|
||||
from borgmatic.commands.arguments import parse_arguments
|
||||
from borgmatic.config import checks, collect, convert, validate
|
||||
from borgmatic.hooks import command, cronitor, healthchecks, postgresql
|
||||
from borgmatic.hooks import command, cronhub, cronitor, healthchecks, postgresql
|
||||
from borgmatic.logger import configure_logging, should_do_markup
|
||||
from borgmatic.signals import configure_signals
|
||||
from borgmatic.verbosity import verbosity_to_log_level
|
||||
|
@ -59,6 +59,9 @@ def run_configuration(config_filename, config, arguments):
|
|||
cronitor.ping_cronitor(
|
||||
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run'
|
||||
)
|
||||
cronhub.ping_cronhub(
|
||||
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'start'
|
||||
)
|
||||
command.execute_hook(
|
||||
hooks.get('before_backup'),
|
||||
hooks.get('umask'),
|
||||
|
@ -114,6 +117,9 @@ def run_configuration(config_filename, config, arguments):
|
|||
cronitor.ping_cronitor(
|
||||
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete'
|
||||
)
|
||||
cronhub.ping_cronhub(
|
||||
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'finish'
|
||||
)
|
||||
except (OSError, CalledProcessError) as error:
|
||||
encountered_error = error
|
||||
yield from make_error_log_records(
|
||||
|
@ -138,6 +144,9 @@ def run_configuration(config_filename, config, arguments):
|
|||
cronitor.ping_cronitor(
|
||||
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail'
|
||||
)
|
||||
cronhub.ping_cronhub(
|
||||
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'fail'
|
||||
)
|
||||
except (OSError, CalledProcessError) as error:
|
||||
yield from make_error_log_records(
|
||||
'{}: Error running on-error hook'.format(config_filename), error
|
||||
|
|
|
@ -439,7 +439,8 @@ map:
|
|||
desc: |
|
||||
Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors.
|
||||
Create an account at https://healthchecks.io if you'd like to use this service.
|
||||
See http://localhost:8080/docs/how-to/monitor-your-backups/#healthchecks-hook
|
||||
See
|
||||
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
|
||||
for details.
|
||||
example:
|
||||
https://hc-ping.com/your-uuid-here
|
||||
|
@ -448,10 +449,19 @@ map:
|
|||
desc: |
|
||||
Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
|
||||
account at https://cronitor.io if you'd like to use this service. See
|
||||
http://localhost:8080/docs/how-to/monitor-your-backups/#cronitor-hook for
|
||||
details.
|
||||
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
|
||||
for details.
|
||||
example:
|
||||
https://cronitor.link/d3x0c1
|
||||
cronhub:
|
||||
type: str
|
||||
desc: |
|
||||
Cronhub ping URL to notify when a backup begins, ends, or errors. Create an
|
||||
account at https://cronhub.io if you'd like to use this service. See
|
||||
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook for
|
||||
details.
|
||||
example:
|
||||
https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
|
||||
before_everything:
|
||||
seq:
|
||||
- type: str
|
||||
|
|
26
borgmatic/hooks/cronhub.py
Normal file
26
borgmatic/hooks/cronhub.py
Normal file
|
@ -0,0 +1,26 @@
|
|||
import logging
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ping_cronhub(ping_url, config_filename, dry_run, state):
|
||||
'''
|
||||
Ping the given Cronhub URL, substituting in the state string. Use the given configuration
|
||||
filename in any log entries. If this is a dry run, then don't actually ping anything.
|
||||
'''
|
||||
if not ping_url:
|
||||
logger.debug('{}: No Cronhub hook set'.format(config_filename))
|
||||
return
|
||||
|
||||
dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
|
||||
formatted_state = '/{}/'.format(state)
|
||||
ping_url = ping_url.replace('/start/', formatted_state).replace('/ping/', formatted_state)
|
||||
|
||||
logger.info('{}: Pinging Cronhub {}{}'.format(config_filename, state, dry_run_label))
|
||||
logger.debug('{}: Using Cronhub ping URL {}'.format(config_filename, ping_url))
|
||||
|
||||
if not dry_run:
|
||||
logging.getLogger('urllib3').setLevel(logging.ERROR)
|
||||
requests.get(ping_url)
|
|
@ -20,5 +20,6 @@ def ping_cronitor(ping_url, config_filename, dry_run, append):
|
|||
logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label))
|
||||
logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
|
||||
|
||||
logging.getLogger('urllib3').setLevel(logging.ERROR)
|
||||
requests.get(ping_url)
|
||||
if not dry_run:
|
||||
logging.getLogger('urllib3').setLevel(logging.ERROR)
|
||||
requests.get(ping_url)
|
||||
|
|
|
@ -32,5 +32,6 @@ def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
|
|||
)
|
||||
logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
|
||||
|
||||
logging.getLogger('urllib3').setLevel(logging.ERROR)
|
||||
requests.get(ping_url)
|
||||
if not dry_run:
|
||||
logging.getLogger('urllib3').setLevel(logging.ERROR)
|
||||
requests.get(ping_url)
|
||||
|
|
|
@ -27,14 +27,15 @@ See [error
|
|||
hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
|
||||
below for how to configure this.
|
||||
4. **borgmatic monitoring hooks**: This feature integrates with monitoring
|
||||
services like [Healthchecks](https://healthchecks.io/) and
|
||||
[Cronitor](https://cronitor.io), and pings these services whenever borgmatic
|
||||
runs. That way, you'll receive an alert when something goes wrong or the
|
||||
service doesn't hear from borgmatic for a configured interval. See
|
||||
services like [Healthchecks](https://healthchecks.io/),
|
||||
[Cronitor](https://cronitor.io), and [Cronhub](https://cronhub.io), and pings
|
||||
these services whenever borgmatic runs. That way, you'll receive an alert when
|
||||
something goes wrong or the service doesn't hear from borgmatic for a
|
||||
configured interval. See
|
||||
[Healthchecks
|
||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook)
|
||||
and [Cronitor
|
||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook)
|
||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook), [Cronitor
|
||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook), and [Cronhub
|
||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook)
|
||||
below for how to configure this.
|
||||
3. **Third-party monitoring software**: You can use traditional monitoring
|
||||
software to consume borgmatic JSON output and track when the last
|
||||
|
@ -151,6 +152,37 @@ mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
|
|||
or it doesn't hear from borgmatic for a certain period of time.
|
||||
|
||||
|
||||
## Cronhub hook
|
||||
|
||||
[Cronhub](https://cronhub.io/) provides "instant alerts when any of your
|
||||
background jobs fail silently or run longer than expected", and borgmatic has
|
||||
built-in integration with it. Once you create a Cronhub account and monitor on
|
||||
their site, all you need to do is configure borgmatic with the unique "Ping
|
||||
URL" for your monitor. Here's an example:
|
||||
|
||||
|
||||
```yaml
|
||||
hooks:
|
||||
cronhub: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
|
||||
```
|
||||
|
||||
With this hook in place, borgmatic pings your Cronhub monitor when a backup
|
||||
begins, ends, or errors. Specifically, before the <a
|
||||
href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
|
||||
hooks</a> run, borgmatic lets Cronhub know that a backup has started. Then,
|
||||
if the backup completes successfully, borgmatic notifies Cronhub of the
|
||||
success after the `after_backup` hooks run. And if an error occurs during the
|
||||
backup, borgmatic notifies Cronhub after the `on_error` hooks run.
|
||||
|
||||
Note that even though you configure borgmatic with the "start" variant of the
|
||||
ping URL, borgmatic substitutes the correct state into the URL when pinging
|
||||
Cronhub ("start", "finish", or "fail").
|
||||
|
||||
You can configure Cronhub to notify you by a [variety of
|
||||
mechanisms](https://docs.cronhub.io/integrations.html) when backups fail
|
||||
or it doesn't hear from borgmatic for a certain period of time.
|
||||
|
||||
|
||||
## Scripting borgmatic
|
||||
|
||||
To consume the output of borgmatic in other software, you can include an
|
||||
|
|
2
setup.py
2
setup.py
|
@ -1,6 +1,6 @@
|
|||
from setuptools import find_packages, setup
|
||||
|
||||
VERSION = '1.4.7'
|
||||
VERSION = '1.4.8'
|
||||
|
||||
|
||||
setup(
|
||||
|
|
32
tests/unit/hooks/test_cronhub.py
Normal file
32
tests/unit/hooks/test_cronhub.py
Normal file
|
@ -0,0 +1,32 @@
|
|||
from flexmock import flexmock
|
||||
|
||||
from borgmatic.hooks import cronhub as module
|
||||
|
||||
|
||||
def test_ping_cronhub_hits_ping_url_with_start_state():
|
||||
ping_url = 'https://example.com/start/abcdef'
|
||||
state = 'bork'
|
||||
flexmock(module.requests).should_receive('get').with_args('https://example.com/bork/abcdef')
|
||||
|
||||
module.ping_cronhub(ping_url, 'config.yaml', dry_run=False, state=state)
|
||||
|
||||
|
||||
def test_ping_cronhub_hits_ping_url_with_ping_state():
|
||||
ping_url = 'https://example.com/ping/abcdef'
|
||||
state = 'bork'
|
||||
flexmock(module.requests).should_receive('get').with_args('https://example.com/bork/abcdef')
|
||||
|
||||
module.ping_cronhub(ping_url, 'config.yaml', dry_run=False, state=state)
|
||||
|
||||
|
||||
def test_ping_cronhub_without_ping_url_does_not_raise():
|
||||
flexmock(module.requests).should_receive('get').never()
|
||||
|
||||
module.ping_cronhub(ping_url=None, config_filename='config.yaml', dry_run=False, state='oops')
|
||||
|
||||
|
||||
def test_ping_cronhub_dry_run_does_not_hit_ping_url():
|
||||
ping_url = 'https://example.com'
|
||||
flexmock(module.requests).should_receive('get').never()
|
||||
|
||||
module.ping_cronhub(ping_url, 'config.yaml', dry_run=True, state='yay')
|
|
@ -15,3 +15,10 @@ def test_ping_cronitor_without_ping_url_does_not_raise():
|
|||
flexmock(module.requests).should_receive('get').never()
|
||||
|
||||
module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops')
|
||||
|
||||
|
||||
def test_ping_cronitor_dry_run_does_not_hit_ping_url():
|
||||
ping_url = 'https://example.com'
|
||||
flexmock(module.requests).should_receive('get').never()
|
||||
|
||||
module.ping_cronitor(ping_url, 'config.yaml', dry_run=True, append='yay')
|
||||
|
|
|
@ -31,3 +31,10 @@ def test_ping_healthchecks_hits_ping_url_with_append():
|
|||
flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
|
||||
|
||||
module.ping_healthchecks(ping_url, 'config.yaml', dry_run=False, append=append)
|
||||
|
||||
|
||||
def test_ping_healthchecks_dry_run_does_not_hit_ping_url():
|
||||
ping_url = 'https://example.com'
|
||||
flexmock(module.requests).should_receive('get').never()
|
||||
|
||||
module.ping_healthchecks(ping_url, 'config.yaml', dry_run=True)
|
||||
|
|
Loading…
Reference in a new issue