Add support for healthchecks "log" feature (#628).

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/645
This commit is contained in:
Dan Helfman 2023-03-07 22:21:30 +00:00
commit 332f7c4bb6
11 changed files with 110 additions and 14 deletions

View file

@ -152,6 +152,25 @@ def run_configuration(config_filename, config, arguments):
encountered_error = error encountered_error = error
error_repository = repository_path error_repository = repository_path
try:
if using_primary_action:
# send logs irrespective of error
dispatch.call_hooks(
'ping_monitor',
hooks,
config_filename,
monitor.MONITOR_HOOK_NAMES,
monitor.State.LOG,
monitoring_log_level,
global_arguments.dry_run,
)
except (OSError, CalledProcessError) as error:
if command.considered_soft_failure(config_filename, error):
return
encountered_error = error
yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error)
if not encountered_error: if not encountered_error:
try: try:
if using_primary_action: if using_primary_action:

View file

@ -951,9 +951,9 @@ properties:
name: name:
type: string type: string
description: | description: |
This is used to tag the database dump file This is used to tag the database dump file
with a name. It is not the path to the database with a name. It is not the path to the database
file itself. The name "all" has no special file itself. The name "all" has no special
meaning for SQLite databases. meaning for SQLite databases.
example: users example: users
path: path:
@ -1168,7 +1168,7 @@ properties:
type: string type: string
description: | description: |
Healthchecks ping URL or UUID to notify when a Healthchecks ping URL or UUID to notify when a
backup begins, ends, or errors. backup begins, ends, errors or just to send logs.
example: https://hc-ping.com/your-uuid-here example: https://hc-ping.com/your-uuid-here
verify_tls: verify_tls:
type: boolean type: boolean
@ -1199,6 +1199,7 @@ properties:
- start - start
- finish - finish
- fail - fail
- log
uniqueItems: true uniqueItems: true
description: | description: |
List of one or more monitoring states to ping for: List of one or more monitoring states to ping for:

View file

@ -27,6 +27,12 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
Ping the configured Cronhub URL, modified with the monitor.State. Use the given configuration Ping the configured Cronhub URL, modified with the monitor.State. Use the given configuration
filename in any log entries. If this is a dry run, then don't actually ping anything. filename in any log entries. If this is a dry run, then don't actually ping anything.
''' '''
if state not in MONITOR_STATE_TO_CRONHUB:
logger.debug(
f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in Cronhub hook'
)
return
dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
formatted_state = '/{}/'.format(MONITOR_STATE_TO_CRONHUB[state]) formatted_state = '/{}/'.format(MONITOR_STATE_TO_CRONHUB[state])
ping_url = ( ping_url = (

View file

@ -27,6 +27,12 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
Ping the configured Cronitor URL, modified with the monitor.State. Use the given configuration Ping the configured Cronitor URL, modified with the monitor.State. Use the given configuration
filename in any log entries. If this is a dry run, then don't actually ping anything. filename in any log entries. If this is a dry run, then don't actually ping anything.
''' '''
if state not in MONITOR_STATE_TO_CRONITOR:
logger.debug(
f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in Cronitor hook'
)
return
dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
ping_url = '{}/{}'.format(hook_config['ping_url'], MONITOR_STATE_TO_CRONITOR[state]) ping_url = '{}/{}'.format(hook_config['ping_url'], MONITOR_STATE_TO_CRONITOR[state])

View file

@ -10,6 +10,7 @@ MONITOR_STATE_TO_HEALTHCHECKS = {
monitor.State.START: 'start', monitor.State.START: 'start',
monitor.State.FINISH: None, # Healthchecks doesn't append to the URL for the finished state. monitor.State.FINISH: None, # Healthchecks doesn't append to the URL for the finished state.
monitor.State.FAIL: 'fail', monitor.State.FAIL: 'fail',
monitor.State.LOG: 'log',
} }
PAYLOAD_TRUNCATION_INDICATOR = '...\n' PAYLOAD_TRUNCATION_INDICATOR = '...\n'
@ -117,7 +118,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
) )
logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url)) logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
if state in (monitor.State.FINISH, monitor.State.FAIL): if state in (monitor.State.FINISH, monitor.State.FAIL, monitor.State.LOG):
payload = format_buffered_logs_for_payload() payload = format_buffered_logs_for_payload()
else: else:
payload = '' payload = ''

View file

@ -7,3 +7,4 @@ class State(Enum):
START = 1 START = 1
FINISH = 2 FINISH = 2
FAIL = 3 FAIL = 3
LOG = 4

View file

@ -6,12 +6,6 @@ from borgmatic.hooks import monitor
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
MONITOR_STATE_TO_NTFY = {
monitor.State.START: None,
monitor.State.FINISH: None,
monitor.State.FAIL: None,
}
def initialize_monitor( def initialize_monitor(
ping_url, config_filename, monitoring_log_level, dry_run ping_url, config_filename, monitoring_log_level, dry_run

View file

@ -40,7 +40,7 @@ def test_run_configuration_logs_monitor_start_error():
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.dispatch).should_receive('call_hooks').and_raise(OSError).and_return( flexmock(module.dispatch).should_receive('call_hooks').and_raise(OSError).and_return(
None None
).and_return(None) ).and_return(None).and_return(None)
expected_results = [flexmock()] expected_results = [flexmock()]
flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('log_error_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').never() flexmock(module).should_receive('run_actions').never()
@ -99,7 +99,7 @@ def test_run_configuration_bails_for_actions_soft_failure():
assert results == [] assert results == []
def test_run_configuration_logs_monitor_finish_error(): def test_run_configuration_logs_monitor_log_error():
flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO)
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
@ -116,13 +116,48 @@ def test_run_configuration_logs_monitor_finish_error():
assert results == expected_results assert results == expected_results
def test_run_configuration_bails_for_monitor_log_soft_failure():
flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO)
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
None
).and_raise(error)
flexmock(module).should_receive('log_error_records').never()
flexmock(module).should_receive('run_actions').and_return([])
flexmock(module.command).should_receive('considered_soft_failure').and_return(True)
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == []
def test_run_configuration_logs_monitor_finish_error():
flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO)
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
None
).and_return(None).and_raise(OSError)
expected_results = [flexmock()]
flexmock(module).should_receive('log_error_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').and_return([])
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_run_configuration_bails_for_monitor_finish_soft_failure(): def test_run_configuration_bails_for_monitor_finish_soft_failure():
flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO)
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
None None
).and_raise(error) ).and_raise(None).and_raise(error)
flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('log_error_records').never()
flexmock(module).should_receive('run_actions').and_return([]) flexmock(module).should_receive('run_actions').and_return([])
flexmock(module.command).should_receive('considered_soft_failure').and_return(True) flexmock(module.command).should_receive('considered_soft_failure').and_return(True)

View file

@ -102,3 +102,11 @@ def test_ping_monitor_with_other_error_logs_warning():
monitoring_log_level=1, monitoring_log_level=1,
dry_run=False, dry_run=False,
) )
def test_ping_monitor_with_unsupported_monitoring_state():
hook_config = {'ping_url': 'https://example.com'}
flexmock(module.requests).should_receive('get').never()
module.ping_monitor(
hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False,
)

View file

@ -87,3 +87,11 @@ def test_ping_monitor_with_other_error_logs_warning():
monitoring_log_level=1, monitoring_log_level=1,
dry_run=False, dry_run=False,
) )
def test_ping_monitor_with_unsupported_monitoring_state():
hook_config = {'ping_url': 'https://example.com'}
flexmock(module.requests).should_receive('get').never()
module.ping_monitor(
hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False,
)

View file

@ -184,6 +184,23 @@ def test_ping_monitor_hits_ping_url_for_fail_state():
) )
def test_ping_monitor_hits_ping_url_for_log_state():
hook_config = {'ping_url': 'https://example.com'}
payload = 'data'
flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload)
flexmock(module.requests).should_receive('post').with_args(
'https://example.com/log', data=payload.encode('utf'), verify=True
).and_return(flexmock(ok=True))
module.ping_monitor(
hook_config,
'config.yaml',
state=module.monitor.State.LOG,
monitoring_log_level=1,
dry_run=False,
)
def test_ping_monitor_with_ping_uuid_hits_corresponding_url(): def test_ping_monitor_with_ping_uuid_hits_corresponding_url():
hook_config = {'ping_url': 'abcd-efgh-ijkl-mnop'} hook_config = {'ping_url': 'abcd-efgh-ijkl-mnop'}
payload = 'data' payload = 'data'