Add documentation for Grafana Loki hook (#743).
This commit is contained in:
parent
fa9a061033
commit
32019ea8f3
7 changed files with 154 additions and 83 deletions
3
NEWS
3
NEWS
|
@ -6,6 +6,9 @@
|
||||||
only restorable with a "mysql_databases:" configuration.
|
only restorable with a "mysql_databases:" configuration.
|
||||||
* #738: Fix for potential data loss (data not getting restored) in which the database "restore"
|
* #738: Fix for potential data loss (data not getting restored) in which the database "restore"
|
||||||
action didn't actually restore anything and indicated success anyway.
|
action didn't actually restore anything and indicated success anyway.
|
||||||
|
* #743: Add a monitoring hook for sending backup status and logs to to Grafana Loki. See the
|
||||||
|
documentation for more information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook
|
||||||
* Remove the deprecated use of the MongoDB hook's "--db" flag for database restoration.
|
* Remove the deprecated use of the MongoDB hook's "--db" flag for database restoration.
|
||||||
* Add source code reference documentation for getting oriented with the borgmatic code as a
|
* Add source code reference documentation for getting oriented with the borgmatic code as a
|
||||||
developer: https://torsion.org/borgmatic/docs/reference/source-code/
|
developer: https://torsion.org/borgmatic/docs/reference/source-code/
|
||||||
|
|
|
@ -67,6 +67,7 @@ class Loki_log_buffer:
|
||||||
request_body = self.to_request()
|
request_body = self.to_request()
|
||||||
self.root['streams'][0]['values'] = []
|
self.root['streams'][0]['values'] = []
|
||||||
request_header = {'Content-Type': 'application/json'}
|
request_header = {'Content-Type': 'application/json'}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = requests.post(self.url, headers=request_header, data=request_body, timeout=5)
|
result = requests.post(self.url, headers=request_header, data=request_body, timeout=5)
|
||||||
result.raise_for_status()
|
result.raise_for_status()
|
||||||
|
@ -100,6 +101,7 @@ class Loki_log_handler(logging.Handler):
|
||||||
Add an arbitrary string as a log entry to the stream.
|
Add an arbitrary string as a log entry to the stream.
|
||||||
'''
|
'''
|
||||||
self.buffer.add_value(msg)
|
self.buffer.add_value(msg)
|
||||||
|
|
||||||
if len(self.buffer) > MAX_BUFFER_LINES:
|
if len(self.buffer) > MAX_BUFFER_LINES:
|
||||||
self.buffer.flush()
|
self.buffer.flush()
|
||||||
|
|
||||||
|
@ -116,6 +118,7 @@ def initialize_monitor(hook_config, config, config_filename, monitoring_log_leve
|
||||||
'''
|
'''
|
||||||
url = hook_config.get('url')
|
url = hook_config.get('url')
|
||||||
loki = Loki_log_handler(url, dry_run)
|
loki = Loki_log_handler(url, dry_run)
|
||||||
|
|
||||||
for key, value in hook_config.get('labels').items():
|
for key, value in hook_config.get('labels').items():
|
||||||
if value == '__hostname':
|
if value == '__hostname':
|
||||||
loki.add_label(key, platform.node())
|
loki.add_label(key, platform.node())
|
||||||
|
@ -125,6 +128,7 @@ def initialize_monitor(hook_config, config, config_filename, monitoring_log_leve
|
||||||
loki.add_label(key, config_filename)
|
loki.add_label(key, config_filename)
|
||||||
else:
|
else:
|
||||||
loki.add_label(key, value)
|
loki.add_label(key, value)
|
||||||
|
|
||||||
logging.getLogger().addHandler(loki)
|
logging.getLogger().addHandler(loki)
|
||||||
|
|
||||||
|
|
||||||
|
@ -143,6 +147,7 @@ def destroy_monitor(hook_config, config, config_filename, monitoring_log_level,
|
||||||
Remove the monitor handler that was added to the root logger.
|
Remove the monitor handler that was added to the root logger.
|
||||||
'''
|
'''
|
||||||
logger = logging.getLogger()
|
logger = logging.getLogger()
|
||||||
|
|
||||||
for handler in tuple(logger.handlers):
|
for handler in tuple(logger.handlers):
|
||||||
if isinstance(handler, Loki_log_handler):
|
if isinstance(handler, Loki_log_handler):
|
||||||
handler.flush()
|
handler.flush()
|
||||||
|
|
|
@ -62,7 +62,7 @@ def execute_dump_command(
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named
|
Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named
|
||||||
pipe constructed from the given dump path and database names. Use the given log prefix in any
|
pipe constructed from the given dump path and database name. Use the given log prefix in any
|
||||||
log entries.
|
log entries.
|
||||||
|
|
||||||
Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if
|
Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if
|
||||||
|
@ -72,6 +72,7 @@ def execute_dump_command(
|
||||||
dump_filename = dump.make_data_source_dump_filename(
|
dump_filename = dump.make_data_source_dump_filename(
|
||||||
dump_path, database['name'], database.get('hostname')
|
dump_path, database['name'], database.get('hostname')
|
||||||
)
|
)
|
||||||
|
|
||||||
if os.path.exists(dump_filename):
|
if os.path.exists(dump_filename):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f'{log_prefix}: Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}'
|
f'{log_prefix}: Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}'
|
||||||
|
|
|
@ -62,7 +62,7 @@ def execute_dump_command(
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a
|
Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a
|
||||||
named pipe constructed from the given dump path and database names. Use the given log prefix in
|
named pipe constructed from the given dump path and database name. Use the given log prefix in
|
||||||
any log entries.
|
any log entries.
|
||||||
|
|
||||||
Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if
|
Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if
|
||||||
|
@ -72,6 +72,7 @@ def execute_dump_command(
|
||||||
dump_filename = dump.make_data_source_dump_filename(
|
dump_filename = dump.make_data_source_dump_filename(
|
||||||
dump_path, database['name'], database.get('hostname')
|
dump_path, database['name'], database.get('hostname')
|
||||||
)
|
)
|
||||||
|
|
||||||
if os.path.exists(dump_filename):
|
if os.path.exists(dump_filename):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f'{log_prefix}: Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}'
|
f'{log_prefix}: Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}'
|
||||||
|
|
|
@ -38,11 +38,11 @@ below for how to configure this.
|
||||||
|
|
||||||
borgmatic integrates with monitoring services like
|
borgmatic integrates with monitoring services like
|
||||||
[Healthchecks](https://healthchecks.io/), [Cronitor](https://cronitor.io),
|
[Healthchecks](https://healthchecks.io/), [Cronitor](https://cronitor.io),
|
||||||
[Cronhub](https://cronhub.io), [PagerDuty](https://www.pagerduty.com/), and
|
[Cronhub](https://cronhub.io), [PagerDuty](https://www.pagerduty.com/),
|
||||||
[ntfy](https://ntfy.sh/) and pings these services whenever borgmatic runs.
|
[ntfy](https://ntfy.sh/), and [Grafana Loki](https://grafana.com/oss/loki/)
|
||||||
That way, you'll receive an alert when something goes wrong or (for certain
|
and pings these services whenever borgmatic runs. That way, you'll receive an
|
||||||
hooks) the service doesn't hear from borgmatic for a configured interval. See
|
alert when something goes wrong or (for certain hooks) the service doesn't
|
||||||
[Healthchecks
|
hear from borgmatic for a configured interval. See [Healthchecks
|
||||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook),
|
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook),
|
||||||
[Cronitor
|
[Cronitor
|
||||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook),
|
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook),
|
||||||
|
@ -50,7 +50,10 @@ hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-h
|
||||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook),
|
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook),
|
||||||
[PagerDuty
|
[PagerDuty
|
||||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook),
|
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook),
|
||||||
and [ntfy hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook)
|
[ntfy
|
||||||
|
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook),
|
||||||
|
and [Loki
|
||||||
|
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook),
|
||||||
below for how to configure this.
|
below for how to configure this.
|
||||||
|
|
||||||
While these services offer different features, you probably only need to use
|
While these services offer different features, you probably only need to use
|
||||||
|
@ -129,7 +132,7 @@ especially the security information.
|
||||||
## Healthchecks hook
|
## Healthchecks hook
|
||||||
|
|
||||||
[Healthchecks](https://healthchecks.io/) is a service that provides "instant
|
[Healthchecks](https://healthchecks.io/) is a service that provides "instant
|
||||||
alerts when your cron jobs fail silently", and borgmatic has built-in
|
alerts when your cron jobs fail silently," and borgmatic has built-in
|
||||||
integration with it. Once you create a Healthchecks account and project on
|
integration with it. Once you create a Healthchecks account and project on
|
||||||
their site, all you need to do is configure borgmatic with the unique "Ping
|
their site, all you need to do is configure borgmatic with the unique "Ping
|
||||||
URL" for your project. Here's an example:
|
URL" for your project. Here's an example:
|
||||||
|
@ -144,21 +147,19 @@ healthchecks:
|
||||||
this option in the `hooks:` section of your configuration.
|
this option in the `hooks:` section of your configuration.
|
||||||
|
|
||||||
With this hook in place, borgmatic pings your Healthchecks project when a
|
With this hook in place, borgmatic pings your Healthchecks project when a
|
||||||
backup begins, ends, or errors. Specifically, after the <a
|
backup begins, ends, or errors, but only when any of the `create`, `prune`,
|
||||||
href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
|
`compact`, or `check` actions are run.
|
||||||
hooks</a> run, borgmatic lets Healthchecks know that it has started if any of
|
|
||||||
the `create`, `prune`, `compact`, or `check` actions are run.
|
|
||||||
|
|
||||||
Then, if the actions complete successfully, borgmatic notifies Healthchecks of
|
Then, if the actions complete successfully, borgmatic notifies Healthchecks of
|
||||||
the success after the `after_backup` hooks run and includes borgmatic logs in
|
the success and includes borgmatic logs in the payload data sent to
|
||||||
the payload data sent to Healthchecks. This means that borgmatic logs show up
|
Healthchecks. This means that borgmatic logs show up in the Healthchecks UI,
|
||||||
in the Healthchecks UI, although be aware that Healthchecks currently has a
|
although be aware that Healthchecks currently has a 10-kilobyte limit for the
|
||||||
10-kilobyte limit for the logs in each ping.
|
logs in each ping.
|
||||||
|
|
||||||
If an error occurs during any action or hook, borgmatic notifies Healthchecks
|
If an error occurs during any action or hook, borgmatic notifies Healthchecks,
|
||||||
after the `on_error` hooks run, also tacking on logs including the error
|
also tacking on logs including the error itself. But the logs are only
|
||||||
itself. But the logs are only included for errors that occur when a `create`,
|
included for errors that occur when a `create`, `prune`, `compact`, or `check`
|
||||||
`prune`, `compact`, or `check` action is run.
|
action is run.
|
||||||
|
|
||||||
You can customize the verbosity of the logs that are sent to Healthchecks with
|
You can customize the verbosity of the logs that are sent to Healthchecks with
|
||||||
borgmatic's `--monitoring-verbosity` flag. The `--list` and `--stats` flags
|
borgmatic's `--monitoring-verbosity` flag. The `--list` and `--stats` flags
|
||||||
|
@ -175,7 +176,7 @@ or it doesn't hear from borgmatic for a certain period of time.
|
||||||
## Cronitor hook
|
## Cronitor hook
|
||||||
|
|
||||||
[Cronitor](https://cronitor.io/) provides "Cron monitoring and uptime healthchecks
|
[Cronitor](https://cronitor.io/) provides "Cron monitoring and uptime healthchecks
|
||||||
for websites, services and APIs", and borgmatic has built-in
|
for websites, services and APIs," and borgmatic has built-in
|
||||||
integration with it. Once you create a Cronitor account and cron job monitor on
|
integration with it. Once you create a Cronitor account and cron job monitor on
|
||||||
their site, all you need to do is configure borgmatic with the unique "Ping
|
their site, all you need to do is configure borgmatic with the unique "Ping
|
||||||
API URL" for your monitor. Here's an example:
|
API URL" for your monitor. Here's an example:
|
||||||
|
@ -190,13 +191,9 @@ cronitor:
|
||||||
this option in the `hooks:` section of your configuration.
|
this option in the `hooks:` section of your configuration.
|
||||||
|
|
||||||
With this hook in place, borgmatic pings your Cronitor monitor when a backup
|
With this hook in place, borgmatic pings your Cronitor monitor when a backup
|
||||||
begins, ends, or errors. Specifically, after the <a
|
begins, ends, or errors, but only when any of the `prune`, `compact`,
|
||||||
href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
|
`create`, or `check` actions are run. Then, if the actions complete
|
||||||
hooks</a> run, borgmatic lets Cronitor know that it has started if any of the
|
successfully or errors, borgmatic notifies Cronitor accordingly.
|
||||||
`prune`, `compact`, `create`, or `check` actions are run. Then, if the actions
|
|
||||||
complete successfully, borgmatic notifies Cronitor of the success after the
|
|
||||||
`after_backup` hooks run. And if an error occurs during any action or hook,
|
|
||||||
borgmatic notifies Cronitor after the `on_error` hooks run.
|
|
||||||
|
|
||||||
You can configure Cronitor to notify you by a [variety of
|
You can configure Cronitor to notify you by a [variety of
|
||||||
mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
|
mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
|
||||||
|
@ -206,7 +203,7 @@ or it doesn't hear from borgmatic for a certain period of time.
|
||||||
## Cronhub hook
|
## Cronhub hook
|
||||||
|
|
||||||
[Cronhub](https://cronhub.io/) provides "instant alerts when any of your
|
[Cronhub](https://cronhub.io/) provides "instant alerts when any of your
|
||||||
background jobs fail silently or run longer than expected", and borgmatic has
|
background jobs fail silently or run longer than expected," and borgmatic has
|
||||||
built-in integration with it. Once you create a Cronhub account and monitor on
|
built-in integration with it. Once you create a Cronhub account and monitor on
|
||||||
their site, all you need to do is configure borgmatic with the unique "Ping
|
their site, all you need to do is configure borgmatic with the unique "Ping
|
||||||
URL" for your monitor. Here's an example:
|
URL" for your monitor. Here's an example:
|
||||||
|
@ -221,13 +218,9 @@ cronhub:
|
||||||
this option in the `hooks:` section of your configuration.
|
this option in the `hooks:` section of your configuration.
|
||||||
|
|
||||||
With this hook in place, borgmatic pings your Cronhub monitor when a backup
|
With this hook in place, borgmatic pings your Cronhub monitor when a backup
|
||||||
begins, ends, or errors. Specifically, after the <a
|
begins, ends, or errors, but only when any of the `prune`, `compact`,
|
||||||
href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
|
`create`, or `check` actions are run. Then, if the actions complete
|
||||||
hooks</a> run, borgmatic lets Cronhub know that it has started if any of the
|
successfully or errors, borgmatic notifies Cronhub accordingly.
|
||||||
`prune`, `compact`, `create`, or `check` actions are run. Then, if the actions
|
|
||||||
complete successfully, borgmatic notifies Cronhub of the success after the
|
|
||||||
`after_backup` hooks run. And if an error occurs during any action or hook,
|
|
||||||
borgmatic notifies Cronhub after the `on_error` hooks run.
|
|
||||||
|
|
||||||
Note that even though you configure borgmatic with the "start" variant of the
|
Note that even though you configure borgmatic with the "start" variant of the
|
||||||
ping URL, borgmatic substitutes the correct state into the URL when pinging
|
ping URL, borgmatic substitutes the correct state into the URL when pinging
|
||||||
|
@ -266,10 +259,9 @@ pagerduty:
|
||||||
this option in the `hooks:` section of your configuration.
|
this option in the `hooks:` section of your configuration.
|
||||||
|
|
||||||
With this hook in place, borgmatic creates a PagerDuty event for your service
|
With this hook in place, borgmatic creates a PagerDuty event for your service
|
||||||
whenever backups fail. Specifically, if an error occurs during a `create`,
|
whenever backups fail, but only when any of the `create`, `prune`, `compact`,
|
||||||
`prune`, `compact`, or `check` action, borgmatic sends an event to PagerDuty
|
or `check` actions are run. Note that borgmatic does not contact PagerDuty
|
||||||
before the `on_error` hooks run. Note that borgmatic does not contact
|
when a backup starts or when it ends without error.
|
||||||
PagerDuty when a backup starts or ends without error.
|
|
||||||
|
|
||||||
You can configure PagerDuty to notify you by a [variety of
|
You can configure PagerDuty to notify you by a [variety of
|
||||||
mechanisms](https://support.pagerduty.com/docs/notifications) when backups
|
mechanisms](https://support.pagerduty.com/docs/notifications) when backups
|
||||||
|
@ -328,6 +320,58 @@ ntfy:
|
||||||
the `ntfy:` option in the `hooks:` section of your configuration.
|
the `ntfy:` option in the `hooks:` section of your configuration.
|
||||||
|
|
||||||
|
|
||||||
|
## Loki hook
|
||||||
|
|
||||||
|
[Grafana Loki](https://grafana.com/oss/loki/) is a "horizontally scalable,
|
||||||
|
highly available, multi-tenant log aggregation system inspired by Prometheus."
|
||||||
|
borgmatic has built-in integration with Loki, sending both backup status and
|
||||||
|
borgmatic logs.
|
||||||
|
|
||||||
|
You can configure borgmatic to use either a [self-hosted Loki
|
||||||
|
instance](https://grafana.com/docs/loki/latest/installation/) or [a Grafana
|
||||||
|
Cloud account](https://grafana.com/auth/sign-up/create-user). Start by setting
|
||||||
|
your Loki API push URL. Here's an example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
loki:
|
||||||
|
url: http://localhost:3100/loki/api/v1/push
|
||||||
|
```
|
||||||
|
|
||||||
|
With this hook in place, borgmatic sends its logs to your Loki instance as any
|
||||||
|
of the `prune`, `compact`, `create`, or `check` actions are run. Then, after
|
||||||
|
the actions complete, borgmatic notifies Loki of success or failure.
|
||||||
|
|
||||||
|
This hook supports sending arbitrary labels to Loki. For instance:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
loki:
|
||||||
|
url: http://localhost:3100/loki/api/v1/push
|
||||||
|
|
||||||
|
labels:
|
||||||
|
app: borgmatic
|
||||||
|
hostname: example.org
|
||||||
|
```
|
||||||
|
|
||||||
|
There are also a few placeholders you can optionally use as label values:
|
||||||
|
|
||||||
|
* `__config`: name of the borgmatic configuration file
|
||||||
|
* `__config_path`: full path of the borgmatic configuration file
|
||||||
|
* `__hostname`: the local machine hostname
|
||||||
|
|
||||||
|
These placeholders are only substituted for the whole label value, not
|
||||||
|
interpolated into a larger string. For instance:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
loki:
|
||||||
|
url: http://localhost:3100/loki/api/v1/push
|
||||||
|
|
||||||
|
labels:
|
||||||
|
app: borgmatic
|
||||||
|
config: __config
|
||||||
|
hostname: __hostname
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
## Scripting borgmatic
|
## Scripting borgmatic
|
||||||
|
|
||||||
To consume the output of borgmatic in other software, you can include an
|
To consume the output of borgmatic in other software, you can include an
|
||||||
|
|
|
@ -6,9 +6,9 @@ from flexmock import flexmock
|
||||||
from borgmatic.hooks import loki as module
|
from borgmatic.hooks import loki as module
|
||||||
|
|
||||||
|
|
||||||
def test_log_handler_label_replacment():
|
def test_initialize_monitor_replaces_labels():
|
||||||
'''
|
'''
|
||||||
Assert that label placeholders get replaced
|
Assert that label placeholders get replaced.
|
||||||
'''
|
'''
|
||||||
hook_config = {
|
hook_config = {
|
||||||
'url': 'http://localhost:3100/loki/api/v1/push',
|
'url': 'http://localhost:3100/loki/api/v1/push',
|
||||||
|
@ -17,18 +17,20 @@ def test_log_handler_label_replacment():
|
||||||
config_filename = '/mock/path/test.yaml'
|
config_filename = '/mock/path/test.yaml'
|
||||||
dry_run = True
|
dry_run = True
|
||||||
module.initialize_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run)
|
module.initialize_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run)
|
||||||
|
|
||||||
for handler in tuple(logging.getLogger().handlers):
|
for handler in tuple(logging.getLogger().handlers):
|
||||||
if isinstance(handler, module.Loki_log_handler):
|
if isinstance(handler, module.Loki_log_handler):
|
||||||
assert handler.buffer.root['streams'][0]['stream']['hostname'] == platform.node()
|
assert handler.buffer.root['streams'][0]['stream']['hostname'] == platform.node()
|
||||||
assert handler.buffer.root['streams'][0]['stream']['config'] == 'test.yaml'
|
assert handler.buffer.root['streams'][0]['stream']['config'] == 'test.yaml'
|
||||||
assert handler.buffer.root['streams'][0]['stream']['config_full'] == config_filename
|
assert handler.buffer.root['streams'][0]['stream']['config_full'] == config_filename
|
||||||
return
|
return
|
||||||
|
|
||||||
assert False
|
assert False
|
||||||
|
|
||||||
|
|
||||||
def test_initalize_adds_log_handler():
|
def test_initialize_monitor_adds_log_handler():
|
||||||
'''
|
'''
|
||||||
Assert that calling initialize_monitor adds our logger to the root logger
|
Assert that calling initialize_monitor adds our logger to the root logger.
|
||||||
'''
|
'''
|
||||||
hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}}
|
hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}}
|
||||||
module.initialize_monitor(
|
module.initialize_monitor(
|
||||||
|
@ -38,15 +40,17 @@ def test_initalize_adds_log_handler():
|
||||||
monitoring_log_level=flexmock(),
|
monitoring_log_level=flexmock(),
|
||||||
dry_run=True,
|
dry_run=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
for handler in tuple(logging.getLogger().handlers):
|
for handler in tuple(logging.getLogger().handlers):
|
||||||
if isinstance(handler, module.Loki_log_handler):
|
if isinstance(handler, module.Loki_log_handler):
|
||||||
return
|
return
|
||||||
|
|
||||||
assert False
|
assert False
|
||||||
|
|
||||||
|
|
||||||
def test_ping_adds_log_message():
|
def test_ping_monitor_adds_log_message():
|
||||||
'''
|
'''
|
||||||
Assert that calling ping_monitor adds a message to our logger
|
Assert that calling ping_monitor adds a message to our logger.
|
||||||
'''
|
'''
|
||||||
hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}}
|
hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}}
|
||||||
config_filename = 'test.yaml'
|
config_filename = 'test.yaml'
|
||||||
|
@ -55,6 +59,7 @@ def test_ping_adds_log_message():
|
||||||
module.ping_monitor(
|
module.ping_monitor(
|
||||||
hook_config, flexmock(), config_filename, module.monitor.State.FINISH, flexmock(), dry_run
|
hook_config, flexmock(), config_filename, module.monitor.State.FINISH, flexmock(), dry_run
|
||||||
)
|
)
|
||||||
|
|
||||||
for handler in tuple(logging.getLogger().handlers):
|
for handler in tuple(logging.getLogger().handlers):
|
||||||
if isinstance(handler, module.Loki_log_handler):
|
if isinstance(handler, module.Loki_log_handler):
|
||||||
assert any(
|
assert any(
|
||||||
|
@ -65,18 +70,20 @@ def test_ping_adds_log_message():
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
assert False
|
assert False
|
||||||
|
|
||||||
|
|
||||||
def test_log_handler_gets_removed():
|
def test_destroy_monitor_removes_log_handler():
|
||||||
'''
|
'''
|
||||||
Assert that destroy_monitor removes the logger from the root logger
|
Assert that destroy_monitor removes the logger from the root logger.
|
||||||
'''
|
'''
|
||||||
hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}}
|
hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}}
|
||||||
config_filename = 'test.yaml'
|
config_filename = 'test.yaml'
|
||||||
dry_run = True
|
dry_run = True
|
||||||
module.initialize_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run)
|
module.initialize_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run)
|
||||||
module.destroy_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run)
|
module.destroy_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run)
|
||||||
|
|
||||||
for handler in tuple(logging.getLogger().handlers):
|
for handler in tuple(logging.getLogger().handlers):
|
||||||
if isinstance(handler, module.Loki_log_handler):
|
if isinstance(handler, module.Loki_log_handler):
|
||||||
assert False
|
assert False
|
||||||
|
|
|
@ -6,93 +6,103 @@ from flexmock import flexmock
|
||||||
from borgmatic.hooks import loki as module
|
from borgmatic.hooks import loki as module
|
||||||
|
|
||||||
|
|
||||||
def test_log_handler_gets_labels():
|
def test_loki_log_buffer_add_value_gets_raw():
|
||||||
'''
|
'''
|
||||||
Assert that adding labels works
|
Assert that adding values to the log buffer increases it's length.
|
||||||
'''
|
|
||||||
buffer = module.Loki_log_buffer(flexmock(), False)
|
|
||||||
buffer.add_label('test', 'label')
|
|
||||||
assert buffer.root['streams'][0]['stream']['test'] == 'label'
|
|
||||||
buffer.add_label('test2', 'label2')
|
|
||||||
assert buffer.root['streams'][0]['stream']['test2'] == 'label2'
|
|
||||||
|
|
||||||
|
|
||||||
def test_log_buffer_gets_raw():
|
|
||||||
'''
|
|
||||||
Assert that adding values to the log buffer increases it's length
|
|
||||||
'''
|
'''
|
||||||
buffer = module.Loki_log_buffer(flexmock(), False)
|
buffer = module.Loki_log_buffer(flexmock(), False)
|
||||||
assert len(buffer) == 0
|
assert len(buffer) == 0
|
||||||
|
|
||||||
buffer.add_value('Some test log line')
|
buffer.add_value('Some test log line')
|
||||||
assert len(buffer) == 1
|
assert len(buffer) == 1
|
||||||
|
|
||||||
buffer.add_value('Another test log line')
|
buffer.add_value('Another test log line')
|
||||||
assert len(buffer) == 2
|
assert len(buffer) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_log_buffer_gets_log_messages():
|
def test_loki_log_buffer_json_serializes_empty_buffer():
|
||||||
'''
|
'''
|
||||||
Assert that adding log records works
|
Assert that the buffer correctly serializes when empty.
|
||||||
'''
|
|
||||||
handler = module.Loki_log_handler(flexmock(), False)
|
|
||||||
handler.emit(flexmock(getMessage=lambda: 'Some test log line'))
|
|
||||||
assert len(handler.buffer) == 1
|
|
||||||
|
|
||||||
|
|
||||||
def test_log_buffer_json():
|
|
||||||
'''
|
|
||||||
Assert that the buffer correctly serializes when empty
|
|
||||||
'''
|
'''
|
||||||
buffer = module.Loki_log_buffer(flexmock(), False)
|
buffer = module.Loki_log_buffer(flexmock(), False)
|
||||||
|
|
||||||
assert json.loads(buffer.to_request()) == json.loads('{"streams":[{"stream":{},"values":[]}]}')
|
assert json.loads(buffer.to_request()) == json.loads('{"streams":[{"stream":{},"values":[]}]}')
|
||||||
|
|
||||||
|
|
||||||
def test_log_buffer_json_labels():
|
def test_loki_log_buffer_json_serializes_labels():
|
||||||
'''
|
'''
|
||||||
Assert that the buffer correctly serializes with labels
|
Assert that the buffer correctly serializes with labels.
|
||||||
'''
|
'''
|
||||||
buffer = module.Loki_log_buffer(flexmock(), False)
|
buffer = module.Loki_log_buffer(flexmock(), False)
|
||||||
buffer.add_label('test', 'label')
|
buffer.add_label('test', 'label')
|
||||||
|
|
||||||
assert json.loads(buffer.to_request()) == json.loads(
|
assert json.loads(buffer.to_request()) == json.loads(
|
||||||
'{"streams":[{"stream":{"test": "label"},"values":[]}]}'
|
'{"streams":[{"stream":{"test": "label"},"values":[]}]}'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_log_buffer_json_log_lines():
|
def test_loki_log_buffer_json_serializes_log_lines():
|
||||||
'''
|
'''
|
||||||
Assert that log lines end up in the correct place in the log buffer
|
Assert that log lines end up in the correct place in the log buffer.
|
||||||
'''
|
'''
|
||||||
buffer = module.Loki_log_buffer(flexmock(), False)
|
buffer = module.Loki_log_buffer(flexmock(), False)
|
||||||
buffer.add_value('Some test log line')
|
buffer.add_value('Some test log line')
|
||||||
|
|
||||||
assert json.loads(buffer.to_request())['streams'][0]['values'][0][1] == 'Some test log line'
|
assert json.loads(buffer.to_request())['streams'][0]['values'][0][1] == 'Some test log line'
|
||||||
|
|
||||||
|
|
||||||
def test_log_handler_post():
|
def test_loki_log_handler_add_label_gets_labels():
|
||||||
'''
|
'''
|
||||||
Assert that the flush function sends a post request after a certain limit
|
Assert that adding labels works.
|
||||||
|
'''
|
||||||
|
buffer = module.Loki_log_buffer(flexmock(), False)
|
||||||
|
|
||||||
|
buffer.add_label('test', 'label')
|
||||||
|
assert buffer.root['streams'][0]['stream']['test'] == 'label'
|
||||||
|
|
||||||
|
buffer.add_label('test2', 'label2')
|
||||||
|
assert buffer.root['streams'][0]['stream']['test2'] == 'label2'
|
||||||
|
|
||||||
|
|
||||||
|
def test_loki_log_handler_emit_gets_log_messages():
|
||||||
|
'''
|
||||||
|
Assert that adding log records works.
|
||||||
|
'''
|
||||||
|
handler = module.Loki_log_handler(flexmock(), False)
|
||||||
|
handler.emit(flexmock(getMessage=lambda: 'Some test log line'))
|
||||||
|
|
||||||
|
assert len(handler.buffer) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_loki_log_handler_raw_posts_to_server():
|
||||||
|
'''
|
||||||
|
Assert that the flush function sends a post request after a certain limit.
|
||||||
'''
|
'''
|
||||||
handler = module.Loki_log_handler(flexmock(), False)
|
handler = module.Loki_log_handler(flexmock(), False)
|
||||||
flexmock(module.requests).should_receive('post').and_return(
|
flexmock(module.requests).should_receive('post').and_return(
|
||||||
flexmock(raise_for_status=lambda: '')
|
flexmock(raise_for_status=lambda: '')
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
for num in range(int(module.MAX_BUFFER_LINES * 1.5)):
|
for num in range(int(module.MAX_BUFFER_LINES * 1.5)):
|
||||||
handler.raw(num)
|
handler.raw(num)
|
||||||
|
|
||||||
|
|
||||||
def test_log_handler_post_failiure():
|
def test_loki_log_handler_raw_post_failure_does_not_raise():
|
||||||
'''
|
'''
|
||||||
Assert that the flush function catches request exceptions
|
Assert that the flush function catches request exceptions.
|
||||||
'''
|
'''
|
||||||
handler = module.Loki_log_handler(flexmock(), False)
|
handler = module.Loki_log_handler(flexmock(), False)
|
||||||
flexmock(module.requests).should_receive('post').and_return(
|
flexmock(module.requests).should_receive('post').and_return(
|
||||||
flexmock(raise_for_status=lambda: (_ for _ in ()).throw(requests.RequestException()))
|
flexmock(raise_for_status=lambda: (_ for _ in ()).throw(requests.RequestException()))
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
for num in range(int(module.MAX_BUFFER_LINES * 1.5)):
|
for num in range(int(module.MAX_BUFFER_LINES * 1.5)):
|
||||||
handler.raw(num)
|
handler.raw(num)
|
||||||
|
|
||||||
|
|
||||||
def test_log_handler_empty_flush_noop():
|
def test_loki_log_handler_flush_with_empty_buffer_does_not_raise():
|
||||||
'''
|
'''
|
||||||
Test that flushing an empty buffer does indeed nothing
|
Test that flushing an empty buffer does indeed nothing.
|
||||||
'''
|
'''
|
||||||
handler = module.Loki_log_handler(flexmock(), False)
|
handler = module.Loki_log_handler(flexmock(), False)
|
||||||
handler.flush()
|
handler.flush()
|
||||||
|
|
Loading…
Reference in a new issue