From 02781662f85bcdbab795bf67fdecf28f85cb50d7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 23 May 2022 20:02:10 -0700 Subject: [PATCH] Change monitoring hooks to specify the ping URL / integration key as a named option. --- NEWS | 3 + borgmatic/config/normalize.py | 20 +++++- borgmatic/config/schema.yaml | 88 ++++++++++++++++++--------- borgmatic/hooks/cronhub.py | 10 ++- borgmatic/hooks/cronitor.py | 6 +- borgmatic/hooks/healthchecks.py | 14 ++--- borgmatic/hooks/pagerduty.py | 8 +-- docs/how-to/monitor-your-backups.md | 12 ++-- tests/unit/config/test_normalize.py | 16 +++++ tests/unit/hooks/test_cronhub.py | 32 +++++++--- tests/unit/hooks/test_cronitor.py | 30 +++++---- tests/unit/hooks/test_healthchecks.py | 28 ++++----- tests/unit/hooks/test_pagerduty.py | 24 ++++++-- 13 files changed, 199 insertions(+), 92 deletions(-) diff --git a/NEWS b/NEWS index 4445849..714c483 100644 --- a/NEWS +++ b/NEWS @@ -8,6 +8,9 @@ directory or from the directory containing the file doing the including. Previously, only the working directory was used. * Add a randomized delay to the sample systemd timer to spread out the load on a server. + * Change the configuration format for borgmatic monitoring hooks (Healthchecks, Cronitor, + PagerDuty, and Cronhub) to specify the ping URL / integration key as a named option. The intent + is to support additional options in the future. This change is backwards-compatible. * Add emojis to documentation table of contents to make it easier to find particular how-to and reference guides at a glance. diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 05080bb..59256ff 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -3,8 +3,24 @@ def normalize(config): Given a configuration dict, apply particular hard-coded rules to normalize its contents to adhere to the configuration schema. ''' + # Upgrade exclude_if_present from a string to a list. exclude_if_present = config.get('location', {}).get('exclude_if_present') - - # "Upgrade" exclude_if_present from a string to a list. if isinstance(exclude_if_present, str): config['location']['exclude_if_present'] = [exclude_if_present] + + # Upgrade various monitoring hooks from a string to a dict. + healthchecks = config.get('hooks', {}).get('healthchecks') + if isinstance(healthchecks, str): + config['hooks']['healthchecks'] = {'ping_url': healthchecks} + + cronitor = config.get('hooks', {}).get('cronitor') + if isinstance(cronitor, str): + config['hooks']['cronitor'] = {'ping_url': cronitor} + + pagerduty = config.get('hooks', {}).get('pagerduty') + if isinstance(pagerduty, str): + config['hooks']['pagerduty'] = {'integration_key': pagerduty} + + cronhub = config.get('hooks', {}).get('cronhub') + if isinstance(cronhub, str): + config['hooks']['cronhub'] = {'ping_url': cronhub} diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 16a3fd5..0a16bab 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -882,41 +882,69 @@ properties: https://docs.mongodb.com/database-tools/mongorestore/ for details. healthchecks: - type: string + type: object + required: ['ping_url'] + additionalProperties: false + properties: + ping_url: + type: string + description: | + Healthchecks ping URL or UUID to notify when a + backup begins, ends, or errors. + example: https://hc-ping.com/your-uuid-here description: | - Healthchecks ping URL or UUID to notify when a backup - begins, ends, or errors. Create an account at - https://healthchecks.io if you'd like to use this service. - See borgmatic monitoring documentation for details. - example: - https://hc-ping.com/your-uuid-here - cronitor: - type: string - description: | - Cronitor ping URL to notify when a backup begins, ends, or - errors. Create an account at https://cronitor.io if you'd - like to use this service. See borgmatic monitoring - documentation for details. - example: - https://cronitor.link/d3x0c1 - pagerduty: - type: string - description: | - PagerDuty integration key used to notify PagerDuty when a - backup errors. Create an account at - https://www.pagerduty.com/ if you'd like to use this + Configuration for a monitoring integration with + Healthchecks. Create an account at https://healthchecks.io + (or self-host Healthchecks) if you'd like to use this service. See borgmatic monitoring documentation for details. - example: - a177cad45bd374409f78906a810a3074 - cronhub: - type: string + cronitor: + type: object + required: ['ping_url'] + additionalProperties: false + properties: + ping_url: + type: string + description: | + Cronitor ping URL to notify when a backup begins, + ends, or errors. + example: https://cronitor.link/d3x0c1 description: | - Cronhub ping URL to notify when a backup begins, ends, or - errors. Create an account at https://cronhub.io if you'd + Configuration for a monitoring integration with Cronitor. + Create an account at https://cronitor.io if you'd like to use this service. See borgmatic monitoring documentation for details. - example: - https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d01 + pagerduty: + type: object + required: ['integration_key'] + additionalProperties: false + properties: + integration_key: + type: string + description: | + PagerDuty integration key used to notify PagerDuty + when a backup errors. + example: a177cad45bd374409f78906a810a3074 + description: | + Configuration for a monitoring integration with PagerDuty. + Create an account at https://www.pagerduty.com/ if you'd + like to use this service. See borgmatic monitoring + documentation for details. + cronhub: + type: object + required: ['ping_url'] + additionalProperties: false + properties: + ping_url: + type: string + description: | + Cronhub ping URL to notify when a backup begins, + ends, or errors. + example: https://cronhub.io/ping/1f5e3410-254c-5587 + description: | + Configuration for a monitoring integration with Crunhub. + Create an account at https://cronhub.io if you'd like to + use this service. See borgmatic monitoring documentation + for details. umask: type: integer description: | diff --git a/borgmatic/hooks/cronhub.py b/borgmatic/hooks/cronhub.py index ee3cee1..68ef4fd 100644 --- a/borgmatic/hooks/cronhub.py +++ b/borgmatic/hooks/cronhub.py @@ -22,14 +22,18 @@ def initialize_monitor( pass -def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): ''' - Ping the given Cronhub URL, modified with the monitor.State. Use the given configuration + Ping the configured Cronhub URL, modified with the monitor.State. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. ''' dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' formatted_state = '/{}/'.format(MONITOR_STATE_TO_CRONHUB[state]) - ping_url = ping_url.replace('/start/', formatted_state).replace('/ping/', formatted_state) + ping_url = ( + hook_config['ping_url'] + .replace('/start/', formatted_state) + .replace('/ping/', formatted_state) + ) logger.info( '{}: Pinging Cronhub {}{}'.format(config_filename, state.name.lower(), dry_run_label) diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index e8275f0..a9acec3 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -22,13 +22,13 @@ def initialize_monitor( pass -def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): ''' - Ping the given Cronitor URL, modified with the monitor.State. Use the given configuration + Ping the configured Cronitor URL, modified with the monitor.State. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. ''' dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' - ping_url = '{}/{}'.format(ping_url, MONITOR_STATE_TO_CRONITOR[state]) + ping_url = '{}/{}'.format(hook_config['ping_url'], MONITOR_STATE_TO_CRONITOR[state]) logger.info( '{}: Pinging Cronitor {}{}'.format(config_filename, state.name.lower(), dry_run_label) diff --git a/borgmatic/hooks/healthchecks.py b/borgmatic/hooks/healthchecks.py index 1196620..3fe3ec9 100644 --- a/borgmatic/hooks/healthchecks.py +++ b/borgmatic/hooks/healthchecks.py @@ -66,7 +66,7 @@ def format_buffered_logs_for_payload(): def initialize_monitor( - ping_url_or_uuid, config_filename, monitoring_log_level, dry_run + hook_config, config_filename, monitoring_log_level, dry_run ): # pragma: no cover ''' Add a handler to the root logger that stores in memory the most recent logs emitted. That @@ -77,16 +77,16 @@ def initialize_monitor( ) -def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): ''' - Ping the given Healthchecks URL or UUID, modified with the monitor.State. Use the given + Ping the configured Healthchecks URL or UUID, modified with the monitor.State. Use the given configuration filename in any log entries, and log to Healthchecks with the giving log level. If this is a dry run, then don't actually ping anything. ''' ping_url = ( - ping_url_or_uuid - if ping_url_or_uuid.startswith('http') - else 'https://hc-ping.com/{}'.format(ping_url_or_uuid) + hook_config['ping_url'] + if hook_config['ping_url'].startswith('http') + else 'https://hc-ping.com/{}'.format(hook_config['ping_url']) ) dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' @@ -109,7 +109,7 @@ def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level, requests.post(ping_url, data=payload.encode('utf-8')) -def destroy_monitor(ping_url_or_uuid, config_filename, monitoring_log_level, dry_run): +def destroy_monitor(hook_config, config_filename, monitoring_log_level, dry_run): ''' Remove the monitor handler that was added to the root logger. This prevents the handler from getting reused by other instances of this monitor. diff --git a/borgmatic/hooks/pagerduty.py b/borgmatic/hooks/pagerduty.py index ccddfdf..8d96f56 100644 --- a/borgmatic/hooks/pagerduty.py +++ b/borgmatic/hooks/pagerduty.py @@ -21,10 +21,10 @@ def initialize_monitor( pass -def ping_monitor(integration_key, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): ''' - If this is an error state, create a PagerDuty event with the given integration key. Use the - given configuration filename in any log entries. If this is a dry run, then don't actually + If this is an error state, create a PagerDuty event with the configured integration key. Use + the given configuration filename in any log entries. If this is a dry run, then don't actually create an event. ''' if state != monitor.State.FAIL: @@ -47,7 +47,7 @@ def ping_monitor(integration_key, config_filename, state, monitoring_log_level, ) payload = json.dumps( { - 'routing_key': integration_key, + 'routing_key': hook_config['integration_key'], 'event_action': 'trigger', 'payload': { 'summary': 'backup failed on {}'.format(hostname), diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 025f0fe..70a3e0c 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -136,7 +136,8 @@ URL" for your project. Here's an example: ```yaml hooks: - healthchecks: https://hc-ping.com/addffa72-da17-40ae-be9c-ff591afb942a + healthchecks: + ping_url: https://hc-ping.com/addffa72-da17-40ae-be9c-ff591afb942a ``` With this hook in place, borgmatic pings your Healthchecks project when a @@ -176,7 +177,8 @@ API URL" for your monitor. Here's an example: ```yaml hooks: - cronitor: https://cronitor.link/d3x0c1 + cronitor: + ping_url: https://cronitor.link/d3x0c1 ``` With this hook in place, borgmatic pings your Cronitor monitor when a backup @@ -204,7 +206,8 @@ URL" for your monitor. Here's an example: ```yaml hooks: - cronhub: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031 + cronhub: + ping_url: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031 ``` With this hook in place, borgmatic pings your Cronhub monitor when a backup @@ -246,7 +249,8 @@ Here's an example: ```yaml hooks: - pagerduty: a177cad45bd374409f78906a810a3074 + pagerduty: + integration_key: a177cad45bd374409f78906a810a3074 ``` With this hook in place, borgmatic creates a PagerDuty event for your service diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index 58e5105..4fe5e67 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -19,6 +19,22 @@ from borgmatic.config import normalize as module {'location': {'source_directories': ['foo', 'bar']}}, ), ({'storage': {'compression': 'yes_please'}}, {'storage': {'compression': 'yes_please'}}), + ( + {'hooks': {'healthchecks': 'https://example.com'}}, + {'hooks': {'healthchecks': {'ping_url': 'https://example.com'}}}, + ), + ( + {'hooks': {'cronitor': 'https://example.com'}}, + {'hooks': {'cronitor': {'ping_url': 'https://example.com'}}}, + ), + ( + {'hooks': {'pagerduty': 'https://example.com'}}, + {'hooks': {'pagerduty': {'integration_key': 'https://example.com'}}}, + ), + ( + {'hooks': {'cronhub': 'https://example.com'}}, + {'hooks': {'cronhub': {'ping_url': 'https://example.com'}}}, + ), ), ) def test_normalize_applies_hard_coded_normalization_to_config(config, expected_config): diff --git a/tests/unit/hooks/test_cronhub.py b/tests/unit/hooks/test_cronhub.py index a258283..0a5abff 100644 --- a/tests/unit/hooks/test_cronhub.py +++ b/tests/unit/hooks/test_cronhub.py @@ -4,45 +4,57 @@ from borgmatic.hooks import cronhub as module def test_ping_monitor_rewrites_ping_url_for_start_state(): - ping_url = 'https://example.com/start/abcdef' + hook_config = {'ping_url': 'https://example.com/start/abcdef'} flexmock(module.requests).should_receive('get').with_args('https://example.com/start/abcdef') module.ping_monitor( - ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + module.monitor.State.START, + monitoring_log_level=1, + dry_run=False, ) def test_ping_monitor_rewrites_ping_url_and_state_for_start_state(): - ping_url = 'https://example.com/ping/abcdef' + hook_config = {'ping_url': 'https://example.com/ping/abcdef'} flexmock(module.requests).should_receive('get').with_args('https://example.com/start/abcdef') module.ping_monitor( - ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + module.monitor.State.START, + monitoring_log_level=1, + dry_run=False, ) def test_ping_monitor_rewrites_ping_url_for_finish_state(): - ping_url = 'https://example.com/start/abcdef' + hook_config = {'ping_url': 'https://example.com/start/abcdef'} flexmock(module.requests).should_receive('get').with_args('https://example.com/finish/abcdef') module.ping_monitor( - ping_url, 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + module.monitor.State.FINISH, + monitoring_log_level=1, + dry_run=False, ) def test_ping_monitor_rewrites_ping_url_for_fail_state(): - ping_url = 'https://example.com/start/abcdef' + hook_config = {'ping_url': 'https://example.com/start/abcdef'} flexmock(module.requests).should_receive('get').with_args('https://example.com/fail/abcdef') module.ping_monitor( - ping_url, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False ) def test_ping_monitor_dry_run_does_not_hit_ping_url(): - ping_url = 'https://example.com' + hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True + hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True ) diff --git a/tests/unit/hooks/test_cronitor.py b/tests/unit/hooks/test_cronitor.py index e006508..d080424 100644 --- a/tests/unit/hooks/test_cronitor.py +++ b/tests/unit/hooks/test_cronitor.py @@ -4,36 +4,44 @@ from borgmatic.hooks import cronitor as module def test_ping_monitor_hits_ping_url_for_start_state(): - ping_url = 'https://example.com' - flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, 'run')) + hook_config = {'ping_url': 'https://example.com'} + flexmock(module.requests).should_receive('get').with_args('https://example.com/run') module.ping_monitor( - ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + module.monitor.State.START, + monitoring_log_level=1, + dry_run=False, ) def test_ping_monitor_hits_ping_url_for_finish_state(): - ping_url = 'https://example.com' - flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, 'complete')) + hook_config = {'ping_url': 'https://example.com'} + flexmock(module.requests).should_receive('get').with_args('https://example.com/complete') module.ping_monitor( - ping_url, 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + module.monitor.State.FINISH, + monitoring_log_level=1, + dry_run=False, ) def test_ping_monitor_hits_ping_url_for_fail_state(): - ping_url = 'https://example.com' - flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, 'fail')) + hook_config = {'ping_url': 'https://example.com'} + flexmock(module.requests).should_receive('get').with_args('https://example.com/fail') module.ping_monitor( - ping_url, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False ) def test_ping_monitor_dry_run_does_not_hit_ping_url(): - ping_url = 'https://example.com' + hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True + hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True ) diff --git a/tests/unit/hooks/test_healthchecks.py b/tests/unit/hooks/test_healthchecks.py index 59e86d5..346ffbe 100644 --- a/tests/unit/hooks/test_healthchecks.py +++ b/tests/unit/hooks/test_healthchecks.py @@ -62,13 +62,13 @@ def test_format_buffered_logs_for_payload_without_handler_produces_empty_payload def test_ping_monitor_hits_ping_url_for_start_state(): flexmock(module).should_receive('Forgetful_buffering_handler') - ping_url = 'https://example.com' + hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('post').with_args( - '{}/{}'.format(ping_url, 'start'), data=''.encode('utf-8') + 'https://example.com/start', data=''.encode('utf-8') ) module.ping_monitor( - ping_url, + hook_config, 'config.yaml', state=module.monitor.State.START, monitoring_log_level=1, @@ -77,15 +77,15 @@ def test_ping_monitor_hits_ping_url_for_start_state(): def test_ping_monitor_hits_ping_url_for_finish_state(): - ping_url = 'https://example.com' + hook_config = {'ping_url': 'https://example.com'} payload = 'data' flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload) flexmock(module.requests).should_receive('post').with_args( - ping_url, data=payload.encode('utf-8') + 'https://example.com', data=payload.encode('utf-8') ) module.ping_monitor( - ping_url, + hook_config, 'config.yaml', state=module.monitor.State.FINISH, monitoring_log_level=1, @@ -94,15 +94,15 @@ def test_ping_monitor_hits_ping_url_for_finish_state(): def test_ping_monitor_hits_ping_url_for_fail_state(): - ping_url = 'https://example.com' + hook_config = {'ping_url': 'https://example.com'} payload = 'data' flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload) flexmock(module.requests).should_receive('post').with_args( - '{}/{}'.format(ping_url, 'fail'), data=payload.encode('utf') + 'https://example.com/fail', data=payload.encode('utf') ) module.ping_monitor( - ping_url, + hook_config, 'config.yaml', state=module.monitor.State.FAIL, monitoring_log_level=1, @@ -111,15 +111,15 @@ def test_ping_monitor_hits_ping_url_for_fail_state(): def test_ping_monitor_with_ping_uuid_hits_corresponding_url(): - ping_uuid = 'abcd-efgh-ijkl-mnop' + hook_config = {'ping_url': 'abcd-efgh-ijkl-mnop'} payload = 'data' flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload) flexmock(module.requests).should_receive('post').with_args( - 'https://hc-ping.com/{}'.format(ping_uuid), data=payload.encode('utf-8') + 'https://hc-ping.com/{}'.format(hook_config['ping_url']), data=payload.encode('utf-8') ) module.ping_monitor( - ping_uuid, + hook_config, 'config.yaml', state=module.monitor.State.FINISH, monitoring_log_level=1, @@ -129,11 +129,11 @@ def test_ping_monitor_with_ping_uuid_hits_corresponding_url(): def test_ping_monitor_dry_run_does_not_hit_ping_url(): flexmock(module).should_receive('Forgetful_buffering_handler') - ping_url = 'https://example.com' + hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('post').never() module.ping_monitor( - ping_url, + hook_config, 'config.yaml', state=module.monitor.State.START, monitoring_log_level=1, diff --git a/tests/unit/hooks/test_pagerduty.py b/tests/unit/hooks/test_pagerduty.py index 76c5451..4f425de 100644 --- a/tests/unit/hooks/test_pagerduty.py +++ b/tests/unit/hooks/test_pagerduty.py @@ -7,7 +7,11 @@ def test_ping_monitor_ignores_start_state(): flexmock(module.requests).should_receive('post').never() module.ping_monitor( - 'abc123', 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False + {'integration_key': 'abc123'}, + 'config.yaml', + module.monitor.State.START, + monitoring_log_level=1, + dry_run=False, ) @@ -15,7 +19,11 @@ def test_ping_monitor_ignores_finish_state(): flexmock(module.requests).should_receive('post').never() module.ping_monitor( - 'abc123', 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False + {'integration_key': 'abc123'}, + 'config.yaml', + module.monitor.State.FINISH, + monitoring_log_level=1, + dry_run=False, ) @@ -23,7 +31,11 @@ def test_ping_monitor_calls_api_for_fail_state(): flexmock(module.requests).should_receive('post') module.ping_monitor( - 'abc123', 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + {'integration_key': 'abc123'}, + 'config.yaml', + module.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, ) @@ -31,5 +43,9 @@ def test_ping_monitor_dry_run_does_not_call_api(): flexmock(module.requests).should_receive('post').never() module.ping_monitor( - 'abc123', 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=True + {'integration_key': 'abc123'}, + 'config.yaml', + module.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=True, )