Rename retry_timeout to retry_wait and standardize log formatting (#28).

This commit is contained in:
Dan Helfman 2021-11-15 11:51:17 -08:00
parent 180018fd81
commit 38ebfd2969
4 changed files with 21 additions and 17 deletions

1
NEWS
View file

@ -1,4 +1,5 @@
1.5.21.dev0 1.5.21.dev0
* #28: Optionally retry failing backups via "retries" and "retry_wait" configuration options.
* #459: Add support for old version (2.x) of jsonschema library. * #459: Add support for old version (2.x) of jsonschema library.
1.5.20 1.5.20

View file

@ -55,7 +55,7 @@ def run_configuration(config_filename, config, arguments):
local_path = location.get('local_path', 'borg') local_path = location.get('local_path', 'borg')
remote_path = location.get('remote_path') remote_path = location.get('remote_path')
retries = storage.get('retries', 0) retries = storage.get('retries', 0)
retry_timeout = storage.get('retry_timeout', 0) retry_wait = storage.get('retry_wait', 0)
borg_environment.initialize(storage) borg_environment.initialize(storage)
encountered_error = None encountered_error = None
error_repository = '' error_repository = ''
@ -130,9 +130,9 @@ def run_configuration(config_filename, config, arguments):
while not repo_queue.empty(): while not repo_queue.empty():
repository_path, retry_num = repo_queue.get() repository_path, retry_num = repo_queue.get()
timeout = retry_num * retry_timeout timeout = retry_num * retry_wait
if timeout: if timeout:
logger.warning(f'Sleeping {timeout}s before next retry') logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry')
time.sleep(timeout) time.sleep(timeout)
try: try:
yield from run_actions( yield from run_actions(
@ -152,7 +152,9 @@ def run_configuration(config_filename, config, arguments):
) )
if retry_num < retries: if retry_num < retries:
repo_queue.put((repository_path, retry_num + 1),) repo_queue.put((repository_path, retry_num + 1),)
logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}') logger.warning(
f'{config_filename}: Retrying... attempt {retry_num + 1}/{retries}'
)
continue continue
encountered_error = error encountered_error = error
error_repository = repository_path error_repository = repository_path

View file

@ -254,14 +254,15 @@ properties:
retries: retries:
type: integer type: integer
description: | description: |
Number of times to retry a backup before failing. Defaults Number of times to retry a failing backup before giving up.
to 0 (i.e. does not attempt retry). Defaults to 0 (i.e., does not attempt retry).
example: 3 example: 3
retry_timeout: retry_wait:
type: integer type: integer
description: | description: |
Wait time between retries, to allow transient issues to pass Wait time between retries (in seconds) to allow transient
Defaults to 0s. issues to pass. Increases after each retry as a form of
backoff. Defaults to 0 (no wait).
example: 10 example: 10
temporary_directory: temporary_directory:
type: string type: string

View file

@ -185,7 +185,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure():
assert results == expected_results assert results == expected_results
def test_run_retries_soft_error(): def test_run_configuration_retries_soft_error():
# Run action first fails, second passes # Run action first fails, second passes
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
@ -198,7 +198,7 @@ def test_run_retries_soft_error():
assert results == expected_results assert results == expected_results
def test_run_retries_hard_error(): def test_run_configuration_retries_hard_error():
# Run action fails twice # Run action fails twice
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
@ -234,7 +234,7 @@ def test_run_repos_ordered():
assert results == expected_results assert results == expected_results
def test_run_retries_round_robbin(): def test_run_configuration_retries_round_robbin():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
@ -257,7 +257,7 @@ def test_run_retries_round_robbin():
assert results == expected_results assert results == expected_results
def test_run_retries_one_passes(): def test_run_configuration_retries_one_passes():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
@ -279,7 +279,7 @@ def test_run_retries_one_passes():
assert results == expected_results assert results == expected_results
def test_run_retry_timeout(): def test_run_configuration_retry_wait():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
@ -302,13 +302,13 @@ def test_run_retry_timeout():
flexmock(module).should_receive('make_error_log_records').with_args( flexmock(module).should_receive('make_error_log_records').with_args(
'foo: Error running actions for repository', OSError 'foo: Error running actions for repository', OSError
).and_return(expected_results[3:4]).ordered() ).and_return(expected_results[3:4]).ordered()
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_timeout': 10}} config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_wait': 10}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results assert results == expected_results
def test_run_retries_timeout_multiple_repos(): def test_run_configuration_retries_timeout_multiple_repos():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
@ -332,7 +332,7 @@ def test_run_retries_timeout_multiple_repos():
).and_return(expected_results[2:3]).ordered() ).and_return(expected_results[2:3]).ordered()
config = { config = {
'location': {'repositories': ['foo', 'bar']}, 'location': {'repositories': ['foo', 'bar']},
'storage': {'retries': 1, 'retry_timeout': 10}, 'storage': {'retries': 1, 'retry_wait': 10},
} }
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))