Formatting
This commit is contained in:
parent
b4117916b8
commit
976a877a25
3 changed files with 83 additions and 52 deletions
|
@ -126,9 +126,7 @@ def run_configuration(config_filename, config, arguments):
|
||||||
if not encountered_error:
|
if not encountered_error:
|
||||||
repo_queue = Queue()
|
repo_queue = Queue()
|
||||||
for repo in location['repositories']:
|
for repo in location['repositories']:
|
||||||
repo_queue.put(
|
repo_queue.put((repo, 0),)
|
||||||
(repo, 0),
|
|
||||||
)
|
|
||||||
|
|
||||||
while not repo_queue.empty():
|
while not repo_queue.empty():
|
||||||
repository_path, retry_num = repo_queue.get()
|
repository_path, retry_num = repo_queue.get()
|
||||||
|
@ -153,9 +151,7 @@ def run_configuration(config_filename, config, arguments):
|
||||||
'{}: Error running actions for repository'.format(repository_path), error
|
'{}: Error running actions for repository'.format(repository_path), error
|
||||||
)
|
)
|
||||||
if retry_num < retries:
|
if retry_num < retries:
|
||||||
repo_queue.put(
|
repo_queue.put((repository_path, retry_num + 1),)
|
||||||
(repository_path, retry_num + 1),
|
|
||||||
)
|
|
||||||
logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}')
|
logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}')
|
||||||
continue
|
continue
|
||||||
encountered_error = error
|
encountered_error = error
|
||||||
|
|
|
@ -260,7 +260,7 @@ properties:
|
||||||
retry_timeout:
|
retry_timeout:
|
||||||
type: integer
|
type: integer
|
||||||
description: |
|
description: |
|
||||||
Wait time between retries, to allow transient issues to pass.
|
Wait time between retries, to allow transient issues to pass
|
||||||
Defaults to 0s.
|
Defaults to 0s.
|
||||||
example: 10
|
example: 10
|
||||||
temporary_directory:
|
temporary_directory:
|
||||||
|
|
|
@ -184,6 +184,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure():
|
||||||
|
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retries_soft_error():
|
def test_run_retries_soft_error():
|
||||||
# Run action first fails, second passes
|
# Run action first fails, second passes
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
|
@ -196,114 +197,148 @@ def test_run_retries_soft_error():
|
||||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retries_hard_error():
|
def test_run_retries_hard_error():
|
||||||
# Run action fails twice
|
# Run action fails twice
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
|
flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
|
||||||
expected_results = [flexmock(), flexmock()]
|
expected_results = [flexmock(), flexmock()]
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[:1]) \
|
'foo: Error running actions for repository', OSError
|
||||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:]).twice()
|
).and_return(expected_results[:1]).with_args(
|
||||||
|
'foo: Error running actions for repository', OSError
|
||||||
|
).and_return(
|
||||||
|
expected_results[1:]
|
||||||
|
).twice()
|
||||||
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}}
|
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}}
|
||||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_repos_ordered():
|
def test_run_repos_ordered():
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
|
flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
|
||||||
expected_results = [flexmock(), flexmock()]
|
expected_results = [flexmock(), flexmock()]
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[:1]).ordered()
|
'foo: Error running actions for repository', OSError
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
).and_return(expected_results[:1]).ordered()
|
||||||
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:]).ordered()
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
|
'bar: Error running actions for repository', OSError
|
||||||
|
).and_return(expected_results[1:]).ordered()
|
||||||
config = {'location': {'repositories': ['foo', 'bar']}}
|
config = {'location': {'repositories': ['foo', 'bar']}}
|
||||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retries_round_robbin():
|
def test_run_retries_round_robbin():
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
|
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
|
||||||
expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
|
expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered()
|
'foo: Error running actions for repository', OSError
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
).and_return(expected_results[0:1]).ordered()
|
||||||
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered()
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
'bar: Error running actions for repository', OSError
|
||||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered()
|
).and_return(expected_results[1:2]).ordered()
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered()
|
'foo: Error running actions for repository', OSError
|
||||||
|
).and_return(expected_results[2:3]).ordered()
|
||||||
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
|
'bar: Error running actions for repository', OSError
|
||||||
|
).and_return(expected_results[3:4]).ordered()
|
||||||
config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
|
config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
|
||||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retries_one_passes():
|
def test_run_retries_one_passes():
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4)
|
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
|
||||||
|
[]
|
||||||
|
).and_raise(OSError).times(4)
|
||||||
expected_results = [flexmock(), flexmock(), flexmock()]
|
expected_results = [flexmock(), flexmock(), flexmock()]
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered()
|
'foo: Error running actions for repository', OSError
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
).and_return(expected_results[0:1]).ordered()
|
||||||
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered()
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
'bar: Error running actions for repository', OSError
|
||||||
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered()
|
).and_return(expected_results[1:2]).ordered()
|
||||||
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
|
'bar: Error running actions for repository', OSError
|
||||||
|
).and_return(expected_results[2:3]).ordered()
|
||||||
config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
|
config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
|
||||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retry_timeout():
|
def test_run_retry_timeout():
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
|
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
|
||||||
expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
|
expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered()
|
'foo: Error running actions for repository', OSError
|
||||||
|
).and_return(expected_results[0:1]).ordered()
|
||||||
|
|
||||||
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
|
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered()
|
'foo: Error running actions for repository', OSError
|
||||||
|
).and_return(expected_results[1:2]).ordered()
|
||||||
|
|
||||||
flexmock(time).should_receive('sleep').with_args(20).and_return().ordered()
|
flexmock(time).should_receive('sleep').with_args(20).and_return().ordered()
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered()
|
'foo: Error running actions for repository', OSError
|
||||||
|
).and_return(expected_results[2:3]).ordered()
|
||||||
|
|
||||||
flexmock(time).should_receive('sleep').with_args(30).and_return().ordered()
|
flexmock(time).should_receive('sleep').with_args(30).and_return().ordered()
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered()
|
'foo: Error running actions for repository', OSError
|
||||||
|
).and_return(expected_results[3:4]).ordered()
|
||||||
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_timeout': 10}}
|
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_timeout': 10}}
|
||||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retries_timeout_multiple_repos():
|
def test_run_retries_timeout_multiple_repos():
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4)
|
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
|
||||||
|
[]
|
||||||
|
).and_raise(OSError).times(4)
|
||||||
expected_results = [flexmock(), flexmock(), flexmock()]
|
expected_results = [flexmock(), flexmock(), flexmock()]
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered()
|
'foo: Error running actions for repository', OSError
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
).and_return(expected_results[0:1]).ordered()
|
||||||
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered()
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
|
'bar: Error running actions for repository', OSError
|
||||||
|
).and_return(expected_results[1:2]).ordered()
|
||||||
|
|
||||||
# Sleep before retrying foo (and passing)
|
# Sleep before retrying foo (and passing)
|
||||||
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
|
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
|
||||||
|
|
||||||
# Sleep before retrying bar (and failing)
|
# Sleep before retrying bar (and failing)
|
||||||
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
|
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
|
||||||
flexmock(module).should_receive('make_error_log_records') \
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered()
|
'bar: Error running actions for repository', OSError
|
||||||
config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1, 'retry_timeout':10}}
|
).and_return(expected_results[2:3]).ordered()
|
||||||
|
config = {
|
||||||
|
'location': {'repositories': ['foo', 'bar']},
|
||||||
|
'storage': {'retries': 1, 'retry_timeout': 10},
|
||||||
|
}
|
||||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_load_configurations_collects_parsed_configurations():
|
def test_load_configurations_collects_parsed_configurations():
|
||||||
configuration = flexmock()
|
configuration = flexmock()
|
||||||
other_configuration = flexmock()
|
other_configuration = flexmock()
|
||||||
|
|
Loading…
Reference in a new issue