diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index b3796c6..1bf601c 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -126,9 +126,7 @@ def run_configuration(config_filename, config, arguments): if not encountered_error: repo_queue = Queue() for repo in location['repositories']: - repo_queue.put( - (repo, 0), - ) + repo_queue.put((repo, 0),) while not repo_queue.empty(): repository_path, retry_num = repo_queue.get() @@ -153,9 +151,7 @@ def run_configuration(config_filename, config, arguments): '{}: Error running actions for repository'.format(repository_path), error ) if retry_num < retries: - repo_queue.put( - (repository_path, retry_num + 1), - ) + repo_queue.put((repository_path, retry_num + 1),) logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}') continue encountered_error = error diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index ae92f9b..0cc9074 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -260,7 +260,7 @@ properties: retry_timeout: type: integer description: | - Wait time between retries, to allow transient issues to pass. + Wait time between retries, to allow transient issues to pass Defaults to 0s. example: 10 temporary_directory: diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 5e67ee9..bb81a94 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -184,6 +184,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): assert results == expected_results + def test_run_retries_soft_error(): # Run action first fails, second passes flexmock(module.borg_environment).should_receive('initialize') @@ -191,119 +192,153 @@ def test_run_retries_soft_error(): flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([]) expected_results = [flexmock()] flexmock(module).should_receive('make_error_log_records').and_return(expected_results).once() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries':1}} + config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_retries_hard_error(): # Run action fails twice flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) expected_results = [flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[:1]) \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:]).twice() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries':1}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[:1]).with_args( + 'foo: Error running actions for repository', OSError + ).and_return( + expected_results[1:] + ).twice() + config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_repos_ordered(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) expected_results = [flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[:1]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:]).ordered() - config = {'location': {'repositories': ['foo','bar']}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[:1]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[1:]).ordered() + config = {'location': {'repositories': ['foo', 'bar']}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_retries_round_robbin(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) expected_results = [flexmock(), flexmock(), flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered() - config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[1:2]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[2:3]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[3:4]).ordered() + config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_retries_one_passes(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') - flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4) + flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( + [] + ).and_raise(OSError).times(4) expected_results = [flexmock(), flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() - config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[1:2]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[2:3]).ordered() + config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_retry_timeout(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) expected_results = [flexmock(), flexmock(), flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[0:1]).ordered() flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[1:2]).ordered() flexmock(time).should_receive('sleep').with_args(20).and_return().ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[2:3]).ordered() flexmock(time).should_receive('sleep').with_args(30).and_return().ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries':3, 'retry_timeout':10}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[3:4]).ordered() + config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_timeout': 10}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_retries_timeout_multiple_repos(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') - flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4) + flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( + [] + ).and_raise(OSError).times(4) expected_results = [flexmock(), flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[1:2]).ordered() # Sleep before retrying foo (and passing) flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() - + # Sleep before retrying bar (and failing) flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() - config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1, 'retry_timeout':10}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[2:3]).ordered() + config = { + 'location': {'repositories': ['foo', 'bar']}, + 'storage': {'retries': 1, 'retry_timeout': 10}, + } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_load_configurations_collects_parsed_configurations(): configuration = flexmock() other_configuration = flexmock()