Add queue based retry logic

This commit is contained in:
cadamswaite 2021-07-14 22:46:02 +01:00
parent 9c972cb0e5
commit 4cc4b8d484
2 changed files with 20 additions and 3 deletions

View file

@ -5,6 +5,7 @@ import logging
import os import os
import sys import sys
from subprocess import CalledProcessError from subprocess import CalledProcessError
from queue import Queue
import colorama import colorama
import pkg_resources import pkg_resources
@ -52,6 +53,7 @@ def run_configuration(config_filename, config, arguments):
local_path = location.get('local_path', 'borg') local_path = location.get('local_path', 'borg')
remote_path = location.get('remote_path') remote_path = location.get('remote_path')
retries = storage.get('retries',1)
borg_environment.initialize(storage) borg_environment.initialize(storage)
encountered_error = None encountered_error = None
error_repository = '' error_repository = ''
@ -120,7 +122,12 @@ def run_configuration(config_filename, config, arguments):
) )
if not encountered_error: if not encountered_error:
for repository_path in location['repositories']: repo_queue = Queue()
for repo in location['repositories']:
repo_queue.put((repo,0),)
while not repo_queue.empty():
repository_path, retry_num = repo_queue.get()
try: try:
yield from run_actions( yield from run_actions(
arguments=arguments, arguments=arguments,
@ -134,11 +141,15 @@ def run_configuration(config_filename, config, arguments):
repository_path=repository_path, repository_path=repository_path,
) )
except (OSError, CalledProcessError, ValueError) as error: except (OSError, CalledProcessError, ValueError) as error:
encountered_error = error
error_repository = repository_path
yield from make_error_log_records( yield from make_error_log_records(
'{}: Error running actions for repository'.format(repository_path), error '{}: Error running actions for repository'.format(repository_path), error
) )
if retry_num < retries:
repo_queue.put((repository_path,retry_num + 1),)
logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}')
continue
encountered_error = error
error_repository = repository_path
if not encountered_error: if not encountered_error:
try: try:

View file

@ -249,6 +249,12 @@ properties:
Remote network upload rate limit in kiBytes/second. Defaults Remote network upload rate limit in kiBytes/second. Defaults
to unlimited. to unlimited.
example: 100 example: 100
retries:
type: integer
description: |
Number of times to retry a backup before failing. Defaults
to 1.
example: 3
temporary_directory: temporary_directory:
type: string type: string
description: | description: |