Add queue based retry logic
This commit is contained in:
parent
9c972cb0e5
commit
4cc4b8d484
2 changed files with 20 additions and 3 deletions
|
@ -5,6 +5,7 @@ import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
from subprocess import CalledProcessError
|
from subprocess import CalledProcessError
|
||||||
|
from queue import Queue
|
||||||
|
|
||||||
import colorama
|
import colorama
|
||||||
import pkg_resources
|
import pkg_resources
|
||||||
|
@ -52,6 +53,7 @@ def run_configuration(config_filename, config, arguments):
|
||||||
|
|
||||||
local_path = location.get('local_path', 'borg')
|
local_path = location.get('local_path', 'borg')
|
||||||
remote_path = location.get('remote_path')
|
remote_path = location.get('remote_path')
|
||||||
|
retries = storage.get('retries',1)
|
||||||
borg_environment.initialize(storage)
|
borg_environment.initialize(storage)
|
||||||
encountered_error = None
|
encountered_error = None
|
||||||
error_repository = ''
|
error_repository = ''
|
||||||
|
@ -120,7 +122,12 @@ def run_configuration(config_filename, config, arguments):
|
||||||
)
|
)
|
||||||
|
|
||||||
if not encountered_error:
|
if not encountered_error:
|
||||||
for repository_path in location['repositories']:
|
repo_queue = Queue()
|
||||||
|
for repo in location['repositories']:
|
||||||
|
repo_queue.put((repo,0),)
|
||||||
|
|
||||||
|
while not repo_queue.empty():
|
||||||
|
repository_path, retry_num = repo_queue.get()
|
||||||
try:
|
try:
|
||||||
yield from run_actions(
|
yield from run_actions(
|
||||||
arguments=arguments,
|
arguments=arguments,
|
||||||
|
@ -134,11 +141,15 @@ def run_configuration(config_filename, config, arguments):
|
||||||
repository_path=repository_path,
|
repository_path=repository_path,
|
||||||
)
|
)
|
||||||
except (OSError, CalledProcessError, ValueError) as error:
|
except (OSError, CalledProcessError, ValueError) as error:
|
||||||
encountered_error = error
|
|
||||||
error_repository = repository_path
|
|
||||||
yield from make_error_log_records(
|
yield from make_error_log_records(
|
||||||
'{}: Error running actions for repository'.format(repository_path), error
|
'{}: Error running actions for repository'.format(repository_path), error
|
||||||
)
|
)
|
||||||
|
if retry_num < retries:
|
||||||
|
repo_queue.put((repository_path,retry_num + 1),)
|
||||||
|
logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}')
|
||||||
|
continue
|
||||||
|
encountered_error = error
|
||||||
|
error_repository = repository_path
|
||||||
|
|
||||||
if not encountered_error:
|
if not encountered_error:
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -249,6 +249,12 @@ properties:
|
||||||
Remote network upload rate limit in kiBytes/second. Defaults
|
Remote network upload rate limit in kiBytes/second. Defaults
|
||||||
to unlimited.
|
to unlimited.
|
||||||
example: 100
|
example: 100
|
||||||
|
retries:
|
||||||
|
type: integer
|
||||||
|
description: |
|
||||||
|
Number of times to retry a backup before failing. Defaults
|
||||||
|
to 1.
|
||||||
|
example: 3
|
||||||
temporary_directory:
|
temporary_directory:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
|
|
Loading…
Reference in a new issue