From f4fcf92bd63c96672cc1aef36370576bde93d2f5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Jun 2024 16:26:22 -0700 Subject: [PATCH] Add an "upload_buffer_size" option to set the size of the upload buffer used in "create" action (#865). --- NEWS | 2 ++ borgmatic/borg/create.py | 2 ++ borgmatic/config/schema.yaml | 5 +++++ tests/unit/borg/test_create.py | 1 + 4 files changed, 10 insertions(+) diff --git a/NEWS b/NEWS index 31ff50f..ed84732 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,8 @@ * #860: Fix interaction between environment variable interpolation in constants and shell escaping. * #863: When color output is disabled (explicitly or implicitly), don't prefix each log line with the log level. + * #865: Add an "upload_buffer_size" option to set the size of the upload buffer used in "create" + action. * #866: Fix "Argument list too long" error in the "spot" check when checking hundreds of thousands of files at once. * #874: Add the configured repository label as "repository_label" to the interpolated variables diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 1cb9563..b39f72e 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -371,6 +371,7 @@ def make_base_create_command( chunker_params = config.get('chunker_params', None) compression = config.get('compression', None) upload_rate_limit = config.get('upload_rate_limit', None) + upload_buffer_size = config.get('upload_buffer_size', None) umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) list_filter_flags = make_list_filter_flags(local_borg_version, dry_run) @@ -412,6 +413,7 @@ def make_base_create_command( + (('--chunker-params', chunker_params) if chunker_params else ()) + (('--compression', compression) if compression else ()) + upload_ratelimit_flags + + (('--upload-buffer', str(upload_buffer_size)) if upload_buffer_size else ()) + (('--one-file-system',) if config.get('one_file_system') or stream_processes else ()) + numeric_ids_flags + atime_flags diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 5d02771..56a8987 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -280,6 +280,11 @@ properties: Remote network upload rate limit in kiBytes/second. Defaults to unlimited. example: 100 + upload_buffer_size: + type: integer + description: | + Size of network upload buffer in MiB. Defaults to no buffer. + example: 160 retries: type: integer description: | diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 3e3dd13..9959919 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -693,6 +693,7 @@ def test_make_base_create_command_includes_exclude_patterns_in_borg_command(): ('one_file_system', True, True, ('--one-file-system',)), ('upload_rate_limit', 100, True, ('--upload-ratelimit', '100')), ('upload_rate_limit', 100, False, ('--remote-ratelimit', '100')), + ('upload_buffer_size', 160, True, ('--upload-buffer', '160')), ('numeric_ids', True, True, ('--numeric-ids',)), ('numeric_ids', True, False, ('--numeric-owner',)), ('read_special', True, True, ('--read-special',)),