Add an "upload_buffer_size" option to set the size of the upload buffer used in "create" action (#865).

This commit is contained in:
Dan Helfman 2024-06-23 16:26:22 -07:00
parent a2c139245d
commit f4fcf92bd6
4 changed files with 10 additions and 0 deletions

2
NEWS
View file

@ -5,6 +5,8 @@
* #860: Fix interaction between environment variable interpolation in constants and shell escaping. * #860: Fix interaction between environment variable interpolation in constants and shell escaping.
* #863: When color output is disabled (explicitly or implicitly), don't prefix each log line with * #863: When color output is disabled (explicitly or implicitly), don't prefix each log line with
the log level. the log level.
* #865: Add an "upload_buffer_size" option to set the size of the upload buffer used in "create"
action.
* #866: Fix "Argument list too long" error in the "spot" check when checking hundreds of thousands * #866: Fix "Argument list too long" error in the "spot" check when checking hundreds of thousands
of files at once. of files at once.
* #874: Add the configured repository label as "repository_label" to the interpolated variables * #874: Add the configured repository label as "repository_label" to the interpolated variables

View file

@ -371,6 +371,7 @@ def make_base_create_command(
chunker_params = config.get('chunker_params', None) chunker_params = config.get('chunker_params', None)
compression = config.get('compression', None) compression = config.get('compression', None)
upload_rate_limit = config.get('upload_rate_limit', None) upload_rate_limit = config.get('upload_rate_limit', None)
upload_buffer_size = config.get('upload_buffer_size', None)
umask = config.get('umask', None) umask = config.get('umask', None)
lock_wait = config.get('lock_wait', None) lock_wait = config.get('lock_wait', None)
list_filter_flags = make_list_filter_flags(local_borg_version, dry_run) list_filter_flags = make_list_filter_flags(local_borg_version, dry_run)
@ -412,6 +413,7 @@ def make_base_create_command(
+ (('--chunker-params', chunker_params) if chunker_params else ()) + (('--chunker-params', chunker_params) if chunker_params else ())
+ (('--compression', compression) if compression else ()) + (('--compression', compression) if compression else ())
+ upload_ratelimit_flags + upload_ratelimit_flags
+ (('--upload-buffer', str(upload_buffer_size)) if upload_buffer_size else ())
+ (('--one-file-system',) if config.get('one_file_system') or stream_processes else ()) + (('--one-file-system',) if config.get('one_file_system') or stream_processes else ())
+ numeric_ids_flags + numeric_ids_flags
+ atime_flags + atime_flags

View file

@ -280,6 +280,11 @@ properties:
Remote network upload rate limit in kiBytes/second. Defaults to Remote network upload rate limit in kiBytes/second. Defaults to
unlimited. unlimited.
example: 100 example: 100
upload_buffer_size:
type: integer
description: |
Size of network upload buffer in MiB. Defaults to no buffer.
example: 160
retries: retries:
type: integer type: integer
description: | description: |

View file

@ -693,6 +693,7 @@ def test_make_base_create_command_includes_exclude_patterns_in_borg_command():
('one_file_system', True, True, ('--one-file-system',)), ('one_file_system', True, True, ('--one-file-system',)),
('upload_rate_limit', 100, True, ('--upload-ratelimit', '100')), ('upload_rate_limit', 100, True, ('--upload-ratelimit', '100')),
('upload_rate_limit', 100, False, ('--remote-ratelimit', '100')), ('upload_rate_limit', 100, False, ('--remote-ratelimit', '100')),
('upload_buffer_size', 160, True, ('--upload-buffer', '160')),
('numeric_ids', True, True, ('--numeric-ids',)), ('numeric_ids', True, True, ('--numeric-ids',)),
('numeric_ids', True, False, ('--numeric-owner',)), ('numeric_ids', True, False, ('--numeric-owner',)),
('read_special', True, True, ('--read-special',)), ('read_special', True, True, ('--read-special',)),