Revamp "borg" action to support REPOSITORY and ARCHIVE env vars instead of implicitly injecting repository/archive into the Borg command (#575).

This commit is contained in:
Dan Helfman 2023-06-26 14:35:07 -07:00
parent b242078f54
commit bb6004fc4f
5 changed files with 140 additions and 137 deletions

6
NEWS
View file

@ -1,4 +1,8 @@
1.7.16.dev0
1.8.0.dev0
* #575: BREAKING: For the "borgmatic borg" action, instead of implicitly injecting
repository/archive into the resulting Borg command-line, make repository and archive environment
variables available for explicit use in your commands. See the documentation for more
information: https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/
* #719: Fix an error when running "borg key export" through borgmatic.
1.7.15

View file

@ -10,7 +10,6 @@ logger = logging.getLogger(__name__)
REPOSITORYLESS_BORG_COMMANDS = {'serve', None}
BORG_SUBCOMMANDS_WITH_SUBCOMMANDS = {'key', 'debug'}
BORG_SUBCOMMANDS_WITHOUT_REPOSITORY = (('debug', 'info'), ('debug', 'convert-profile'), ())
def run_arbitrary_borg(
@ -25,7 +24,8 @@ def run_arbitrary_borg(
'''
Given a local or remote repository path, a storage config dict, the local Borg version, a
sequence of arbitrary command-line Borg options, and an optional archive name, run an arbitrary
Borg command on the given repository/archive.
Borg command, passing in $REPOSITORY and $ARCHIVE environment variables for optional use in the
commmand.
'''
borgmatic.logger.add_custom_log_levels()
lock_wait = storage_config.get('lock_wait', None)
@ -46,29 +46,26 @@ def run_arbitrary_borg(
borg_command = ()
command_options = ()
if borg_command in BORG_SUBCOMMANDS_WITHOUT_REPOSITORY:
repository_archive_flags = ()
elif archive:
repository_archive_flags = flags.make_repository_archive_flags(
repository_path, archive, local_borg_version
)
else:
repository_archive_flags = flags.make_repository_flags(repository_path, local_borg_version)
full_command = (
(local_path,)
+ borg_command
+ repository_archive_flags
+ command_options
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+ flags.make_flags('remote-path', remote_path)
+ flags.make_flags('lock-wait', lock_wait)
+ command_options
)
return execute_command(
full_command,
output_file=DO_NOT_CAPTURE,
borg_local_path=local_path,
extra_environment=environment.make_environment(storage_config),
shell=True,
extra_environment=dict(
(environment.make_environment(storage_config) or {}),
**{
'REPOSITORY': repository_path,
'ARCHIVE': archive if archive else '',
},
),
)

View file

@ -7,7 +7,7 @@ eleventyNavigation:
---
## Running Borg with borgmatic
Borg has several commands (and options) that borgmatic does not currently
Borg has several commands and options that borgmatic does not currently
support. Sometimes though, as a borgmatic user, you may find yourself wanting
to take advantage of these off-the-beaten-path Borg features. You could of
course drop down to running Borg directly. But then you'd give up all the
@ -17,11 +17,11 @@ request](https://torsion.org/borgmatic/#contributing) to add the feature. But
what if you need it *now*?
That's where borgmatic's support for running "arbitrary" Borg commands comes
in. Running Borg commands with borgmatic takes advantage of the following, all
based on your borgmatic configuration files or command-line arguments:
in. Running these Borg commands with borgmatic can take advantage of the
following, all based on your borgmatic configuration files or command-line
arguments:
* configured repositories (automatically runs your Borg command once for each
one)
* configured repositories, running your Borg command once for each one
* local and remote Borg binary paths
* SSH settings and Borg environment variables
* lock wait settings
@ -33,37 +33,78 @@ based on your borgmatic configuration files or command-line arguments:
<span class="minilink minilink-addedin">New in version 1.5.15</span> The way
you run Borg with borgmatic is via the `borg` action. Here's a simple example:
```bash
borgmatic borg break-lock '$REPOSITORY'
```
This runs Borg's `break-lock` command once on each configured borgmatic
repository, passing the repository path in as an environment variable named
`REPOSITORY`. The single quotes are necessary in order to pass in a literal
`$REPOSITORY` string instead of trying to resolve it from borgmatic's shell
where it's not yet set.
<span class="minilink minilink-addedin">Prior to version 1.8.0</span>borgmatic
provided the repository name implicitly, attempting to inject it into your
Borg arguments in the right place (which didn't always work). So your
command-line in these older versions looked more like:
```bash
borgmatic borg break-lock
```
(No `borg` action in borgmatic? Time to upgrade!)
This runs Borg's `break-lock` command once on each configured borgmatic
repository. Notice how the repository isn't present in the specified Borg
options, as that part is provided by borgmatic.
You can also specify Borg options for relevant commands:
You can also specify Borg options for relevant commands. In borgmatic 1.8.0+,
that looks like:
```bash
borgmatic borg rlist --short
borgmatic borg rlist --short '$REPOSITORY'
```
This runs Borg's `rlist` command once on each configured borgmatic repository.
(The native `borgmatic rlist` action should be preferred for most use.)
However, the native `borgmatic rlist` action should be preferred for most uses.
What if you only want to run Borg on a single configured borgmatic repository
when you've got several configured? Not a problem. The `--repository` argument
lets you specify the repository to use, either by its path or its label:
```bash
borgmatic borg --repository repo.borg break-lock
borgmatic borg --repository repo.borg break-lock '$REPOSITORY'
```
And what about a single archive?
### Specifying an archive
For borg commands that expect an archive name, you have a few approaches.
Here's one:
```bash
borgmatic borg --archive your-archive-name rlist
borgmatic borg --archive latest list '$REPOSITORY::$ARCHIVE'
```
Or if you don't need borgmatic to resolve an archive name like `latest`, you
can just do:
```bash
borgmatic borg list '$REPOSITORY::your-actual-archive-name'
```
<span class="minilink minilink-addedin">Prior to version 1.8.0</span>borgmatic
provided the archive name implicitly along with the repository, attempting to
inject it into your Borg arguments in the right place (which didn't always
work). So your command-line in these older versions of borgmatic looked more
like:
```bash
borgmatic borg --archive latest list
```
<span class="minilink minilink-addedin">With Borg version 2.x</span> Either of
these will list an archive:
```bash
borgmatic borg --archive latest list --repo '$REPOSITORY' '$ARCHIVE'
```
```bash
borgmatic borg list --repo '$REPOSITORY' your-actual-archive-name
```
### Limitations
@ -71,14 +112,10 @@ borgmatic borg --archive your-archive-name rlist
borgmatic's `borg` action is not without limitations:
* The Borg command you want to run (`create`, `list`, etc.) *must* come first
after the `borg` action. If you have any other Borg options to specify,
provide them after. For instance, `borgmatic borg list --progress` will work,
but `borgmatic borg --progress list` will not.
* borgmatic supplies the repository/archive name to Borg for you (based on
your borgmatic configuration or the `borgmatic borg --repository`/`--archive`
arguments), so do not specify the repository/archive otherwise.
* The `borg` action will not currently work for any Borg commands like `borg
serve` that do not accept a repository/archive name.
after the `borg` action (and any borgmatic-specific arguments). If you have
other Borg options to specify, provide them after. For instance,
`borgmatic borg list --progress ...` will work, but
`borgmatic borg --progress list ...` will not.
* Do not specify any global borgmatic arguments to the right of the `borg`
action. (They will be passed to Borg instead of borgmatic.) If you have
global borgmatic arguments, specify them *before* the `borg` action.
@ -88,10 +125,17 @@ borgmatic's `borg` action is not without limitations:
borgmatic action. In this case, only the Borg command is run.
* Unlike normal borgmatic actions that support JSON, the `borg` action will
not disable certain borgmatic logs to avoid interfering with JSON output.
* Unlike other borgmatic actions, the `borg` action captures (and logs) all
output, so interactive prompts and flags like `--progress` will not work as
expected. <span class="minilink minilink-addedin">New in version
1.7.13</span> borgmatic now runs the `borg` action without capturing output,
* <span class="minilink minilink-addedin">Prior to version 1.8.0</span>
borgmatic implicitly supplied the repository/archive name to Borg for you
(based on your borgmatic configuration or the
`borgmatic borg --repository`/`--archive` arguments)—which meant you couldn't
specify the repository/archive directly in the Borg command. Also, in these
older versions of borgmatic, the `borg` action didn't work for any Borg
commands like `borg serve` that do not accept a repository/archive name.
* <span class="minilink minilink-addedin">Prior to version 1.7.13</span> Unlike
other borgmatic actions, the `borg` action captured (and logged) all output,
so interactive prompts and flags like `--progress` dit not work as expected.
In new versions, borgmatic runs the `borg` action without capturing output,
so interactive prompts work.
In general, this `borgmatic borg` feature should be considered an escape

View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup
VERSION = '1.7.16.dev0'
VERSION = '1.8.0.dev0'
setup(

View file

@ -10,35 +10,35 @@ from ..test_verbosity import insert_logging_mock
def test_run_arbitrary_borg_calls_borg_with_flags():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', 'repo'),
('borg', 'break-lock', '$REPOSITORY'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''},
)
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['break-lock'],
options=['break-lock', '$REPOSITORY'],
)
def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', 'repo', '--info'),
('borg', 'break-lock', '--info', '$REPOSITORY'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''},
)
insert_logging_mock(logging.INFO)
@ -46,21 +46,21 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag():
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['break-lock'],
options=['break-lock', '$REPOSITORY'],
)
def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', 'repo', '--debug', '--show-rc'),
('borg', 'break-lock', '--debug', '--show-rc', '$REPOSITORY'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''},
)
insert_logging_mock(logging.DEBUG)
@ -68,7 +68,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag():
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['break-lock'],
options=['break-lock', '$REPOSITORY'],
)
@ -76,46 +76,44 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
storage_config = {'lock_wait': 5}
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(
('--lock-wait', '5')
)
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', 'repo', '--lock-wait', '5'),
('borg', 'break-lock', '--lock-wait', '5', '$REPOSITORY'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''},
)
module.run_arbitrary_borg(
repository_path='repo',
storage_config=storage_config,
local_borg_version='1.2.3',
options=['break-lock'],
options=['break-lock', '$REPOSITORY'],
)
def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
('repo::archive',)
)
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', 'repo::archive'),
('borg', 'break-lock', '$REPOSITORY::$ARCHIVE'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': 'archive'},
)
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['break-lock'],
options=['break-lock', '$REPOSITORY::$ARCHIVE'],
archive='archive',
)
@ -123,21 +121,21 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag():
def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg1', 'break-lock', 'repo'),
('borg1', 'break-lock', '$REPOSITORY'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg1',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''},
)
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['break-lock'],
options=['break-lock', '$REPOSITORY'],
local_path='borg1',
)
@ -145,23 +143,23 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.flags).should_receive('make_flags').and_return(
('--remote-path', 'borg1')
).and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', 'repo', '--remote-path', 'borg1'),
('borg', 'break-lock', '--remote-path', 'borg1', '$REPOSITORY'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''},
)
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['break-lock'],
options=['break-lock', '$REPOSITORY'],
remote_path='borg1',
)
@ -169,56 +167,56 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags()
def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', 'repo', '--progress'),
('borg', 'list', '--progress', '$REPOSITORY'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''},
)
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['list', '--progress'],
options=['list', '--progress', '$REPOSITORY'],
)
def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', 'repo'),
('borg', 'break-lock', '$REPOSITORY'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''},
)
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['--', 'break-lock'],
options=['--', 'break-lock', '$REPOSITORY'],
)
def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').never()
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg',),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''},
)
module.run_arbitrary_borg(
@ -229,85 +227,45 @@ def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise():
)
def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository():
def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'key', 'export', 'repo'),
('borg', 'key', 'export', '--info', '$REPOSITORY'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''},
)
insert_logging_mock(logging.INFO)
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['key', 'export'],
options=['key', 'export', '$REPOSITORY'],
)
def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository():
def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_flags():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'debug', 'dump-manifest', 'repo', 'path'),
('borg', 'debug', 'dump-manifest', '--info', '$REPOSITORY', 'path'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
shell=True,
extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''},
)
insert_logging_mock(logging.INFO)
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['debug', 'dump-manifest', 'path'],
)
def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repository():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').never()
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'debug', 'info'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
)
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['debug', 'info'],
)
def test_run_arbitrary_borg_with_debug_convert_profile_command_does_not_pass_borg_repository():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').never()
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'debug', 'convert-profile', 'in', 'out'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
extra_environment=None,
)
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
local_borg_version='1.2.3',
options=['debug', 'convert-profile', 'in', 'out'],
options=['debug', 'dump-manifest', '$REPOSITORY', 'path'],
)