Merge branch 'master' into master

This commit is contained in:
networkjanitor 2020-08-12 21:11:45 +00:00
commit 3c723e8d99
8 changed files with 107 additions and 64 deletions

5
NEWS
View file

@ -1,6 +1,9 @@
1.5.10.dev0 1.5.10.dev0
* #350: Fix traceback when a configuration directory is non-readable due to directory permissions.
* Clarify documentation on configuration overrides, specifically the portion about list syntax: * Clarify documentation on configuration overrides, specifically the portion about list syntax:
http://localhost:8080/docs/how-to/make-per-application-backups/#configuration-overrides http://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides
* Clarify documentation overview of monitoring options:
http://torsion.org/borgmatic/docs/how-to/monitor-your-backups/
1.5.9 1.5.9
* #300: Add "borgmatic export-tar" action to export an archive to a tar-formatted file or stream. * #300: Add "borgmatic export-tar" action to export an archive to a tar-formatted file or stream.

View file

@ -22,8 +22,8 @@ location:
# Paths of local or remote repositories to backup to. # Paths of local or remote repositories to backup to.
repositories: repositories:
- 1234@usw-s001.rsync.net:backups.borg
- k8pDxu32@k8pDxu32.repo.borgbase.com:repo - k8pDxu32@k8pDxu32.repo.borgbase.com:repo
- 1234@usw-s001.rsync.net:backups.borg
- /var/lib/backups/local.borg - /var/lib/backups/local.borg
retention: retention:
@ -66,9 +66,9 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; <a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; <a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; <a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; <a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px;"></a>
<a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic"><img src="docs/static/rsyncnet.png" alt="rsync.net" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; <a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<img src="docs/static/rsyncnet.png" alt="rsync.net" height="60px" style="margin-bottom:20px;">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
## How-to guides ## How-to guides
@ -101,8 +101,8 @@ services helps support borgmatic development and hosting. (These are referral
links, but without any tracking scripts or cookies.) links, but without any tracking scripts or cookies.)
<ul> <ul>
<li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
<li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li> <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
<li>rsync.net: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
</ul> </ul>
## Support and contributing ## Support and contributing

View file

@ -44,6 +44,9 @@ def collect_config_filenames(config_paths):
yield path yield path
continue continue
if not os.access(path, os.R_OK):
continue
for filename in sorted(os.listdir(path)): for filename in sorted(os.listdir(path)):
full_filename = os.path.join(path, filename) full_filename = os.path.join(path, filename)
matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml') matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')

View file

@ -123,7 +123,7 @@ map:
patterns" for more details. patterns" for more details.
example: example:
- '*.pyc' - '*.pyc'
- ~/*/.cache - /home/*/.cache
- /etc/ssl - /etc/ssl
exclude_from: exclude_from:
seq: seq:
@ -522,6 +522,26 @@ map:
"check" action or an associated before/after hook. "check" action or an associated before/after hook.
example: example:
- echo "Error during prune/create/check." - echo "Error during prune/create/check."
before_everything:
seq:
- type: str
desc: |
List of one or more shell commands or scripts to execute
before running all actions (if one of them is "create").
These are collected from all configuration files and then
run once before all of them (prior to all actions).
example:
- echo "Starting actions."
after_everything:
seq:
- type: str
desc: |
List of one or more shell commands or scripts to execute
after running all actions (if one of them is "create").
These are collected from all configuration files and then
run once after all of them (after any action).
example:
- echo "Completed actions."
postgresql_databases: postgresql_databases:
seq: seq:
- map: - map:
@ -709,26 +729,6 @@ map:
documentation for details. documentation for details.
example: example:
https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d01 https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d01
before_everything:
seq:
- type: str
desc: |
List of one or more shell commands or scripts to execute
before running all actions (if one of them is "create").
These are collected from all configuration files and then
run once before all of them (prior to all actions).
example:
- echo "Starting actions."
after_everything:
seq:
- type: str
desc: |
List of one or more shell commands or scripts to execute
after running all actions (if one of them is "create").
These are collected from all configuration files and then
run once before all of them (prior to all actions).
example:
- echo "Completed actions."
umask: umask:
type: scalar type: scalar
desc: | desc: |

View file

@ -16,8 +16,8 @@ location:
# Paths of local or remote repositories to backup to. # Paths of local or remote repositories to backup to.
repositories: repositories:
- 1234@usw-s001.rsync.net:backups.borg
- k8pDxu32@k8pDxu32.repo.borgbase.com:repo - k8pDxu32@k8pDxu32.repo.borgbase.com:repo
- 1234@usw-s001.rsync.net:backups.borg
- /var/lib/backups/local.borg - /var/lib/backups/local.borg
``` ```
@ -28,8 +28,8 @@ your source directories.
Here's a way of visualizing what borgmatic does with the above configuration: Here's a way of visualizing what borgmatic does with the above configuration:
1. Backup `/home` and `/etc` to `1234@usw-s001.rsync.net:backups.borg`
2. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo` 2. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo`
1. Backup `/home` and `/etc` to `1234@usw-s001.rsync.net:backups.borg`
3. Backup `/home` and `/etc` to `/var/lib/backups/local.borg` 3. Backup `/home` and `/etc` to `/var/lib/backups/local.borg`
This gives you redundancy of your data across repositories and even This gives you redundancy of your data across repositories and even

View file

@ -10,48 +10,68 @@ and alerting comes in.
There are several different ways you can monitor your backups and find out There are several different ways you can monitor your backups and find out
whether they're succeeding. Which of these you choose to do is up to you and whether they're succeeding. Which of these you choose to do is up to you and
your particular infrastructure: your particular infrastructure.
1. **Job runner alerts**: The easiest place to start is with failure alerts ### Job runner alerts
from the [scheduled job
runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) (cron, The easiest place to start is with failure alerts from the [scheduled job
systemd, etc.) that's running borgmatic. But note that if the job doesn't even runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot)
get scheduled (e.g. due to the job runner not running), you probably won't get (cron, systemd, etc.) that's running borgmatic. But note that if the job
an alert at all! Still, this is a decent first line of defense, especially doesn't even get scheduled (e.g. due to the job runner not running), you
when combined with some of the other approaches below. probably won't get an alert at all! Still, this is a decent first line of
2. **borgmatic error hooks**: The `on_error` hook allows you to run an arbitrary defense, especially when combined with some of the other approaches below.
command or script when borgmatic itself encounters an error running your
backups. So for instance, you can run a script to send yourself a text message ### Commands run on error
alert. But note that if borgmatic doesn't actually run, this alert won't fire.
See [error The `on_error` hook allows you to run an arbitrary command or script when
borgmatic itself encounters an error running your backups. So for instance,
you can run a script to send yourself a text message alert. But note that if
borgmatic doesn't actually run, this alert won't fire. See [error
hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks) hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
below for how to configure this. below for how to configure this.
4. **borgmatic monitoring hooks**: This feature integrates with monitoring
services like [Healthchecks](https://healthchecks.io/), ### Third-party monitoring services
[Cronitor](https://cronitor.io), [Cronhub](https://cronhub.io), and
[PagerDuty](https://www.pagerduty.com/) and pings these services whenever borgmatic integrates with monitoring services like
borgmatic runs. That way, you'll receive an alert when something goes wrong or [Healthchecks](https://healthchecks.io/), [Cronitor](https://cronitor.io),
(for certain hooks) the service doesn't hear from borgmatic for a configured [Cronhub](https://cronhub.io), and [PagerDuty](https://www.pagerduty.com/) and
interval. See [Healthchecks pings these services whenever borgmatic runs. That way, you'll receive an
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook), [Cronitor alert when something goes wrong or (for certain hooks) the service doesn't
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook), [Cronhub hear from borgmatic for a configured interval. See [Healthchecks
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook), and hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook),
[PagerDuty hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook) [Cronitor
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook),
[Cronhub
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook),
and [PagerDuty
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook)
below for how to configure this. below for how to configure this.
3. **Third-party monitoring software**: You can use traditional monitoring
software to consume borgmatic JSON output and track when the last While these services offer different features, you probably only need to use
successful backup occurred. See [scripting one of them at most.
### Third-party monitoring software
You can use traditional monitoring software to consume borgmatic JSON output
and track when the last successful backup occurred. See [scripting
borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic) borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic)
and [related software](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#related-software) and [related
software](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#related-software)
below for how to configure this. below for how to configure this.
5. **Borg hosting providers**: Most [Borg hosting
### Borg hosting providers
Most [Borg hosting
providers](https://torsion.org/borgmatic/#hosting-providers) include providers](https://torsion.org/borgmatic/#hosting-providers) include
monitoring and alerting as part of their offering. This gives you a dashboard monitoring and alerting as part of their offering. This gives you a dashboard
to check on all of your backups, and can alert you if the service doesn't hear to check on all of your backups, and can alert you if the service doesn't hear
from borgmatic for a configured interval. from borgmatic for a configured interval.
6. **borgmatic consistency checks**: While not strictly part of monitoring, if you
really want confidence that your backups are not only running but are ### Consistency checks
restorable as well, you can configure particular [consistency
While not strictly part of monitoring, if you really want confidence that your
backups are not only running but are restorable as well, you can configure
particular [consistency
checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration) checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration)
or even script full [extract or even script full [extract
tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/). tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/).

View file

@ -96,8 +96,8 @@ services helps support borgmatic development and hosting. (These are referral
links, but without any tracking scripts or cookies.) links, but without any tracking scripts or cookies.)
<ul> <ul>
<li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
<li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li> <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
<li>rsync.net: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
</ul> </ul>
## Configuration ## Configuration
@ -135,9 +135,8 @@ configuration](https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your
### Encryption ### Encryption
Note that if you plan to run borgmatic on a schedule with cron, and you If you encrypt your Borg repository with a passphrase instead of a key file,
encrypt your Borg repository with a passphrase instead of a key file, you'll you'll either need to set the borgmatic `encryption_passphrase` configuration
either need to set the borgmatic `encryption_passphrase` configuration
variable or set the `BORG_PASSPHRASE` environment variable. See the variable or set the `BORG_PASSPHRASE` environment variable. See the
[repository encryption [repository encryption
section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption) section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption)

View file

@ -45,6 +45,7 @@ def test_collect_config_filenames_collects_yml_file_endings():
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yml').and_return(False)
flexmock(module.os).should_receive('access').and_return(True)
flexmock(module.os).should_receive('listdir') flexmock(module.os).should_receive('listdir')
flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['foo.yml']) flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['foo.yml'])
@ -62,6 +63,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False)
flexmock(module.os).should_receive('access').and_return(True)
flexmock(module.os).should_receive('listdir') flexmock(module.os).should_receive('listdir')
flexmock(sys.modules['builtins']).should_receive('sorted').and_return( flexmock(sys.modules['builtins']).should_receive('sorted').and_return(
['foo.yaml', 'bar', 'baz.yaml'] ['foo.yaml', 'bar', 'baz.yaml']
@ -84,6 +86,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar.yaml~').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar.yaml~').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.txt').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.txt').and_return(False)
flexmock(module.os).should_receive('access').and_return(True)
flexmock(module.os).should_receive('listdir') flexmock(module.os).should_receive('listdir')
flexmock(sys.modules['builtins']).should_receive('sorted').and_return( flexmock(sys.modules['builtins']).should_receive('sorted').and_return(
['foo.yaml', 'bar.yaml~', 'baz.txt'] ['foo.yaml', 'bar.yaml~', 'baz.txt']
@ -94,6 +97,21 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
assert config_filenames == ('/etc/borgmatic.d/foo.yaml',) assert config_filenames == ('/etc/borgmatic.d/foo.yaml',)
def test_collect_config_filenames_skips_permission_denied_directories():
config_paths = ('config.yaml', '/etc/borgmatic.d')
mock_path = flexmock(module.os.path)
mock_path.should_receive('exists').and_return(True)
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
flexmock(module.os).should_receive('access').and_return(False)
flexmock(module.os).should_receive('listdir')
flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['config.yaml'])
config_filenames = tuple(module.collect_config_filenames(config_paths))
assert config_filenames == ('config.yaml',)
def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does_not_exist(): def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does_not_exist():
config_paths = ('config.yaml', '/etc/borgmatic/config.yaml') config_paths = ('config.yaml', '/etc/borgmatic/config.yaml')
mock_path = flexmock(module.os.path) mock_path = flexmock(module.os.path)