From 1709f57ff0734d119e89f90f604c4142f801e939 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 25 Jul 2021 22:30:15 -0700 Subject: [PATCH 01/34] Fix hang when restoring a PostgreSQL "tar" format database dump (#430). --- NEWS | 1 + borgmatic/execute.py | 20 ++++++++++++++---- tests/integration/test_execute.py | 34 ++++++++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index 80b8b4b..0dc9126 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 1.5.16.dev0 * #379: Suppress console output in sample crontab and systemd service files. * #407: Fix syslog logging on FreeBSD. + * #430: Fix hang when restoring a PostgreSQL "tar" format database dump. * Better error messages! Switch the library used for validating configuration files (from pykwalify to jsonschema). * Link borgmatic Ansible role from installation documentation: diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 95a8f6c..39b2e50 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -59,11 +59,12 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): ''' # Map from output buffer to sequence of last lines. buffer_last_lines = collections.defaultdict(list) - output_buffers = [ - output_buffer_for_process(process, exclude_stdouts) + process_for_output_buffer = { + output_buffer_for_process(process, exclude_stdouts): process for process in processes if process.stdout or process.stderr - ] + } + output_buffers = list(process_for_output_buffer.keys()) # Log output for each process until they all exit. while True: @@ -71,8 +72,19 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): (ready_buffers, _, _) = select.select(output_buffers, [], []) for ready_buffer in ready_buffers: + ready_process = process_for_output_buffer.get(ready_buffer) + + # The "ready" process has exited, but it might be a pipe destination with other + # processes (pipe sources) waiting to be read from. So as a measure to prevent + # hangs, vent all processes when one exits. + if ready_process and ready_process.poll() is not None: + for other_process in processes: + if other_process.poll() is None: + # Add the process's output to output_buffers to ensure it'll get read. + output_buffers.append(other_process.stdout) + line = ready_buffer.readline().rstrip().decode() - if not line: + if not line or not ready_process: continue # Keep the last few lines of output in case the process errors, and we need the output for diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index 490e3ab..d144f4d 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -98,7 +98,7 @@ def test_log_outputs_kills_other_processes_when_one_errors(): process, 2, 'borg' ).and_return(True) other_process = subprocess.Popen( - ['watch', 'true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ['sleep', '2'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) flexmock(module).should_receive('exit_code_indicates_error').with_args( other_process, None, 'borg' @@ -123,6 +123,38 @@ def test_log_outputs_kills_other_processes_when_one_errors(): assert error.value.output +def test_log_outputs_vents_other_processes_when_one_exits(): + ''' + Execute a command to generate a longish random string and pipe it into another command that + exits quickly. The test is basically to ensure we don't hang forever waiting for the exited + process to read the pipe, and that the string-generating process eventually gets vented and + exits. + ''' + flexmock(module.logger).should_receive('log') + flexmock(module).should_receive('command_for_process').and_return('grep') + + process = subprocess.Popen( + ['shuf', '-zer', '-n10000', '{A..Z}'], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + other_process = subprocess.Popen( + ['true'], stdin=process.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + flexmock(module).should_receive('output_buffer_for_process').with_args( + process, (process.stdout,) + ).and_return(process.stderr) + flexmock(module).should_receive('output_buffer_for_process').with_args( + other_process, (process.stdout,) + ).and_return(other_process.stdout) + flexmock(process.stdout).should_call('readline').once() + + module.log_outputs( + (process, other_process), + exclude_stdouts=(process.stdout,), + output_log_level=logging.INFO, + borg_local_path='borg', + ) + + def test_log_outputs_truncates_long_error_output(): flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0 flexmock(module.logger).should_receive('log') From 2a8692c64f6720a47ef00d664b2ec214f8ae6957 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 25 Jul 2021 22:50:00 -0700 Subject: [PATCH 02/34] Fix integration test to hopefully work on Alpine (#430). --- tests/integration/test_execute.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index d144f4d..6dc6467 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -134,7 +134,7 @@ def test_log_outputs_vents_other_processes_when_one_exits(): flexmock(module).should_receive('command_for_process').and_return('grep') process = subprocess.Popen( - ['shuf', '-zer', '-n10000', '{A..Z}'], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ['xxd', '-l', '40000', '-p', '/dev/urandom'], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) other_process = subprocess.Popen( ['true'], stdin=process.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT @@ -145,7 +145,7 @@ def test_log_outputs_vents_other_processes_when_one_exits(): flexmock(module).should_receive('output_buffer_for_process').with_args( other_process, (process.stdout,) ).and_return(other_process.stdout) - flexmock(process.stdout).should_call('readline').once() + flexmock(process.stdout).should_call('readline').at_least().once() module.log_outputs( (process, other_process), From 1f3907a6a5fdb198ea28d37517a23c6bfd2b3d4e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 26 Jul 2021 09:42:14 -0700 Subject: [PATCH 03/34] Fix for failing PostgreSQL directory format test (#430). --- borgmatic/execute.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 39b2e50..40d70c5 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -79,7 +79,10 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): # hangs, vent all processes when one exits. if ready_process and ready_process.poll() is not None: for other_process in processes: - if other_process.poll() is None: + if ( + other_process.poll() is None + and other_process.stdout not in output_buffers + ): # Add the process's output to output_buffers to ensure it'll get read. output_buffers.append(other_process.stdout) From 0aff497430be00b1152257204f2e68346525028d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 26 Jul 2021 10:17:49 -0700 Subject: [PATCH 04/34] Bump version for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 0dc9126..fa00344 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.5.16.dev0 +1.5.16 * #379: Suppress console output in sample crontab and systemd service files. * #407: Fix syslog logging on FreeBSD. * #430: Fix hang when restoring a PostgreSQL "tar" format database dump. diff --git a/setup.py b/setup.py index ebc47fd..80d1815 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.5.16.dev0' +VERSION = '1.5.16' setup( From c63219936eec0869b80ff4452c8e3bf80eaa2418 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 26 Jul 2021 13:44:14 -0700 Subject: [PATCH 05/34] Wording tweaks to security policy. --- SECURITY.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 81a633e..d82b6f3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,14 +6,13 @@ permalink: security-policy/index.html ## Supported versions While we want to hear about security vulnerabilities in all versions of -borgmatic, security fixes will only be made to the most recently released -version. It's not practical for our small volunteer effort to maintain -multiple different release branches and put out separate security patches for -each. +borgmatic, security fixes are only made to the most recently released version. +It's simply not practical for our small volunteer effort to maintain multiple +release branches and put out separate security patches for each. ## Reporting a vulnerability If you find a security vulnerability, please [file a ticket](https://torsion.org/borgmatic/#issues) or [send email directly](mailto:witten@torsion.org) as appropriate. You should expect to hear -back within a few days at most, and generally sooner. +back within a few days at most and generally sooner. From 92d729a9ddda01517378a1cf6bd5fbbbe4cfe367 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 26 Jul 2021 16:33:41 -0700 Subject: [PATCH 06/34] Try temporary work around for Drone build bug: https://github.com/drone-plugins/drone-docker/pull/327 --- .drone.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index f6c0108..944ca5f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -70,7 +70,9 @@ name: documentation steps: - name: build - image: plugins/docker + #image: plugins/docker + # Temporary work-around for https://github.com/drone-plugins/drone-docker/pull/327 + image: techknowlogick/drone-docker settings: username: from_secret: docker_username From e8b8d86592565acde1ac6e7809947b8a761f2f91 Mon Sep 17 00:00:00 2001 From: Marek Szuba Date: Tue, 27 Jul 2021 13:46:51 +0100 Subject: [PATCH 07/34] tests/integration/test_execute: use plain Python rather than xxd Removes this test's dependencies on vim and /dev/urandom. Signed-off-by: Marek Szuba --- tests/integration/test_execute.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index 6dc6467..3b9bef9 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -1,5 +1,6 @@ import logging import subprocess +import sys import pytest from flexmock import flexmock @@ -134,7 +135,8 @@ def test_log_outputs_vents_other_processes_when_one_exits(): flexmock(module).should_receive('command_for_process').and_return('grep') process = subprocess.Popen( - ['xxd', '-l', '40000', '-p', '/dev/urandom'], stdout=subprocess.PIPE, stderr=subprocess.PIPE + [sys.executable, '-c', "import random, string; print(''.join(random.choice(string.ascii_letters) for _ in range(40000)))"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE ) other_process = subprocess.Popen( ['true'], stdin=process.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT From 80b33fbf8af18fc5bebb341028fe457c9fee67d7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 27 Jul 2021 09:39:48 -0700 Subject: [PATCH 08/34] Code style reformatting. --- tests/integration/test_execute.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index 3b9bef9..7320658 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -135,8 +135,13 @@ def test_log_outputs_vents_other_processes_when_one_exits(): flexmock(module).should_receive('command_for_process').and_return('grep') process = subprocess.Popen( - [sys.executable, '-c', "import random, string; print(''.join(random.choice(string.ascii_letters) for _ in range(40000)))"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE + [ + sys.executable, + '-c', + "import random, string; print(''.join(random.choice(string.ascii_letters) for _ in range(40000)))", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) other_process = subprocess.Popen( ['true'], stdin=process.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT From b3f5a9d18f72639fa6109d8fb3d1d08d426b5c16 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 27 Jul 2021 10:04:22 -0700 Subject: [PATCH 09/34] Fix error when configuration file contains "umask" option (#437). --- NEWS | 4 ++++ borgmatic/config/schema.yaml | 4 ++-- setup.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index fa00344..ba8e675 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.5.17 + * #437: Fix error when configuration file contains "umask" option. + * Remove test dependency on vim and /dev/urandom. + 1.5.16 * #379: Suppress console output in sample crontab and systemd service files. * #407: Fix syslog logging on FreeBSD. diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 00df52e..a2f3bea 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -292,7 +292,7 @@ properties: $borg_base_directory/.config/borg/keys example: /path/to/base/config/keys umask: - type: string + type: integer description: Umask to be used for borg create. Defaults to 0077. example: 0077 lock_wait: @@ -787,7 +787,7 @@ properties: example: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d01 umask: - type: scalar + type: integer description: | Umask used when executing hooks. Defaults to the umask that borgmatic is run with. diff --git a/setup.py b/setup.py index 80d1815..42cf70d 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.5.16' +VERSION = '1.5.17' setup( From 5890a1cb48d962bb8b4d8027f235484ecd3ae1e8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 30 Jul 2021 09:48:13 -0700 Subject: [PATCH 10/34] Fix "message too long" error when logging to rsyslog (#389). --- NEWS | 3 +++ borgmatic/execute.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index ba8e675..8bdb008 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.5.18 + * #389: Fix "message too long" error when logging to rsyslog. + 1.5.17 * #437: Fix error when configuration file contains "umask" option. * Remove test dependency on vim and /dev/urandom. diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 40d70c5..db6da64 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -138,9 +138,12 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): if not output_buffer: continue - remaining_output = output_buffer.read().rstrip().decode() + while True: + remaining_output = output_buffer.readline().rstrip().decode() + + if not remaining_output: # pragma: no cover + break - if remaining_output: # pragma: no cover logger.log(output_log_level, remaining_output) From 760286abe16b274989dd616747c1bca69b8d9f08 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 30 Jul 2021 09:49:07 -0700 Subject: [PATCH 11/34] Dev release bump. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 42cf70d..5f5330f 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.5.17' +VERSION = '1.5.18.dev0' setup( From c9211320e13c05489b0e7858d9c5596c0eb92950 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 4 Aug 2021 15:32:51 -0700 Subject: [PATCH 12/34] Fix dev version in changelog. --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 8bdb008..d196df9 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.5.18 +1.5.18.dev0 * #389: Fix "message too long" error when logging to rsyslog. 1.5.17 From acb2ca79d9e3b6d851a2c017b79a314b9f5baf61 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 6 Aug 2021 08:58:11 -0700 Subject: [PATCH 13/34] Fix traceback that can occur when dumping a database (#440). --- NEWS | 3 ++- borgmatic/execute.py | 5 +++-- setup.py | 2 +- tests/integration/test_execute.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index d196df9..19e605a 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ -1.5.18.dev0 +1.5.18 * #389: Fix "message too long" error when logging to rsyslog. + * #440: Fix traceback that can occur when dumping a database. 1.5.17 * #437: Fix error when configuration file contains "umask" option. diff --git a/borgmatic/execute.py b/borgmatic/execute.py index db6da64..a6741c9 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -81,6 +81,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): for other_process in processes: if ( other_process.poll() is None + and other_process.stdout and other_process.stdout not in output_buffers ): # Add the process's output to output_buffers to ensure it'll get read. @@ -138,10 +139,10 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): if not output_buffer: continue - while True: + while True: # pragma: no cover remaining_output = output_buffer.readline().rstrip().decode() - if not remaining_output: # pragma: no cover + if not remaining_output: break logger.log(output_log_level, remaining_output) diff --git a/setup.py b/setup.py index 5f5330f..49b93d0 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.5.18.dev0' +VERSION = '1.5.18' setup( diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index 7320658..51aa5f4 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -162,6 +162,37 @@ def test_log_outputs_vents_other_processes_when_one_exits(): ) +def test_log_outputs_does_not_error_when_one_process_exits(): + flexmock(module.logger).should_receive('log') + flexmock(module).should_receive('command_for_process').and_return('grep') + + process = subprocess.Popen( + [ + sys.executable, + '-c', + "import random, string; print(''.join(random.choice(string.ascii_letters) for _ in range(40000)))", + ], + stdout=None, # Specifically test the case of a process without stdout captured. + stderr=None, + ) + other_process = subprocess.Popen( + ['true'], stdin=process.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + flexmock(module).should_receive('output_buffer_for_process').with_args( + process, (process.stdout,) + ).and_return(process.stderr) + flexmock(module).should_receive('output_buffer_for_process').with_args( + other_process, (process.stdout,) + ).and_return(other_process.stdout) + + module.log_outputs( + (process, other_process), + exclude_stdouts=(process.stdout,), + output_log_level=logging.INFO, + borg_local_path='borg', + ) + + def test_log_outputs_truncates_long_error_output(): flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0 flexmock(module.logger).should_receive('log') From 6df6176f3a0de22d497b983df2f732c2e10cf9ea Mon Sep 17 00:00:00 2001 From: Vladimir Timofeenko Date: Mon, 30 Aug 2021 11:20:34 -0700 Subject: [PATCH 14/34] Added more strict ProtectHome to systemd unit This commit changes the comment in sample systemd service. Using a combination of 'ProtectHome' and 'BindPaths' it's possible to hide the irrelevant paths inside /root from borgmatic service when it is run. ReadWritePaths are suggested to be used only for paths that contain borg repositories and the backup sources can be specified as ReadOnlyPaths. --- sample/systemd/borgmatic.service | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sample/systemd/borgmatic.service b/sample/systemd/borgmatic.service index 4fc8895..b6adda9 100644 --- a/sample/systemd/borgmatic.service +++ b/sample/systemd/borgmatic.service @@ -37,8 +37,11 @@ SystemCallErrorNumber=EPERM # system read-only be default and uncomment 'ReadWritePaths' for the required write access. # Add local repositroy paths to the list of 'ReadWritePaths' like '-/mnt/my_backup_drive'. ProtectSystem=full -# ProtectHome=read-only -# ReadWritePaths=-/root/.config/borg -/root/.cache/borg -/root/.borgmatic +# ReadWritePaths=-/mnt/my_backup_drive +# ReadOnlyPaths=-/var/lib/my_backup_source +# This will mount a tmpfs on top of /root and pass through needed paths +# ProtectHome=tmpfs +# BindPaths=-/root/.cache/borg -/root/.cache/borg -/root/.borgmatic CapabilityBoundingSet=CAP_DAC_READ_SEARCH CAP_NET_RAW From 2bb1fc98264bbd34b978786746dbd856294a2f5a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 12 Sep 2021 13:15:34 -0700 Subject: [PATCH 15/34] Mention Docker Compose under installation options. --- docs/how-to/set-up-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 0db1e68..0f69fa5 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -77,7 +77,7 @@ on a relatively dedicated system, then a global install can work out fine. Besides the approaches described above, there are several other options for installing borgmatic: - * [Docker image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/) + * [Docker image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/) (+ Docker Compose files) * [Docker base image](https://hub.docker.com/r/monachus/borgmatic/) * [Debian](https://tracker.debian.org/pkg/borgmatic) * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic) From 7ff6066d47df3c4bd832045f721549c217dd4f20 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 14 Sep 2021 10:18:10 -0700 Subject: [PATCH 16/34] Move GitHub hosting from a personal namespace to an organization. --- NEWS | 5 +++++ README.md | 2 +- setup.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 19e605a..524103f 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.5.19.dev0 + * Move GitHub hosting from a personal namespace to an organization for better collaboration with + related projects. + * 1k ★s on GitHub! + 1.5.18 * #389: Fix "message too long" error when logging to rsyslog. * #440: Fix traceback that can occur when dumping a database. diff --git a/README.md b/README.md index 5d1a79e..00124c5 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ Other questions or comments? Contact borgmatic [source code is available](https://projects.torsion.org/witten/borgmatic) and is also mirrored -on [GitHub](https://github.com/witten/borgmatic) for convenience. +on [GitHub](https://github.com/borgmatic-collective/borgmatic) for convenience. borgmatic is licensed under the GNU General Public License version 3 or any later version. diff --git a/setup.py b/setup.py index 49b93d0..357289a 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.5.18' +VERSION = '1.5.19.dev0' setup( From ecc849dd074378e5571456217b1b6fdc30557462 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 14 Sep 2021 11:32:01 -0700 Subject: [PATCH 17/34] Move Gitea hosting from a personal namespace to an organization. --- NEWS | 6 +++--- README.md | 8 ++++---- docs/_includes/components/suggestion-link.html | 14 +------------- docs/how-to/develop-on-borgmatic.md | 4 ++-- docs/how-to/set-up-backups.md | 8 ++++---- scripts/release | 2 +- 6 files changed, 15 insertions(+), 27 deletions(-) diff --git a/NEWS b/NEWS index 524103f..bbcb209 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ 1.5.19.dev0 - * Move GitHub hosting from a personal namespace to an organization for better collaboration with - related projects. + * Move Gitea and GitHub hosting from a personal namespace to an organization for better + collaboration with related projects. * 1k ★s on GitHub! 1.5.18 @@ -573,7 +573,7 @@ * #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed includes/excludes. * Moved issue tracker from Taiga to integrated Gitea tracker at - https://projects.torsion.org/witten/borgmatic/issues + https://projects.torsion.org/borgmatic-collective/borgmatic/issues 1.1.12 * #46: Declare dependency on pykwalify 1.6 or above, as older versions yield "Unknown key: version" diff --git a/README.md b/README.md index 00124c5..ddca709 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ development or hosting. ### Issues You've got issues? Or an idea for a feature enhancement? We've got an [issue -tracker](https://projects.torsion.org/witten/borgmatic/issues). In order to +tracker](https://projects.torsion.org/borgmatic-collective/borgmatic/issues). In order to create a new issue or comment on an issue, you'll need to [login first](https://projects.torsion.org/user/login). Note that you can login with an existing GitHub account if you prefer. @@ -129,15 +129,15 @@ Other questions or comments? Contact ### Contributing borgmatic [source code is -available](https://projects.torsion.org/witten/borgmatic) and is also mirrored +available](https://projects.torsion.org/borgmatic-collective/borgmatic) and is also mirrored on [GitHub](https://github.com/borgmatic-collective/borgmatic) for convenience. borgmatic is licensed under the GNU General Public License version 3 or any later version. If you'd like to contribute to borgmatic development, please feel free to -submit a [Pull Request](https://projects.torsion.org/witten/borgmatic/pulls) -or open an [issue](https://projects.torsion.org/witten/borgmatic/issues) first +submit a [Pull Request](https://projects.torsion.org/borgmatic-collective/borgmatic/pulls) +or open an [issue](https://projects.torsion.org/borgmatic-collective/borgmatic/issues) first to discuss your idea. We also accept Pull Requests on GitHub, if that's more your thing. In general, contributions are very welcome. We don't bite! diff --git a/docs/_includes/components/suggestion-link.html b/docs/_includes/components/suggestion-link.html index 20fc4fc..2c2d142 100644 --- a/docs/_includes/components/suggestion-link.html +++ b/docs/_includes/components/suggestion-link.html @@ -1,17 +1,5 @@

Improve this documentation

Have an idea on how to make this documentation even better? Use our issue tracker to send your +href="https://projects.torsion.org/borgmatic-collective/borgmatic/issues">issue tracker to send your feedback!

- - - - diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index dc0327b..77fa236 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -10,13 +10,13 @@ eleventyNavigation: To get set up to hack on borgmatic, first clone master via HTTPS or SSH: ```bash -git clone https://projects.torsion.org/witten/borgmatic.git +git clone https://projects.torsion.org/borgmatic-collective/borgmatic.git ``` Or: ```bash -git clone ssh://git@projects.torsion.org:3022/witten/borgmatic.git +git clone ssh://git@projects.torsion.org:3022/borgmatic-collective/borgmatic.git ``` Then, install borgmatic diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 0f69fa5..3dc4777 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -250,7 +250,7 @@ that, you can configure a separate job runner to invoke it periodically. ### cron If you're using cron, download the [sample cron -file](https://projects.torsion.org/witten/borgmatic/src/master/sample/cron/borgmatic). +file](https://projects.torsion.org/borgmatic-collective/borgmatic/src/master/sample/cron/borgmatic). Then, from the directory where you downloaded it: ```bash @@ -271,9 +271,9 @@ you may already have borgmatic systemd service and timer files. If so, you may be able to skip some of the steps below.) First, download the [sample systemd service -file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.service) +file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.service) and the [sample systemd timer -file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer). +file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer). Then, from the directory where you downloaded them: @@ -294,7 +294,7 @@ borgmatic to run. If you run borgmatic in macOS with launchd, you may encounter permissions issues when reading files to backup. If that happens to you, you may be interested in an [unofficial work-around for Full Disk -Access](https://projects.torsion.org/witten/borgmatic/issues/293). +Access](https://projects.torsion.org/borgmatic-collective/borgmatic/issues/293). ## Colored output diff --git a/scripts/release b/scripts/release index f728390..b901047 100755 --- a/scripts/release +++ b/scripts/release @@ -38,7 +38,7 @@ twine upload -r pypi dist/borgmatic-*-py3-none-any.whl dist/borgmatic-*-py3-none release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')" escaped_release_changelog="$(echo "$release_changelog" | sed -z 's/\n/\\n/g' | sed -z 's/\"/\\"/g')" curl --silent --request POST \ - "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases" \ + "https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/releases" \ --header "Authorization: token $projects_token" \ --header "Accept: application/json" \ --header "Content-Type: application/json" \ From 4ba206f8f4a396787d3b96a8a8be59253fba82ed Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 14 Sep 2021 11:35:34 -0700 Subject: [PATCH 18/34] Update build server URL to new organization namespace. --- README.md | 2 +- docs/how-to/develop-on-borgmatic.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ddca709..3463c29 100644 --- a/README.md +++ b/README.md @@ -145,5 +145,5 @@ Also, please check out the [borgmatic development how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for info on cloning source code, running tests, etc. -![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg?ref=refs/heads/master) +![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/master) diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 77fa236..721255b 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -118,7 +118,7 @@ See the Black, Flake8, and isort documentation for more information. Each pull request triggers a continuous integration build which runs the test suite. You can view these builds on -[build.torsion.org](https://build.torsion.org/witten/borgmatic), and they're +[build.torsion.org](https://build.torsion.org/borgmatic-collective/borgmatic), and they're also linked from the commits list on each pull request. ## Documentation development From 77980511c68f5f0d5ce91ab9ed53d2e29fd2734f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 16 Sep 2021 09:51:40 -0700 Subject: [PATCH 19/34] Add another glob pattern example to exclude patterns. --- borgmatic/config/schema.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index a2f3bea..c0bf2e3 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -135,12 +135,14 @@ properties: type: string description: | Any paths matching these patterns are excluded from backups. - Globs and tildes are expanded. Do not backslash spaces in - path names. See the output of "borg help patterns" for more - details. + Globs and tildes are expanded. (Note however that a glob + pattern must either start with a glob or be an absolute + path.) Do not backslash spaces in path names. See the output + of "borg help patterns" for more details. example: - '*.pyc' - /home/*/.cache + - '*/.vim*.tmp' - /etc/ssl - /home/user/path with spaces exclude_from: From fbbb096ceca602b9230c71d1d1dce408db3e9b8f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 4 Oct 2021 11:15:51 -0700 Subject: [PATCH 20/34] Note in documentation that borgmatic requires Python 3.6+. --- borgmatic/config/schema.yaml | 2 +- docs/how-to/develop-on-borgmatic.md | 2 -- docs/how-to/set-up-backups.md | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index c0bf2e3..5828f2a 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -635,7 +635,7 @@ properties: Password with which to connect to the database. Omitting a password will only work if PostgreSQL is configured to trust the configured username - without a password, or you create a ~/.pgpass + without a password or you create a ~/.pgpass file. example: trustsome1 format: diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 721255b..0f546e7 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -66,8 +66,6 @@ following: tox -e black ``` -Note that Black requires at minimum Python 3.6. - And if you get a complaint from the [isort](https://github.com/timothycrosley/isort) Python import orderer, you can ask isort to order your imports for you: diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 3dc4777..6ad551a 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -28,7 +28,7 @@ sudo pip3 install --user --upgrade borgmatic This installs borgmatic and its commands at the `/root/.local/bin` path. Your pip binary may have a different name than "pip3". Make sure you're using -Python 3, as borgmatic does not support Python 2. +Python 3.6+, as borgmatic does not support Python 2. The next step is to ensure that borgmatic's commands available are on your system `PATH`, so that you can run borgmatic: From f129e4c3017aa21f3f1d2e03075f538611a549a2 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 4 Oct 2021 13:09:44 -0700 Subject: [PATCH 21/34] Attempt to work-around outdated CA certificates in drone/git Docker image. --- .drone.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.drone.yml b/.drone.yml index 944ca5f..f634fad 100644 --- a/.drone.yml +++ b/.drone.yml @@ -14,6 +14,10 @@ services: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test +clone: + git: + image: witten/drone-git + steps: - name: build image: alpine:3.9 @@ -36,6 +40,10 @@ services: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test +clone: + git: + image: witten/drone-git + steps: - name: build image: alpine:3.10 @@ -58,6 +66,10 @@ services: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test +clone: + git: + image: witten/drone-git + steps: - name: build image: alpine:3.13 From d0c5bf6f6faf332998403fcf0d96514c4e6c9513 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 4 Oct 2021 13:13:35 -0700 Subject: [PATCH 22/34] Another attempt to unbreak build. --- .drone.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index f634fad..6e8759a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -16,7 +16,7 @@ services: clone: git: - image: witten/drone-git + skip_verify: true steps: - name: build @@ -42,7 +42,7 @@ services: clone: git: - image: witten/drone-git + skip_verify: true steps: - name: build @@ -68,7 +68,7 @@ services: clone: git: - image: witten/drone-git + skip_verify: true steps: - name: build From 65503e38b6db6ff5db405e0cf9a01280ffa3ff5c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 4 Oct 2021 13:14:19 -0700 Subject: [PATCH 23/34] Sigh. --- .drone.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.drone.yml b/.drone.yml index 6e8759a..beb7628 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,8 +15,7 @@ services: MYSQL_DATABASE: test clone: - git: - skip_verify: true + skip_verify: true steps: - name: build @@ -41,8 +40,7 @@ services: MYSQL_DATABASE: test clone: - git: - skip_verify: true + skip_verify: true steps: - name: build @@ -67,8 +65,7 @@ services: MYSQL_DATABASE: test clone: - git: - skip_verify: true + skip_verify: true steps: - name: build From 38e35bdb12767a1656ff68cc890469ccc7df4ab0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 4 Oct 2021 14:31:15 -0700 Subject: [PATCH 24/34] Skip TLS verify in documentation build clone to work around old drone/git CA certs. --- .drone.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.drone.yml b/.drone.yml index beb7628..6947ac2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -77,6 +77,9 @@ steps: kind: pipeline name: documentation +clone: + skip_verify: true + steps: - name: build #image: plugins/docker From 1004500d655f969ad52a38bc5264dc3b3ceaa681 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 11 Oct 2021 09:33:07 -0700 Subject: [PATCH 25/34] Update sample systemd service file comments about more granular read-only filesystem settings. --- NEWS | 1 + sample/systemd/borgmatic.service | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index bbcb209..48f39f1 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 1.5.19.dev0 + * Update sample systemd service file with more granular read-only filesystem settings. * Move Gitea and GitHub hosting from a personal namespace to an organization for better collaboration with related projects. * 1k ★s on GitHub! diff --git a/sample/systemd/borgmatic.service b/sample/systemd/borgmatic.service index b6adda9..d025785 100644 --- a/sample/systemd/borgmatic.service +++ b/sample/systemd/borgmatic.service @@ -32,10 +32,10 @@ RestrictSUIDSGID=yes SystemCallArchitectures=native SystemCallFilter=@system-service SystemCallErrorNumber=EPERM -# Restrict write access -# Change to 'ProtectSystem=strict' and uncomment 'ProtectHome' to make the whole file -# system read-only be default and uncomment 'ReadWritePaths' for the required write access. -# Add local repositroy paths to the list of 'ReadWritePaths' like '-/mnt/my_backup_drive'. +# To restrict write access further, change "ProtectSystem" to "strict" and uncomment +# "ReadWritePaths", "ReadOnlyPaths", "ProtectHome", and "BindPaths". Then add any local repository +# paths to the list of "ReadWritePaths" and local backup source paths to "ReadOnlyPaths". This +# leaves most of the filesystem read-only to borgmatic. ProtectSystem=full # ReadWritePaths=-/mnt/my_backup_drive # ReadOnlyPaths=-/var/lib/my_backup_source From 449896f6619053b462131cd20737f09a53d92ade Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 11 Oct 2021 10:40:10 -0700 Subject: [PATCH 26/34] Fix error when configured source directories are not present on the filesystem at the time of backup (#387). --- NEWS | 2 ++ borgmatic/borg/create.py | 14 ++++++++++---- tests/unit/borg/test_create.py | 25 +++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 48f39f1..708c9b6 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.5.19.dev0 + * #387: Fix error when configured source directories are not present on the filesystem at the time + of backup. Now, Borg will complain, but the backup will still continue. * Update sample systemd service file with more granular read-only filesystem settings. * Move Gitea and GitHub hosting from a personal namespace to an organization for better collaboration with related projects. diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index a9aa222..6b02c87 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -44,13 +44,18 @@ def _expand_home_directories(directories): return tuple(os.path.expanduser(directory) for directory in directories) -def map_directories_to_devices(directories): # pragma: no cover +def map_directories_to_devices(directories): ''' Given a sequence of directories, return a map from directory to an identifier for the device on - which that directory resides. This is handy for determining whether two different directories - are on the same filesystem (have the same device identifier). + which that directory resides or None if the path doesn't exist. + + This is handy for determining whether two different directories are on the same filesystem (have + the same device identifier). ''' - return {directory: os.stat(directory).st_dev for directory in directories} + return { + directory: os.stat(directory).st_dev if os.path.exists(directory) else None + for directory in directories + } def deduplicate_directories(directory_devices): @@ -82,6 +87,7 @@ def deduplicate_directories(directory_devices): for parent in parents: if ( pathlib.PurePath(other_directory) == parent + and directory_devices[directory] is not None and directory_devices[other_directory] == directory_devices[directory] ): if directory in deduplicated: diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index cc56881..99f2910 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -60,6 +60,30 @@ def test_expand_home_directories_considers_none_as_no_directories(): assert paths == () +def test_map_directories_to_devices_gives_device_id_per_path(): + flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55)) + flexmock(module.os).should_receive('stat').with_args('/bar').and_return(flexmock(st_dev=66)) + + device_map = module.map_directories_to_devices(('/foo', '/bar')) + + assert device_map == { + '/foo': 55, + '/bar': 66, + } + + +def test_map_directories_to_devices_with_missing_path_does_not_error(): + flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55)) + flexmock(module.os).should_receive('stat').with_args('/bar').and_raise(FileNotFoundError) + + device_map = module.map_directories_to_devices(('/foo', '/bar')) + + assert device_map == { + '/foo': 55, + '/bar': None, + } + + @pytest.mark.parametrize( 'directories,expected_directories', ( @@ -72,6 +96,7 @@ def test_expand_home_directories_considers_none_as_no_directories(): ({'/root': 1, '/root/foo/': 1}, ('/root',)), ({'/root': 1, '/root/foo': 2}, ('/root', '/root/foo')), ({'/root/foo': 1, '/root': 1}, ('/root',)), + ({'/root': None, '/root/foo': None}, ('/root', '/root/foo')), ({'/root': 1, '/etc': 1, '/root/foo/bar': 1}, ('/etc', '/root')), ({'/root': 1, '/root/foo': 1, '/root/foo/bar': 1}, ('/root',)), ({'/dup': 1, '/dup': 1}, ('/dup',)), From c8fcf6b336d7a7639bdb07c51f26d2536472dc9c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 11 Oct 2021 11:02:08 -0700 Subject: [PATCH 27/34] Mention changing borgmatic path in cron documentation (#455). --- NEWS | 1 + docs/how-to/set-up-backups.md | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 708c9b6..173df14 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 1.5.19.dev0 * #387: Fix error when configured source directories are not present on the filesystem at the time of backup. Now, Borg will complain, but the backup will still continue. + * #455: Mention changing borgmatic path in cron documentation. * Update sample systemd service file with more granular read-only filesystem settings. * Move Gitea and GitHub hosting from a personal namespace to an organization for better collaboration with related projects. diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 6ad551a..b8b860e 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -258,7 +258,10 @@ sudo mv borgmatic /etc/cron.d/borgmatic sudo chmod +x /etc/cron.d/borgmatic ``` -You can modify the cron file if you'd like to run borgmatic more or less frequently. +If borgmatic is installed at a different location than +`/root/.local/bin/borgmatic`, edit the cron file with the correct path. You +can also modify the cron file if you'd like to run borgmatic more or less +frequently. ### systemd From a7c8e7c823b286f4bb347d626f329122aab4b58d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 11 Oct 2021 11:13:32 -0700 Subject: [PATCH 28/34] Bump version for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 173df14..7ff9787 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.5.19.dev0 +1.5.19 * #387: Fix error when configured source directories are not present on the filesystem at the time of backup. Now, Borg will complain, but the backup will still continue. * #455: Mention changing borgmatic path in cron documentation. diff --git a/setup.py b/setup.py index 357289a..42cdc8a 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.5.19.dev0' +VERSION = '1.5.19' setup( From 1c6890492be0f754e6b8fba7f2552fd9a5519e0e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 11 Oct 2021 17:02:32 -0700 Subject: [PATCH 29/34] Bump version for release. --- NEWS | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 7ff9787..1cf9e99 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.5.20 + * Re-release with correct version without dev0 tag. + 1.5.19 * #387: Fix error when configured source directories are not present on the filesystem at the time of backup. Now, Borg will complain, but the backup will still continue. diff --git a/setup.py b/setup.py index 42cdc8a..0feb1e1 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.5.19' +VERSION = '1.5.20' setup( From 549aa9a25f3d0bc5543034124f1a5348d41d15c7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 22 Oct 2021 14:06:27 -0700 Subject: [PATCH 30/34] Update editable link. --- docs/how-to/develop-on-borgmatic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 0f546e7..865f0d4 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -20,7 +20,7 @@ git clone ssh://git@projects.torsion.org:3022/borgmatic-collective/borgmatic.git ``` Then, install borgmatic -"[editable](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs)" +"[editable](https://pip.pypa.io/en/stable/cli/pip_install/#editable-installs)" so that you can run borgmatic commands while you're hacking on them to make sure your changes work. From 7881327004fc243a639a5c6b53e5ff87b0dbdf56 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 22 Oct 2021 14:07:14 -0700 Subject: [PATCH 31/34] Upgrade CI test dependencies. --- scripts/run-full-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run-full-tests b/scripts/run-full-tests index ca2f5a7..47e4f5f 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -13,8 +13,8 @@ set -e apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client # If certain dependencies of black are available in this version of Alpine, install them. apk add --no-cache py3-typed-ast py3-regex || true -python3 -m pip install --upgrade pip==20.2.4 setuptools==50.3.2 -pip3 install tox==3.20.1 +python3 -m pip install --upgrade pip==21.3.1 setuptools==58.2.0 +pip3 install tox==3.24.4 export COVERAGE_FILE=/tmp/.coverage tox --workdir /tmp/.tox --sitepackages tox --workdir /tmp/.tox --sitepackages -e end-to-end From 717cfd2d37584b88d1fc92c637223fcd99508459 Mon Sep 17 00:00:00 2001 From: "Kim B. Heino" Date: Sat, 23 Oct 2021 15:04:07 +0300 Subject: [PATCH 32/34] validate: add support for both jsonschema v3 and old v2 RHEL8 and RHEL7 have old jsonschema v2. Try v3 (Draft7) first but fallback to v2 (Draft4) if needed. --- borgmatic/config/validate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 5658c1b..7d41151 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -110,7 +110,10 @@ def parse_configuration(config_filename, schema_filename, overrides=None): override.apply_overrides(config, overrides) normalize.normalize(config) - validator = jsonschema.Draft7Validator(schema) + try: + validator = jsonschema.Draft7Validator(schema) + except AttributeError: + validator = jsonschema.Draft4Validator(schema) validation_errors = tuple(validator.iter_errors(config)) if validation_errors: From 6299d8115d348ae3dcdfb98a2bcbec70c99fced3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 23 Oct 2021 09:45:17 -0700 Subject: [PATCH 33/34] Limit documentation build to master of main repo, as it pushes a Docker image. --- .drone.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.drone.yml b/.drone.yml index 6947ac2..053e019 100644 --- a/.drone.yml +++ b/.drone.yml @@ -94,5 +94,7 @@ steps: dockerfile: docs/Dockerfile trigger: + repo: + - borgmatic-collective/borgmatic branch: - master From 4d6ed27f738fe77c481ac4c3cded781fc06e00ea Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 23 Oct 2021 09:49:16 -0700 Subject: [PATCH 34/34] Add to changelog: Add support for old version (2.x) of jsonschema library. --- NEWS | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 1cf9e99..2d7db02 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.5.21.dev0 + * Add support for old version (2.x) of jsonschema library. + 1.5.20 * Re-release with correct version without dev0 tag. diff --git a/setup.py b/setup.py index 0feb1e1..11a83f1 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.5.20' +VERSION = '1.5.21.dev0' setup(