From 458e7776c554674cb91e8dd5fc280cddbe182ee5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 22 Oct 2019 16:28:42 -0700 Subject: [PATCH] Database dump hooks for PostgreSQL, so you can easily dump your databases before backups run (#225). --- README.md | 10 ++- borgmatic/borg/create.py | 18 +++++- borgmatic/commands/borgmatic.py | 8 ++- borgmatic/config/schema.yaml | 56 +++++++++++++++++ borgmatic/config/validate.py | 25 +++++--- borgmatic/execute.py | 32 +++++++--- borgmatic/hooks/postgresql.py | 88 +++++++++++++++++++++++++++ tests/integration/test_execute.py | 50 ++++++++++++--- tests/unit/borg/test_create.py | 47 ++++++++++++++ tests/unit/commands/test_borgmatic.py | 5 ++ tests/unit/config/test_validate.py | 21 +++++++ tests/unit/test_execute.py | 39 ++++++++++-- 12 files changed, 366 insertions(+), 33 deletions(-) create mode 100644 borgmatic/hooks/postgresql.py diff --git a/README.md b/README.md index 0d2dd49..e4d80ee 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,18 @@ retention: keep_monthly: 6 consistency: - # List of consistency checks to run: "repository", "archives", or both. + # List of consistency checks to run: "repository", "archives", etc. checks: - repository - archives + +hooks: + # Preparation scripts to run, databases to dump, and monitoring to perform. + before_backup: + - prepare-for-backup.sh + postgresql_databases: + - name: users + healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c ``` borgmatic is hosted at with [source code diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 97edf59..783c428 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -94,6 +94,20 @@ def _make_exclude_flags(location_config, exclude_filename=None): return exclude_from_flags + caches_flag + if_present_flags +BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic' + + +def borgmatic_source_directories(): + ''' + Return a list of borgmatic-specific source directories used for state like database backups. + ''' + return ( + [BORGMATIC_SOURCE_DIRECTORY] + if os.path.exists(os.path.expanduser(BORGMATIC_SOURCE_DIRECTORY)) + else [] + ) + + def create_archive( dry_run, repository, @@ -109,7 +123,9 @@ def create_archive( Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a storage config dict, create a Borg archive and return Borg's JSON output (if any). ''' - sources = _expand_directories(location_config['source_directories']) + sources = _expand_directories( + location_config['source_directories'] + borgmatic_source_directories() + ) pattern_file = _write_pattern_file(location_config.get('patterns')) exclude_file = _write_pattern_file( diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index d44c671..70318b3 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list from borgmatic.borg import prune as borg_prune from borgmatic.commands.arguments import parse_arguments from borgmatic.config import checks, collect, convert, validate -from borgmatic.hooks import command, healthchecks +from borgmatic.hooks import command, healthchecks, postgresql from borgmatic.logger import configure_logging, should_do_markup from borgmatic.signals import configure_signals from borgmatic.verbosity import verbosity_to_log_level @@ -60,6 +60,9 @@ def run_configuration(config_filename, config, arguments): 'pre-backup', global_arguments.dry_run, ) + postgresql.dump_databases( + hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run + ) healthchecks.ping_healthchecks( hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start' ) @@ -98,6 +101,9 @@ def run_configuration(config_filename, config, arguments): 'post-backup', global_arguments.dry_run, ) + postgresql.remove_database_dumps( + hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run + ) healthchecks.ping_healthchecks( hooks.get('healthchecks'), config_filename, global_arguments.dry_run ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 071d354..e113580 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -367,6 +367,62 @@ map: occurs during a backup or when running a before_backup or after_backup hook. example: - echo "Error while creating a backup or running a backup hook." + postgresql_databases: + seq: + - map: + name: + required: true + type: str + desc: | + Database name (required if using this hook). Or "all" to dump all + databases on the host. + example: users + hostname: + type: str + desc: | + Database hostname to connect to. Defaults to connecting via local + Unix socket. + example: database.example.org + port: + type: int + desc: Port to connect to. Defaults to 5432. + example: 5433 + username: + type: str + desc: | + Username with which to connect to the database. Defaults to the + username of the current user. You probably want to specify the + "postgres" superuser here when the database name is "all". + example: dbuser + password: + type: str + desc: | + Password with which to connect to the database. Omitting a password + will only work if PostgreSQL is configured to trust the configured + username without a password, or you create a ~/.pgpass file. + example: trustsome1 + format: + type: str + enum: ['plain', 'custom', 'directory', 'tar'] + desc: | + Database dump output format. One of "plain", "custom", "directory", + or "tar". Defaults to "custom" (unlike raw pg_dump). See + https://www.postgresql.org/docs/current/app-pgdump.html for details. + Note that format is ignored when the database name is "all". + example: directory + options: + type: str + desc: | + Additional pg_dump/pg_dumpall options to pass directly to the dump + command, without performing any validation on them. See + https://www.postgresql.org/docs/current/app-pgdump.html for details. + example: --role=someone + desc: | + List of one or more PostgreSQL databases to dump before creating a backup, + run once per configuration file. The database dumps are added to your source + directories at runtime, backed up, and then removed afterwards. Requires + pg_dump/pg_dumpall/pg_restore commands. See + https://www.postgresql.org/docs/current/app-pgdump.html for details. healthchecks: type: str desc: | diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 02ab704..8c9e8e9 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -64,6 +64,23 @@ def apply_logical_validation(config_filename, parsed_configuration): ) +def remove_examples(schema): + ''' + pykwalify gets angry if the example field is not a string. So rather than bend to its will, + remove all examples from the given schema before passing the schema to pykwalify. + ''' + if 'map' in schema: + for item_name, item_schema in schema['map'].items(): + item_schema.pop('example', None) + remove_examples(item_schema) + elif 'seq' in schema: + for item_schema in schema['seq']: + item_schema.pop('example', None) + remove_examples(item_schema) + + return schema + + def parse_configuration(config_filename, schema_filename): ''' Given the path to a config filename in YAML format and the path to a schema filename in @@ -84,13 +101,7 @@ def parse_configuration(config_filename, schema_filename): except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) - # pykwalify gets angry if the example field is not a string. So rather than bend to its will, - # remove all examples before passing the schema to pykwalify. - for section_name, section_schema in schema['map'].items(): - for field_name, field_schema in section_schema['map'].items(): - field_schema.pop('example', None) - - validator = pykwalify.core.Core(source_data=config, schema_data=schema) + validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema)) parsed_result = validator.validate(raise_exception=False) if validator.validation_errors: diff --git a/borgmatic/execute.py b/borgmatic/execute.py index d9ae1fc..0a64140 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -1,4 +1,5 @@ import logging +import os import subprocess logger = logging.getLogger(__name__) @@ -8,10 +9,17 @@ ERROR_OUTPUT_MAX_LINE_COUNT = 25 BORG_ERROR_EXIT_CODE = 2 -def execute_and_log_output(full_command, output_log_level, shell): +def borg_command(full_command): + ''' + Return True if this is a Borg command, or False if it's some other command. + ''' + return 'borg' in full_command[0] + + +def execute_and_log_output(full_command, output_log_level, shell, environment): last_lines = [] process = subprocess.Popen( - full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell + full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, env=environment ) while process.poll() is None: @@ -33,9 +41,11 @@ def execute_and_log_output(full_command, output_log_level, shell): exit_code = process.poll() - # If shell is True, assume we're running something other than Borg and should treat all non-zero - # exit codes as errors. - error = bool(exit_code != 0) if shell else bool(exit_code >= BORG_ERROR_EXIT_CODE) + # If we're running something other than Borg, treat all non-zero exit codes as errors. + if borg_command(full_command): + error = bool(exit_code >= BORG_ERROR_EXIT_CODE) + else: + error = bool(exit_code != 0) if error: # If an error occurs, include its output in the raised exception so that we don't @@ -48,21 +58,25 @@ def execute_and_log_output(full_command, output_log_level, shell): ) -def execute_command(full_command, output_log_level=logging.INFO, shell=False): +def execute_command( + full_command, output_log_level=logging.INFO, shell=False, extra_environment=None +): ''' Execute the given command (a sequence of command/argument strings) and log its output at the given log level. If output log level is None, instead capture and return the output. If - shell is True, execute the command within a shell. + shell is True, execute the command within a shell. If an extra environment dict is given, then + use it to augment the current environment, and pass the result into the command. Raise subprocesses.CalledProcessError if an error occurs while running the command. ''' logger.debug(' '.join(full_command)) + environment = {**os.environ, **extra_environment} if extra_environment else None if output_log_level is None: - output = subprocess.check_output(full_command, shell=shell) + output = subprocess.check_output(full_command, shell=shell, env=environment) return output.decode() if output is not None else None else: - execute_and_log_output(full_command, output_log_level, shell=shell) + execute_and_log_output(full_command, output_log_level, shell=shell, environment=environment) def execute_command_without_capture(full_command): diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py new file mode 100644 index 0000000..c6c8849 --- /dev/null +++ b/borgmatic/hooks/postgresql.py @@ -0,0 +1,88 @@ +import logging +import os + +from borgmatic.execute import execute_command + +DUMP_PATH = '~/.borgmatic/postgresql_databases' +logger = logging.getLogger(__name__) + + +def dump_databases(databases, config_filename, dry_run): + ''' + Dump the given PostgreSQL databases to disk. The databases are supplied as a sequence of dicts, + one dict describing each database as per the configuration schema. Use the given configuration + filename in any log entries. If this is a dry run, then don't actually dump anything. + ''' + if not databases: + logger.debug('{}: No PostgreSQL databases configured'.format(config_filename)) + return + + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + + logger.info('{}: Dumping PostgreSQL databases{}'.format(config_filename, dry_run_label)) + + for database in databases: + if os.path.sep in database['name']: + raise ValueError('Invalid database name {}'.format(database['name'])) + + dump_path = os.path.join( + os.path.expanduser(DUMP_PATH), database.get('hostname', 'localhost') + ) + name = database['name'] + all_databases = bool(name == 'all') + command = ( + ('pg_dumpall' if all_databases else 'pg_dump', '--no-password', '--clean') + + ('--file', os.path.join(dump_path, name)) + + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--port', str(database['port'])) if 'port' in database else ()) + + (('--username', database['username']) if 'username' in database else ()) + + (() if all_databases else ('--format', database.get('format', 'custom'))) + + (tuple(database['options'].split(' ')) if 'options' in database else ()) + + (() if all_databases else (name,)) + ) + extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None + + logger.debug( + '{}: Dumping PostgreSQL database {}{}'.format(config_filename, name, dry_run_label) + ) + if not dry_run: + os.makedirs(dump_path, mode=0o700, exist_ok=True) + execute_command(command, extra_environment=extra_environment) + + +def remove_database_dumps(databases, config_filename, dry_run): + ''' + Remove the database dumps for the given databases. The databases are supplied as a sequence of + dicts, one dict describing each database as per the configuration schema. Use the given + configuration filename in any log entries. If this is a dry run, then don't actually remove + anything. + ''' + if not databases: + logger.debug('{}: No PostgreSQL databases configured'.format(config_filename)) + return + + dry_run_label = ' (dry run; not actually removing anything)' if dry_run else '' + + logger.info('{}: Removing PostgreSQL database dumps{}'.format(config_filename, dry_run_label)) + + for database in databases: + if os.path.sep in database['name']: + raise ValueError('Invalid database name {}'.format(database['name'])) + + name = database['name'] + dump_path = os.path.join( + os.path.expanduser(DUMP_PATH), database.get('hostname', 'localhost') + ) + dump_filename = os.path.join(dump_path, name) + + logger.debug( + '{}: Remove PostgreSQL database dump {} from {}{}'.format( + config_filename, name, dump_filename, dry_run_label + ) + ) + if dry_run: + continue + + os.remove(dump_filename) + if len(os.listdir(dump_path)) == 0: + os.rmdir(dump_path) diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index fa7d4dd..48f3d62 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -7,36 +7,57 @@ from flexmock import flexmock from borgmatic import execute as module +def test_borg_command_identifies_borg_command(): + assert module.borg_command(['/usr/bin/borg1', 'info']) + + +def test_borg_command_does_not_identify_other_command(): + assert not module.borg_command(['grep', 'stuff']) + + def test_execute_and_log_output_logs_each_line_separately(): flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').once() flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once() + flexmock(module).should_receive('borg_command').and_return(False) - module.execute_and_log_output(['echo', 'hi'], output_log_level=logging.INFO, shell=False) - module.execute_and_log_output(['echo', 'there'], output_log_level=logging.INFO, shell=False) + module.execute_and_log_output( + ['echo', 'hi'], output_log_level=logging.INFO, shell=False, environment=None + ) + module.execute_and_log_output( + ['echo', 'there'], output_log_level=logging.INFO, shell=False, environment=None + ) def test_execute_and_log_output_with_borg_warning_does_not_raise(): flexmock(module.logger).should_receive('log') + flexmock(module).should_receive('borg_command').and_return(True) - # Borg's exit code 1 is a warning, not an error. - module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=False) + module.execute_and_log_output( + ['false'], output_log_level=logging.INFO, shell=False, environment=None + ) def test_execute_and_log_output_includes_borg_error_output_in_exception(): flexmock(module.logger).should_receive('log') + flexmock(module).should_receive('borg_command').and_return(True) with pytest.raises(subprocess.CalledProcessError) as error: - module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False) + module.execute_and_log_output( + ['grep'], output_log_level=logging.INFO, shell=False, environment=None + ) assert error.value.returncode == 2 assert error.value.output -def test_execute_and_log_output_with_shell_error_raises(): +def test_execute_and_log_output_with_non_borg_error_raises(): flexmock(module.logger).should_receive('log') + flexmock(module).should_receive('borg_command').and_return(False) with pytest.raises(subprocess.CalledProcessError) as error: - module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=True) + module.execute_and_log_output( + ['false'], output_log_level=logging.INFO, shell=False, environment=None + ) assert error.value.returncode == 1 @@ -44,9 +65,12 @@ def test_execute_and_log_output_with_shell_error_raises(): def test_execute_and_log_output_truncates_long_borg_error_output(): flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0 flexmock(module.logger).should_receive('log') + flexmock(module).should_receive('borg_command').and_return(False) with pytest.raises(subprocess.CalledProcessError) as error: - module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False) + module.execute_and_log_output( + ['grep'], output_log_level=logging.INFO, shell=False, environment=None + ) assert error.value.returncode == 2 assert error.value.output.startswith('...') @@ -54,12 +78,18 @@ def test_execute_and_log_output_truncates_long_borg_error_output(): def test_execute_and_log_output_with_no_output_logs_nothing(): flexmock(module.logger).should_receive('log').never() + flexmock(module).should_receive('borg_command').and_return(False) - module.execute_and_log_output(['true'], output_log_level=logging.INFO, shell=False) + module.execute_and_log_output( + ['true'], output_log_level=logging.INFO, shell=False, environment=None + ) def test_execute_and_log_output_with_error_exit_status_raises(): flexmock(module.logger).should_receive('log') + flexmock(module).should_receive('borg_command').and_return(False) with pytest.raises(subprocess.CalledProcessError): - module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False) + module.execute_and_log_output( + ['grep'], output_log_level=logging.INFO, shell=False, environment=None + ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 43bba80..b54e64b 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -156,11 +156,26 @@ def test_make_exclude_flags_is_empty_when_config_has_no_excludes(): assert exclude_flags == () +def test_borgmatic_source_directories_set_when_directory_exists(): + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('expanduser') + + assert module.borgmatic_source_directories() == [module.BORGMATIC_SOURCE_DIRECTORY] + + +def test_borgmatic_source_directories_empty_when_directory_does_not_exist(): + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.os.path).should_receive('expanduser') + + assert module.borgmatic_source_directories() == [] + + DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar') def test_create_archive_calls_borg_with_parameters(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -184,6 +199,7 @@ def test_create_archive_calls_borg_with_parameters(): def test_create_archive_with_patterns_calls_borg_with_patterns(): pattern_flags = ('--patterns-from', 'patterns') + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return( @@ -209,6 +225,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns(): def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): exclude_flags = ('--exclude-from', 'excludes') + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',)) flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return( @@ -233,6 +250,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): def test_create_archive_with_log_info_calls_borg_with_info_parameter(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -258,6 +276,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -283,6 +302,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -308,6 +328,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -332,6 +353,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -357,6 +379,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_parameter(): # --dry-run and --stats are mutually exclusive, see: # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -385,6 +408,7 @@ def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_param def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_parameter(): # --dry-run and --stats are mutually exclusive, see: # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -411,6 +435,7 @@ def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_para def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -434,6 +459,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -457,6 +483,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param def test_create_archive_with_compression_calls_borg_with_compression_parameters(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -480,6 +507,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -503,6 +531,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -526,6 +555,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -549,6 +579,7 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet def test_create_archive_with_read_special_calls_borg_with_read_special_parameter(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -573,6 +604,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags')) def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -597,6 +629,7 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags')) def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -621,6 +654,7 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -645,6 +679,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( def test_create_archive_with_local_path_calls_borg_via_local_path(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -668,6 +703,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -692,6 +728,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( def test_create_archive_with_umask_calls_borg_with_umask_parameters(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -714,6 +751,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -736,6 +774,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): def test_create_archive_with_stats_calls_borg_with_stats_parameter(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -759,6 +798,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter(): def test_create_archive_with_progress_calls_borg_with_progress_parameter(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -782,6 +822,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): def test_create_archive_with_json_calls_borg_with_json_parameter(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -807,6 +848,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -833,6 +875,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() def test_create_archive_with_source_directories_glob_expands(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -857,6 +900,7 @@ def test_create_archive_with_source_directories_glob_expands(): def test_create_archive_with_non_matching_source_directories_glob_passes_through(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo*',)) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -881,6 +925,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through def test_create_archive_with_glob_calls_borg_with_expanded_directories(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -904,6 +949,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) @@ -926,6 +972,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 1557af7..4712407 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -23,6 +23,9 @@ def test_run_configuration_runs_actions_for_each_repository(): def test_run_configuration_executes_hooks_for_create_action(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook').twice() + flexmock(module.postgresql).should_receive('dump_databases').once() + flexmock(module.healthchecks).should_receive('ping_healthchecks').twice() + flexmock(module.postgresql).should_receive('remove_database_dumps').once() flexmock(module).should_receive('run_actions').and_return([]) config = {'location': {'repositories': ['foo']}} arguments = {'global': flexmock(dry_run=False), 'create': flexmock()} @@ -33,6 +36,8 @@ def test_run_configuration_executes_hooks_for_create_action(): def test_run_configuration_logs_actions_error(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') + flexmock(module.postgresql).should_receive('dump_databases') + flexmock(module.healthchecks).should_receive('ping_healthchecks') expected_results = [flexmock()] flexmock(module).should_receive('make_error_log_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_raise(OSError) diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 99c489a..bdf30be 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -74,6 +74,27 @@ def test_apply_logical_validation_does_not_raise_otherwise(): module.apply_logical_validation('config.yaml', {'retention': {'keep_secondly': 1000}}) +def test_remove_examples_strips_examples_from_map(): + schema = { + 'map': { + 'foo': {'desc': 'thing1', 'example': 'bar'}, + 'baz': {'desc': 'thing2', 'example': 'quux'}, + } + } + + module.remove_examples(schema) + + assert schema == {'map': {'foo': {'desc': 'thing1'}, 'baz': {'desc': 'thing2'}}} + + +def test_remove_examples_strips_examples_from_sequence_of_maps(): + schema = {'seq': [{'map': {'foo': {'desc': 'thing', 'example': 'bar'}}, 'example': 'stuff'}]} + + module.remove_examples(schema) + + assert schema == {'seq': [{'map': {'foo': {'desc': 'thing'}}}]} + + def test_guard_configuration_contains_repository_does_not_raise_when_repository_in_config(): module.guard_configuration_contains_repository( repository='repo', configurations={'config.yaml': {'location': {'repositories': ['repo']}}} diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 81f44ae..91b9fac 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -8,8 +8,9 @@ from borgmatic import execute as module def test_execute_command_calls_full_command(): full_command = ['foo', 'bar'] + flexmock(module.os, environ={'a': 'b'}) flexmock(module).should_receive('execute_and_log_output').with_args( - full_command, output_log_level=logging.INFO, shell=False + full_command, output_log_level=logging.INFO, shell=False, environment=None ).once() output = module.execute_command(full_command) @@ -19,8 +20,9 @@ def test_execute_command_calls_full_command(): def test_execute_command_calls_full_command_with_shell(): full_command = ['foo', 'bar'] + flexmock(module.os, environ={'a': 'b'}) flexmock(module).should_receive('execute_and_log_output').with_args( - full_command, output_log_level=logging.INFO, shell=True + full_command, output_log_level=logging.INFO, shell=True, environment=None ).once() output = module.execute_command(full_command, shell=True) @@ -28,11 +30,24 @@ def test_execute_command_calls_full_command_with_shell(): assert output is None +def test_execute_command_calls_full_command_with_extra_environment(): + full_command = ['foo', 'bar'] + flexmock(module.os, environ={'a': 'b'}) + flexmock(module).should_receive('execute_and_log_output').with_args( + full_command, output_log_level=logging.INFO, shell=False, environment={'a': 'b', 'c': 'd'} + ).once() + + output = module.execute_command(full_command, extra_environment={'c': 'd'}) + + assert output is None + + def test_execute_command_captures_output(): full_command = ['foo', 'bar'] expected_output = '[]' + flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( - full_command, shell=False + full_command, shell=False, env=None ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command(full_command, output_log_level=None) @@ -43,8 +58,9 @@ def test_execute_command_captures_output(): def test_execute_command_captures_output_with_shell(): full_command = ['foo', 'bar'] expected_output = '[]' + flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( - full_command, shell=True + full_command, shell=True, env=None ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command(full_command, output_log_level=None, shell=True) @@ -52,6 +68,21 @@ def test_execute_command_captures_output_with_shell(): assert output == expected_output +def test_execute_command_captures_output_with_extra_environment(): + full_command = ['foo', 'bar'] + expected_output = '[]' + flexmock(module.os, environ={'a': 'b'}) + flexmock(module.subprocess).should_receive('check_output').with_args( + full_command, shell=False, env={'a': 'b', 'c': 'd'} + ).and_return(flexmock(decode=lambda: expected_output)).once() + + output = module.execute_command( + full_command, output_log_level=None, shell=False, extra_environment={'c': 'd'} + ) + + assert output == expected_output + + def test_execute_command_without_capture_does_not_raise_on_success(): flexmock(module.subprocess).should_receive('check_call').and_raise( module.subprocess.CalledProcessError(0, 'borg init')