Stream database dumps and restores directly to/from Borg without using any additional filesystem space (#258).
This commit is contained in:
parent
12cf6913ef
commit
a23fdf946d
14 changed files with 665 additions and 619 deletions
3
NEWS
3
NEWS
|
@ -1,4 +1,7 @@
|
||||||
1.5.3.dev0
|
1.5.3.dev0
|
||||||
|
* #258: Stream database dumps and restores directly to/from Borg without using any additional
|
||||||
|
filesystem space. This feature is automatic, and works even on restores from archives made with
|
||||||
|
previous versions of borgmatic.
|
||||||
* #293: Documentation on macOS launchd permissions issues with work-around for Full Disk Access.
|
* #293: Documentation on macOS launchd permissions issues with work-around for Full Disk Access.
|
||||||
|
|
||||||
1.5.2
|
1.5.2
|
||||||
|
|
|
@ -4,7 +4,11 @@ import logging
|
||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
from borgmatic.execute import execute_command, execute_command_without_capture
|
from borgmatic.execute import (
|
||||||
|
execute_command,
|
||||||
|
execute_command_with_processes,
|
||||||
|
execute_command_without_capture,
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -125,6 +129,9 @@ def borgmatic_source_directories(borgmatic_source_directory):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
|
||||||
|
|
||||||
|
|
||||||
def create_archive(
|
def create_archive(
|
||||||
dry_run,
|
dry_run,
|
||||||
repository,
|
repository,
|
||||||
|
@ -136,10 +143,14 @@ def create_archive(
|
||||||
stats=False,
|
stats=False,
|
||||||
json=False,
|
json=False,
|
||||||
files=False,
|
files=False,
|
||||||
|
stream_processes=None,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
|
Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
|
||||||
storage config dict, create a Borg archive and return Borg's JSON output (if any).
|
storage config dict, create a Borg archive and return Borg's JSON output (if any).
|
||||||
|
|
||||||
|
If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
|
||||||
|
create command while also triggering the given processes to produce output.
|
||||||
'''
|
'''
|
||||||
sources = _expand_directories(
|
sources = _expand_directories(
|
||||||
location_config['source_directories']
|
location_config['source_directories']
|
||||||
|
@ -157,8 +168,7 @@ def create_archive(
|
||||||
umask = storage_config.get('umask', None)
|
umask = storage_config.get('umask', None)
|
||||||
lock_wait = storage_config.get('lock_wait', None)
|
lock_wait = storage_config.get('lock_wait', None)
|
||||||
files_cache = location_config.get('files_cache')
|
files_cache = location_config.get('files_cache')
|
||||||
default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
|
archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
|
||||||
archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
|
|
||||||
extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
|
extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
|
||||||
|
|
||||||
full_command = (
|
full_command = (
|
||||||
|
@ -174,7 +184,7 @@ def create_archive(
|
||||||
+ (('--noatime',) if location_config.get('atime') is False else ())
|
+ (('--noatime',) if location_config.get('atime') is False else ())
|
||||||
+ (('--noctime',) if location_config.get('ctime') is False else ())
|
+ (('--noctime',) if location_config.get('ctime') is False else ())
|
||||||
+ (('--nobirthtime',) if location_config.get('birthtime') is False else ())
|
+ (('--nobirthtime',) if location_config.get('birthtime') is False else ())
|
||||||
+ (('--read-special',) if location_config.get('read_special') else ())
|
+ (('--read-special',) if (location_config.get('read_special') or stream_processes) else ())
|
||||||
+ (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
|
+ (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
|
||||||
+ (('--files-cache', files_cache) if files_cache else ())
|
+ (('--files-cache', files_cache) if files_cache else ())
|
||||||
+ (('--remote-path', remote_path) if remote_path else ())
|
+ (('--remote-path', remote_path) if remote_path else ())
|
||||||
|
@ -198,6 +208,7 @@ def create_archive(
|
||||||
|
|
||||||
# The progress output isn't compatible with captured and logged output, as progress messes with
|
# The progress output isn't compatible with captured and logged output, as progress messes with
|
||||||
# the terminal directly.
|
# the terminal directly.
|
||||||
|
# FIXME: "--progress" and stream_processes can't be used together.
|
||||||
if progress:
|
if progress:
|
||||||
execute_command_without_capture(full_command, error_on_warnings=False)
|
execute_command_without_capture(full_command, error_on_warnings=False)
|
||||||
return
|
return
|
||||||
|
@ -209,4 +220,9 @@ def create_archive(
|
||||||
else:
|
else:
|
||||||
output_log_level = logging.INFO
|
output_log_level = logging.INFO
|
||||||
|
|
||||||
|
if stream_processes:
|
||||||
|
return execute_command_with_processes(
|
||||||
|
full_command, stream_processes, output_log_level, error_on_warnings=False
|
||||||
|
)
|
||||||
|
|
||||||
return execute_command(full_command, output_log_level, error_on_warnings=False)
|
return execute_command(full_command, output_log_level, error_on_warnings=False)
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
from borgmatic.execute import execute_command, execute_command_without_capture
|
from borgmatic.execute import execute_command, execute_command_without_capture
|
||||||
|
|
||||||
|
@ -63,12 +64,16 @@ def extract_archive(
|
||||||
destination_path=None,
|
destination_path=None,
|
||||||
progress=False,
|
progress=False,
|
||||||
error_on_warnings=True,
|
error_on_warnings=True,
|
||||||
|
extract_to_stdout=False,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
|
Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
|
||||||
restore from the archive, location/storage configuration dicts, optional local and remote Borg
|
restore from the archive, location/storage configuration dicts, optional local and remote Borg
|
||||||
paths, and an optional destination path to extract to, extract the archive into the current
|
paths, and an optional destination path to extract to, extract the archive into the current
|
||||||
directory.
|
directory.
|
||||||
|
|
||||||
|
If extract to stdout is True, then start the extraction streaming to stdout, and return that
|
||||||
|
extract process as an instance of subprocess.Popen.
|
||||||
'''
|
'''
|
||||||
umask = storage_config.get('umask', None)
|
umask = storage_config.get('umask', None)
|
||||||
lock_wait = storage_config.get('lock_wait', None)
|
lock_wait = storage_config.get('lock_wait', None)
|
||||||
|
@ -83,6 +88,7 @@ def extract_archive(
|
||||||
+ (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
+ (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||||
+ (('--dry-run',) if dry_run else ())
|
+ (('--dry-run',) if dry_run else ())
|
||||||
+ (('--progress',) if progress else ())
|
+ (('--progress',) if progress else ())
|
||||||
|
+ (('--stdout',) if extract_to_stdout else ())
|
||||||
+ ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
|
+ ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
|
||||||
+ (tuple(paths) if paths else ())
|
+ (tuple(paths) if paths else ())
|
||||||
)
|
)
|
||||||
|
@ -93,7 +99,16 @@ def extract_archive(
|
||||||
execute_command_without_capture(
|
execute_command_without_capture(
|
||||||
full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
|
full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
|
||||||
)
|
)
|
||||||
return
|
return None
|
||||||
|
|
||||||
|
if extract_to_stdout:
|
||||||
|
return execute_command(
|
||||||
|
full_command,
|
||||||
|
output_file=subprocess.PIPE,
|
||||||
|
working_directory=destination_path,
|
||||||
|
error_on_warnings=error_on_warnings,
|
||||||
|
run_to_completion=False,
|
||||||
|
)
|
||||||
|
|
||||||
# Error on warnings by default, as Borg only gives a warning if the restore paths don't exist in
|
# Error on warnings by default, as Borg only gives a warning if the restore paths don't exist in
|
||||||
# the archive!
|
# the archive!
|
||||||
|
|
|
@ -83,14 +83,6 @@ def run_configuration(config_filename, config, arguments):
|
||||||
'pre-backup',
|
'pre-backup',
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
)
|
)
|
||||||
dispatch.call_hooks(
|
|
||||||
'dump_databases',
|
|
||||||
hooks,
|
|
||||||
config_filename,
|
|
||||||
dump.DATABASE_HOOK_NAMES,
|
|
||||||
location,
|
|
||||||
global_arguments.dry_run,
|
|
||||||
)
|
|
||||||
if 'check' in arguments:
|
if 'check' in arguments:
|
||||||
command.execute_hook(
|
command.execute_hook(
|
||||||
hooks.get('before_check'),
|
hooks.get('before_check'),
|
||||||
|
@ -262,6 +254,16 @@ def run_actions(
|
||||||
)
|
)
|
||||||
if 'create' in arguments:
|
if 'create' in arguments:
|
||||||
logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
|
logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
|
||||||
|
active_dumps = dispatch.call_hooks(
|
||||||
|
'dump_databases',
|
||||||
|
hooks,
|
||||||
|
repository,
|
||||||
|
dump.DATABASE_HOOK_NAMES,
|
||||||
|
location,
|
||||||
|
global_arguments.dry_run,
|
||||||
|
)
|
||||||
|
stream_processes = [process for processes in active_dumps.values() for process in processes]
|
||||||
|
|
||||||
json_output = borg_create.create_archive(
|
json_output = borg_create.create_archive(
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
repository,
|
repository,
|
||||||
|
@ -273,9 +275,11 @@ def run_actions(
|
||||||
stats=arguments['create'].stats,
|
stats=arguments['create'].stats,
|
||||||
json=arguments['create'].json,
|
json=arguments['create'].json,
|
||||||
files=arguments['create'].files,
|
files=arguments['create'].files,
|
||||||
|
stream_processes=stream_processes,
|
||||||
)
|
)
|
||||||
if json_output:
|
if json_output:
|
||||||
yield json.loads(json_output)
|
yield json.loads(json_output)
|
||||||
|
|
||||||
if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
|
if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
|
||||||
logger.info('{}: Running consistency checks'.format(repository))
|
logger.info('{}: Running consistency checks'.format(repository))
|
||||||
borg_check.check_archives(
|
borg_check.check_archives(
|
||||||
|
@ -347,57 +351,67 @@ def run_actions(
|
||||||
if 'all' in restore_names:
|
if 'all' in restore_names:
|
||||||
restore_names = []
|
restore_names = []
|
||||||
|
|
||||||
# Extract dumps for the named databases from the archive.
|
archive_name = borg_list.resolve_archive_name(
|
||||||
dump_patterns = dispatch.call_hooks(
|
repository, arguments['restore'].archive, storage, local_path, remote_path
|
||||||
'make_database_dump_patterns',
|
|
||||||
hooks,
|
|
||||||
repository,
|
|
||||||
dump.DATABASE_HOOK_NAMES,
|
|
||||||
location,
|
|
||||||
restore_names,
|
|
||||||
)
|
)
|
||||||
|
found_names = set()
|
||||||
|
|
||||||
borg_extract.extract_archive(
|
for hook_name, per_hook_restore_databases in hooks.items():
|
||||||
global_arguments.dry_run,
|
if hook_name not in dump.DATABASE_HOOK_NAMES:
|
||||||
repository,
|
continue
|
||||||
borg_list.resolve_archive_name(
|
|
||||||
repository, arguments['restore'].archive, storage, local_path, remote_path
|
|
||||||
),
|
|
||||||
dump.convert_glob_patterns_to_borg_patterns(
|
|
||||||
dump.flatten_dump_patterns(dump_patterns, restore_names)
|
|
||||||
),
|
|
||||||
location,
|
|
||||||
storage,
|
|
||||||
local_path=local_path,
|
|
||||||
remote_path=remote_path,
|
|
||||||
destination_path='/',
|
|
||||||
progress=arguments['restore'].progress,
|
|
||||||
# We don't want glob patterns that don't match to error.
|
|
||||||
error_on_warnings=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Map the restore names or detected dumps to the corresponding database configurations.
|
for restore_database in per_hook_restore_databases:
|
||||||
restore_databases = dump.get_per_hook_database_configurations(
|
database_name = restore_database['name']
|
||||||
hooks, restore_names, dump_patterns
|
if restore_names and database_name not in restore_names:
|
||||||
)
|
continue
|
||||||
|
|
||||||
|
found_names.add(database_name)
|
||||||
|
dump_pattern = dispatch.call_hooks(
|
||||||
|
'make_database_dump_pattern',
|
||||||
|
hooks,
|
||||||
|
repository,
|
||||||
|
dump.DATABASE_HOOK_NAMES,
|
||||||
|
location,
|
||||||
|
database_name,
|
||||||
|
)[hook_name]
|
||||||
|
|
||||||
|
# Kick off a single database extract to stdout.
|
||||||
|
extract_process = borg_extract.extract_archive(
|
||||||
|
dry_run=global_arguments.dry_run,
|
||||||
|
repository=repository,
|
||||||
|
archive=archive_name,
|
||||||
|
paths=dump.convert_glob_patterns_to_borg_patterns([dump_pattern]),
|
||||||
|
location_config=location,
|
||||||
|
storage_config=storage,
|
||||||
|
local_path=local_path,
|
||||||
|
remote_path=remote_path,
|
||||||
|
destination_path='/',
|
||||||
|
progress=arguments['restore'].progress,
|
||||||
|
extract_to_stdout=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Run a single database restore, consuming the extract stdout.
|
||||||
|
dispatch.call_hooks(
|
||||||
|
'restore_database_dump',
|
||||||
|
{hook_name: [restore_database]},
|
||||||
|
repository,
|
||||||
|
dump.DATABASE_HOOK_NAMES,
|
||||||
|
location,
|
||||||
|
global_arguments.dry_run,
|
||||||
|
extract_process,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not restore_names and not found_names:
|
||||||
|
raise ValueError('No databases were found to restore')
|
||||||
|
|
||||||
|
missing_names = sorted(set(restore_names) - found_names)
|
||||||
|
if missing_names:
|
||||||
|
raise ValueError(
|
||||||
|
'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format(
|
||||||
|
', '.join(missing_names)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
# Finally, restore the databases and cleanup the dumps.
|
|
||||||
dispatch.call_hooks(
|
|
||||||
'restore_database_dumps',
|
|
||||||
restore_databases,
|
|
||||||
repository,
|
|
||||||
dump.DATABASE_HOOK_NAMES,
|
|
||||||
location,
|
|
||||||
global_arguments.dry_run,
|
|
||||||
)
|
|
||||||
dispatch.call_hooks(
|
|
||||||
'remove_database_dumps',
|
|
||||||
restore_databases,
|
|
||||||
repository,
|
|
||||||
dump.DATABASE_HOOK_NAMES,
|
|
||||||
location,
|
|
||||||
global_arguments.dry_run,
|
|
||||||
)
|
|
||||||
if 'list' in arguments:
|
if 'list' in arguments:
|
||||||
if arguments['list'].repository is None or validate.repositories_match(
|
if arguments['list'].repository is None or validate.repositories_match(
|
||||||
repository, arguments['list'].repository
|
repository, arguments['list'].repository
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
|
import collections
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import select
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -20,12 +22,20 @@ def exit_code_indicates_error(exit_code, error_on_warnings=True):
|
||||||
return bool(exit_code >= BORG_ERROR_EXIT_CODE)
|
return bool(exit_code >= BORG_ERROR_EXIT_CODE)
|
||||||
|
|
||||||
|
|
||||||
def log_output(command, process, output_buffer, output_log_level, error_on_warnings):
|
def process_command(process):
|
||||||
'''
|
'''
|
||||||
Given a command already executed, its process opened by subprocess.Popen(), and the process'
|
Given a process as an instance of subprocess.Popen, return the command string that was used to
|
||||||
relevant output buffer (stderr or stdout), log its output with the requested log level.
|
invoke it.
|
||||||
Additionally, raise a CalledProcessException if the process exits with an error (or a warning,
|
'''
|
||||||
if error on warnings is True).
|
return process.args if isinstance(process.args, str) else ' '.join(process.args)
|
||||||
|
|
||||||
|
|
||||||
|
def log_output(process, output_buffer, output_log_level, error_on_warnings):
|
||||||
|
'''
|
||||||
|
Given an executed command's process opened by subprocess.Popen(), and the process' relevant
|
||||||
|
output buffer (stderr or stdout), log its output with the requested log level. Additionally,
|
||||||
|
raise a CalledProcessError if the process exits with an error (or a warning, if error on
|
||||||
|
warnings is True).
|
||||||
'''
|
'''
|
||||||
last_lines = []
|
last_lines = []
|
||||||
|
|
||||||
|
@ -54,7 +64,86 @@ def log_output(command, process, output_buffer, output_log_level, error_on_warni
|
||||||
if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
|
if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
|
||||||
last_lines.insert(0, '...')
|
last_lines.insert(0, '...')
|
||||||
|
|
||||||
raise subprocess.CalledProcessError(exit_code, ' '.join(command), '\n'.join(last_lines))
|
raise subprocess.CalledProcessError(
|
||||||
|
exit_code, process_command(process), '\n'.join(last_lines)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def output_buffer_for_process(process, exclude_stdouts):
|
||||||
|
'''
|
||||||
|
Given an instance of subprocess.Popen and a sequence of stdouts to exclude, return either the
|
||||||
|
process's stdout or stderr. The idea is that if stdout is excluded for a process, we still have
|
||||||
|
stderr to log.
|
||||||
|
'''
|
||||||
|
return process.stderr if process.stdout in exclude_stdouts else process.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def log_many_outputs(processes, exclude_stdouts, output_log_level, error_on_warnings):
|
||||||
|
'''
|
||||||
|
Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
|
||||||
|
process with the requested log level. Additionally, raise a CalledProcessError if a process
|
||||||
|
exits with an error (or a warning, if error on warnings is True).
|
||||||
|
|
||||||
|
For simplicity, it's assumed that the output buffer for each process is its stdout. But if any
|
||||||
|
stdouts are given to exclude, then for any matching processes, log from their stderr instead.
|
||||||
|
'''
|
||||||
|
# Map from output buffer to sequence of last lines.
|
||||||
|
buffer_last_lines = collections.defaultdict(list)
|
||||||
|
output_buffers = [output_buffer_for_process(process, exclude_stdouts) for process in processes]
|
||||||
|
|
||||||
|
while True:
|
||||||
|
(ready_buffers, _, _) = select.select(output_buffers, [], [])
|
||||||
|
|
||||||
|
for ready_buffer in ready_buffers:
|
||||||
|
line = ready_buffer.readline().rstrip().decode()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Keep the last few lines of output in case the process errors, and we need the output for
|
||||||
|
# the exception below.
|
||||||
|
last_lines = buffer_last_lines[ready_buffer]
|
||||||
|
last_lines.append(line)
|
||||||
|
if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
|
||||||
|
last_lines.pop(0)
|
||||||
|
|
||||||
|
logger.log(output_log_level, line)
|
||||||
|
|
||||||
|
if all(process.poll() is not None for process in processes):
|
||||||
|
break
|
||||||
|
|
||||||
|
for process in processes:
|
||||||
|
remaining_output = (
|
||||||
|
output_buffer_for_process(process, exclude_stdouts).read().rstrip().decode()
|
||||||
|
)
|
||||||
|
if remaining_output: # pragma: no cover
|
||||||
|
logger.log(output_log_level, remaining_output)
|
||||||
|
|
||||||
|
for process in processes:
|
||||||
|
exit_code = process.poll()
|
||||||
|
|
||||||
|
if exit_code_indicates_error(exit_code, error_on_warnings):
|
||||||
|
# If an error occurs, include its output in the raised exception so that we don't
|
||||||
|
# inadvertently hide error output.
|
||||||
|
output_buffer = output_buffer_for_process(process, exclude_stdouts)
|
||||||
|
last_lines = buffer_last_lines[output_buffer]
|
||||||
|
if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
|
||||||
|
last_lines.insert(0, '...')
|
||||||
|
|
||||||
|
raise subprocess.CalledProcessError(
|
||||||
|
exit_code, process_command(process), '\n'.join(last_lines)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def log_command(full_command, input_file, output_file):
|
||||||
|
'''
|
||||||
|
Log the given command (a sequence of command/argument strings), along with its input/output file
|
||||||
|
paths.
|
||||||
|
'''
|
||||||
|
logger.debug(
|
||||||
|
' '.join(full_command)
|
||||||
|
+ (' < {}'.format(getattr(input_file, 'name', '')) if input_file else '')
|
||||||
|
+ (' > {}'.format(getattr(output_file, 'name', '')) if output_file else '')
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def execute_command(
|
def execute_command(
|
||||||
|
@ -66,24 +155,23 @@ def execute_command(
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
working_directory=None,
|
working_directory=None,
|
||||||
error_on_warnings=True,
|
error_on_warnings=True,
|
||||||
|
run_to_completion=True,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Execute the given command (a sequence of command/argument strings) and log its output at the
|
Execute the given command (a sequence of command/argument strings) and log its output at the
|
||||||
given log level. If output log level is None, instead capture and return the output. If an
|
given log level. If output log level is None, instead capture and return the output. (Implies
|
||||||
open output file object is given, then write stdout to the file and only log stderr (but only
|
run_to_completion.) If an open output file object is given, then write stdout to the file and
|
||||||
if an output log level is set). If an open input file object is given, then read stdin from the
|
only log stderr (but only if an output log level is set). If an open input file object is given,
|
||||||
file. If shell is True, execute the command within a shell. If an extra environment dict is
|
then read stdin from the file. If shell is True, execute the command within a shell. If an extra
|
||||||
given, then use it to augment the current environment, and pass the result into the command. If
|
environment dict is given, then use it to augment the current environment, and pass the result
|
||||||
a working directory is given, use that as the present working directory when running the
|
into the command. If a working directory is given, use that as the present working directory
|
||||||
command. If error on warnings is False, then treat exit code 1 as a warning instead of an error.
|
when running the command. If error on warnings is False, then treat exit code 1 as a warning
|
||||||
|
instead of an error. If run to completion is False, then return the process for the command
|
||||||
|
without executing it to completion.
|
||||||
|
|
||||||
Raise subprocesses.CalledProcessError if an error occurs while running the command.
|
Raise subprocesses.CalledProcessError if an error occurs while running the command.
|
||||||
'''
|
'''
|
||||||
logger.debug(
|
log_command(full_command, input_file, output_file)
|
||||||
' '.join(full_command)
|
|
||||||
+ (' < {}'.format(input_file.name) if input_file else '')
|
|
||||||
+ (' > {}'.format(output_file.name) if output_file else '')
|
|
||||||
)
|
|
||||||
environment = {**os.environ, **extra_environment} if extra_environment else None
|
environment = {**os.environ, **extra_environment} if extra_environment else None
|
||||||
|
|
||||||
if output_log_level is None:
|
if output_log_level is None:
|
||||||
|
@ -93,7 +181,7 @@ def execute_command(
|
||||||
return output.decode() if output is not None else None
|
return output.decode() if output is not None else None
|
||||||
else:
|
else:
|
||||||
process = subprocess.Popen(
|
process = subprocess.Popen(
|
||||||
full_command,
|
' '.join(full_command) if shell else full_command,
|
||||||
stdin=input_file,
|
stdin=input_file,
|
||||||
stdout=output_file or subprocess.PIPE,
|
stdout=output_file or subprocess.PIPE,
|
||||||
stderr=subprocess.PIPE if output_file else subprocess.STDOUT,
|
stderr=subprocess.PIPE if output_file else subprocess.STDOUT,
|
||||||
|
@ -101,8 +189,10 @@ def execute_command(
|
||||||
env=environment,
|
env=environment,
|
||||||
cwd=working_directory,
|
cwd=working_directory,
|
||||||
)
|
)
|
||||||
|
if not run_to_completion:
|
||||||
|
return process
|
||||||
|
|
||||||
log_output(
|
log_output(
|
||||||
full_command,
|
|
||||||
process,
|
process,
|
||||||
process.stderr if output_file else process.stdout,
|
process.stderr if output_file else process.stdout,
|
||||||
output_log_level,
|
output_log_level,
|
||||||
|
@ -126,3 +216,61 @@ def execute_command_without_capture(full_command, working_directory=None, error_
|
||||||
except subprocess.CalledProcessError as error:
|
except subprocess.CalledProcessError as error:
|
||||||
if exit_code_indicates_error(error.returncode, error_on_warnings):
|
if exit_code_indicates_error(error.returncode, error_on_warnings):
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def execute_command_with_processes(
|
||||||
|
full_command,
|
||||||
|
processes,
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
output_file=None,
|
||||||
|
input_file=None,
|
||||||
|
shell=False,
|
||||||
|
extra_environment=None,
|
||||||
|
working_directory=None,
|
||||||
|
error_on_warnings=True,
|
||||||
|
):
|
||||||
|
'''
|
||||||
|
Execute the given command (a sequence of command/argument strings) and log its output at the
|
||||||
|
given log level. Simultaneously, continue to poll one or more active processes so that they
|
||||||
|
run as well. This is useful, for instance, for processes that are streaming output to a named
|
||||||
|
pipe that the given command is consuming from.
|
||||||
|
|
||||||
|
If an open output file object is given, then write stdout to the file and only log stderr (but
|
||||||
|
only if an output log level is set). If an open input file object is given, then read stdin from
|
||||||
|
the file. If shell is True, execute the command within a shell. If an extra environment dict is
|
||||||
|
given, then use it to augment the current environment, and pass the result into the command. If
|
||||||
|
a working directory is given, use that as the present working directory when running the
|
||||||
|
command. If error on warnings is False, then treat exit code 1 as a warning instead of an
|
||||||
|
error.
|
||||||
|
|
||||||
|
Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
|
||||||
|
upstream process.
|
||||||
|
'''
|
||||||
|
log_command(full_command, input_file, output_file)
|
||||||
|
environment = {**os.environ, **extra_environment} if extra_environment else None
|
||||||
|
|
||||||
|
try:
|
||||||
|
command_process = subprocess.Popen(
|
||||||
|
full_command,
|
||||||
|
stdin=input_file,
|
||||||
|
stdout=output_file or subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE if output_file else subprocess.STDOUT,
|
||||||
|
shell=shell,
|
||||||
|
env=environment,
|
||||||
|
cwd=working_directory,
|
||||||
|
)
|
||||||
|
except (subprocess.CalledProcessError, OSError):
|
||||||
|
# Something has gone wrong. So vent each process' output buffer to prevent it from hanging.
|
||||||
|
# And then kill the process.
|
||||||
|
for process in processes:
|
||||||
|
if process.poll() is None:
|
||||||
|
process.stdout.read(0)
|
||||||
|
process.kill()
|
||||||
|
raise
|
||||||
|
|
||||||
|
log_many_outputs(
|
||||||
|
tuple(processes) + (command_process,),
|
||||||
|
(input_file, output_file),
|
||||||
|
output_log_level,
|
||||||
|
error_on_warnings,
|
||||||
|
)
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
import glob
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
@ -34,24 +33,13 @@ def make_database_dump_filename(dump_path, name, hostname=None):
|
||||||
return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
|
return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
|
||||||
|
|
||||||
|
|
||||||
def flatten_dump_patterns(dump_patterns, names):
|
def create_named_pipe_for_dump(dump_path):
|
||||||
'''
|
'''
|
||||||
Given a dict from a database hook name to glob patterns matching the dumps for the named
|
Create a named pipe at the given dump path.
|
||||||
databases, flatten out all the glob patterns into a single sequence, and return it.
|
|
||||||
|
|
||||||
Raise ValueError if there are no resulting glob patterns, which indicates that databases are not
|
|
||||||
configured in borgmatic's configuration.
|
|
||||||
'''
|
'''
|
||||||
flattened = [pattern for patterns in dump_patterns.values() for pattern in patterns]
|
os.makedirs(os.path.dirname(dump_path), mode=0o700, exist_ok=True)
|
||||||
|
if not os.path.exists(dump_path):
|
||||||
if not flattened:
|
os.mkfifo(dump_path, mode=0o600)
|
||||||
raise ValueError(
|
|
||||||
'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format(
|
|
||||||
', '.join(names) or '"all"'
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
return flattened
|
|
||||||
|
|
||||||
|
|
||||||
def remove_database_dumps(dump_path, databases, database_type_name, log_prefix, dry_run):
|
def remove_database_dumps(dump_path, databases, database_type_name, log_prefix, dry_run):
|
||||||
|
@ -100,80 +88,3 @@ def convert_glob_patterns_to_borg_patterns(patterns):
|
||||||
patterns like "sh:etc/*".
|
patterns like "sh:etc/*".
|
||||||
'''
|
'''
|
||||||
return ['sh:{}'.format(pattern.lstrip(os.path.sep)) for pattern in patterns]
|
return ['sh:{}'.format(pattern.lstrip(os.path.sep)) for pattern in patterns]
|
||||||
|
|
||||||
|
|
||||||
def get_database_names_from_dumps(patterns):
|
|
||||||
'''
|
|
||||||
Given a sequence of database dump patterns, find the corresponding database dumps on disk and
|
|
||||||
return the database names from their filenames.
|
|
||||||
'''
|
|
||||||
return [os.path.basename(dump_path) for pattern in patterns for dump_path in glob.glob(pattern)]
|
|
||||||
|
|
||||||
|
|
||||||
def get_database_configurations(databases, names):
|
|
||||||
'''
|
|
||||||
Given the full database configuration dicts as per the configuration schema, and a sequence of
|
|
||||||
database names, filter down and yield the configuration for just the named databases.
|
|
||||||
Additionally, if a database configuration is named "all", project out that configuration for
|
|
||||||
each named database.
|
|
||||||
'''
|
|
||||||
named_databases = {database['name']: database for database in databases}
|
|
||||||
|
|
||||||
for name in names:
|
|
||||||
database = named_databases.get(name)
|
|
||||||
if database:
|
|
||||||
yield database
|
|
||||||
continue
|
|
||||||
|
|
||||||
if 'all' in named_databases:
|
|
||||||
yield {**named_databases['all'], **{'name': name}}
|
|
||||||
continue
|
|
||||||
|
|
||||||
|
|
||||||
def get_per_hook_database_configurations(hooks, names, dump_patterns):
|
|
||||||
'''
|
|
||||||
Given the hooks configuration dict as per the configuration schema, a sequence of database
|
|
||||||
names to restore, and a dict from database hook name to glob patterns for matching dumps,
|
|
||||||
filter down the configuration for just the named databases.
|
|
||||||
|
|
||||||
If there are no named databases given, then find the corresponding database dumps on disk and
|
|
||||||
use the database names from their filenames. Additionally, if a database configuration is named
|
|
||||||
"all", project out that configuration for each named database.
|
|
||||||
|
|
||||||
Return the results as a dict from database hook name to a sequence of database configuration
|
|
||||||
dicts for that database type.
|
|
||||||
|
|
||||||
Raise ValueError if one of the database names cannot be matched to a database in borgmatic's
|
|
||||||
database configuration.
|
|
||||||
'''
|
|
||||||
hook_databases = {
|
|
||||||
hook_name: list(
|
|
||||||
get_database_configurations(
|
|
||||||
hooks.get(hook_name),
|
|
||||||
names or get_database_names_from_dumps(dump_patterns[hook_name]),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
for hook_name in DATABASE_HOOK_NAMES
|
|
||||||
if hook_name in hooks
|
|
||||||
}
|
|
||||||
|
|
||||||
if not names or 'all' in names:
|
|
||||||
if not any(hook_databases.values()):
|
|
||||||
raise ValueError(
|
|
||||||
'Cannot restore database "all", as there are no database dumps in the archive'
|
|
||||||
)
|
|
||||||
|
|
||||||
return hook_databases
|
|
||||||
|
|
||||||
found_names = {
|
|
||||||
database['name'] for databases in hook_databases.values() for database in databases
|
|
||||||
}
|
|
||||||
missing_names = sorted(set(names) - found_names)
|
|
||||||
if missing_names:
|
|
||||||
raise ValueError(
|
|
||||||
'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format(
|
|
||||||
', '.join(missing_names)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
return hook_databases
|
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
|
|
||||||
from borgmatic.execute import execute_command
|
from borgmatic.execute import execute_command, execute_command_with_processes
|
||||||
from borgmatic.hooks import dump
|
from borgmatic.hooks import dump
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -55,12 +54,16 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe
|
||||||
|
|
||||||
def dump_databases(databases, log_prefix, location_config, dry_run):
|
def dump_databases(databases, log_prefix, location_config, dry_run):
|
||||||
'''
|
'''
|
||||||
Dump the given MySQL/MariaDB databases to disk. The databases are supplied as a sequence of
|
Dump the given MySQL/MariaDB databases to a named pipe. The databases are supplied as a sequence
|
||||||
dicts, one dict describing each database as per the configuration schema. Use the given log
|
of dicts, one dict describing each database as per the configuration schema. Use the given log
|
||||||
prefix in any log entries. Use the given location configuration dict to construct the
|
prefix in any log entries. Use the given location configuration dict to construct the
|
||||||
destination path. If this is a dry run, then don't actually dump anything.
|
destination path.
|
||||||
|
|
||||||
|
Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
|
||||||
|
pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
|
||||||
|
processes = []
|
||||||
|
|
||||||
logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label))
|
logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label))
|
||||||
|
|
||||||
|
@ -75,7 +78,8 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
|
||||||
)
|
)
|
||||||
|
|
||||||
dump_command = (
|
dump_command = (
|
||||||
('mysqldump', '--add-drop-database')
|
('mysqldump',)
|
||||||
|
+ ('--add-drop-database',)
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
|
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
|
||||||
|
@ -83,6 +87,9 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
|
||||||
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
||||||
+ ('--databases',)
|
+ ('--databases',)
|
||||||
+ dump_command_names
|
+ dump_command_names
|
||||||
|
# Use shell redirection rather than execute_command(output_file=open(...)) to prevent
|
||||||
|
# the open() call on a named pipe from hanging the main borgmatic process.
|
||||||
|
+ ('>', dump_filename)
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.debug(
|
logger.debug(
|
||||||
|
@ -90,13 +97,21 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
|
||||||
log_prefix, requested_name, dump_filename, dry_run_label
|
log_prefix, requested_name, dump_filename, dry_run_label
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if not dry_run:
|
if dry_run:
|
||||||
os.makedirs(os.path.dirname(dump_filename), mode=0o700, exist_ok=True)
|
continue
|
||||||
|
|
||||||
|
dump.create_named_pipe_for_dump(dump_filename)
|
||||||
|
|
||||||
|
processes.append(
|
||||||
execute_command(
|
execute_command(
|
||||||
dump_command,
|
dump_command,
|
||||||
output_file=open(dump_filename, 'w'),
|
shell=True,
|
||||||
extra_environment=extra_environment,
|
extra_environment=extra_environment,
|
||||||
|
run_to_completion=False,
|
||||||
)
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return processes
|
||||||
|
|
||||||
|
|
||||||
def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover
|
def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover
|
||||||
|
@ -111,45 +126,47 @@ def remove_database_dumps(databases, log_prefix, location_config, dry_run): # p
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def make_database_dump_patterns(databases, log_prefix, location_config, names):
|
def make_database_dump_pattern(databases, log_prefix, location_config, name=None):
|
||||||
'''
|
'''
|
||||||
Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
|
Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
|
||||||
and a sequence of database names to match, return the corresponding glob patterns to match the
|
and a database name to match, return the corresponding glob patterns to match the database dump
|
||||||
database dumps in an archive. An empty sequence of names indicates that the patterns should
|
in an archive.
|
||||||
match all dumps.
|
|
||||||
'''
|
'''
|
||||||
return [
|
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
|
||||||
dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
|
|
||||||
for name in (names or ['*'])
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def restore_database_dumps(databases, log_prefix, location_config, dry_run):
|
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
|
||||||
'''
|
'''
|
||||||
Restore the given MySQL/MariaDB databases from disk. The databases are supplied as a sequence of
|
Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
|
||||||
dicts, one dict describing each database as per the configuration schema. Use the given log
|
one-element sequence containing a dict describing the database, as per the configuration schema.
|
||||||
prefix in any log entries. Use the given location configuration dict to construct the
|
Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
|
||||||
destination path. If this is a dry run, then don't actually restore anything.
|
anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
|
||||||
|
output to consume.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
||||||
|
|
||||||
for database in databases:
|
if len(database_config) != 1:
|
||||||
dump_filename = dump.make_database_dump_filename(
|
raise ValueError('The database configuration value is invalid')
|
||||||
make_dump_path(location_config), database['name'], database.get('hostname')
|
|
||||||
)
|
|
||||||
restore_command = (
|
|
||||||
('mysql', '--batch')
|
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
|
||||||
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
|
|
||||||
+ (('--user', database['username']) if 'username' in database else ())
|
|
||||||
)
|
|
||||||
extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
|
|
||||||
|
|
||||||
logger.debug(
|
database = database_config[0]
|
||||||
'{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
|
restore_command = (
|
||||||
)
|
('mysql', '--batch')
|
||||||
if not dry_run:
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
execute_command(
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
restore_command, input_file=open(dump_filename), extra_environment=extra_environment
|
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
|
||||||
)
|
+ (('--user', database['username']) if 'username' in database else ())
|
||||||
|
)
|
||||||
|
extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
'{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
|
||||||
|
)
|
||||||
|
if dry_run:
|
||||||
|
return
|
||||||
|
|
||||||
|
execute_command_with_processes(
|
||||||
|
restore_command,
|
||||||
|
[extract_process],
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment=extra_environment,
|
||||||
|
)
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
|
|
||||||
from borgmatic.execute import execute_command
|
from borgmatic.execute import execute_command, execute_command_with_processes
|
||||||
from borgmatic.hooks import dump
|
from borgmatic.hooks import dump
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -18,12 +17,16 @@ def make_dump_path(location_config): # pragma: no cover
|
||||||
|
|
||||||
def dump_databases(databases, log_prefix, location_config, dry_run):
|
def dump_databases(databases, log_prefix, location_config, dry_run):
|
||||||
'''
|
'''
|
||||||
Dump the given PostgreSQL databases to disk. The databases are supplied as a sequence of dicts,
|
Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of
|
||||||
one dict describing each database as per the configuration schema. Use the given log prefix in
|
dicts, one dict describing each database as per the configuration schema. Use the given log
|
||||||
any log entries. Use the given location configuration dict to construct the destination path. If
|
prefix in any log entries. Use the given location configuration dict to construct the
|
||||||
this is a dry run, then don't actually dump anything.
|
destination path.
|
||||||
|
|
||||||
|
Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
|
||||||
|
pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
|
||||||
|
processes = []
|
||||||
|
|
||||||
logger.info('{}: Dumping PostgreSQL databases{}'.format(log_prefix, dry_run_label))
|
logger.info('{}: Dumping PostgreSQL databases{}'.format(log_prefix, dry_run_label))
|
||||||
|
|
||||||
|
@ -39,6 +42,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
|
||||||
'--no-password',
|
'--no-password',
|
||||||
'--clean',
|
'--clean',
|
||||||
'--if-exists',
|
'--if-exists',
|
||||||
|
'--no-sync',
|
||||||
)
|
)
|
||||||
+ ('--file', dump_filename)
|
+ ('--file', dump_filename)
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
|
@ -55,9 +59,16 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
|
||||||
log_prefix, name, dump_filename, dry_run_label
|
log_prefix, name, dump_filename, dry_run_label
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if not dry_run:
|
if dry_run:
|
||||||
os.makedirs(os.path.dirname(dump_filename), mode=0o700, exist_ok=True)
|
continue
|
||||||
execute_command(command, extra_environment=extra_environment)
|
|
||||||
|
dump.create_named_pipe_for_dump(dump_filename)
|
||||||
|
|
||||||
|
processes.append(
|
||||||
|
execute_command(command, extra_environment=extra_environment, run_to_completion=False)
|
||||||
|
)
|
||||||
|
|
||||||
|
return processes
|
||||||
|
|
||||||
|
|
||||||
def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover
|
def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover
|
||||||
|
@ -72,60 +83,61 @@ def remove_database_dumps(databases, log_prefix, location_config, dry_run): # p
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def make_database_dump_patterns(databases, log_prefix, location_config, names):
|
def make_database_dump_pattern(databases, log_prefix, location_config, name=None):
|
||||||
'''
|
'''
|
||||||
Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
|
Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
|
||||||
and a sequence of database names to match, return the corresponding glob patterns to match the
|
and a database name to match, return the corresponding glob patterns to match the database dump
|
||||||
database dumps in an archive. An empty sequence of names indicates that the patterns should
|
in an archive.
|
||||||
match all dumps.
|
|
||||||
'''
|
'''
|
||||||
return [
|
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
|
||||||
dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
|
|
||||||
for name in (names or ['*'])
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def restore_database_dumps(databases, log_prefix, location_config, dry_run):
|
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
|
||||||
'''
|
'''
|
||||||
Restore the given PostgreSQL databases from disk. The databases are supplied as a sequence of
|
Restore the given PostgreSQL database from an extract stream. The database is supplied as a
|
||||||
dicts, one dict describing each database as per the configuration schema. Use the given log
|
one-element sequence containing a dict describing the database, as per the configuration schema.
|
||||||
prefix in any log entries. Use the given location configuration dict to construct the
|
Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
|
||||||
destination path. If this is a dry run, then don't actually restore anything.
|
anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
|
||||||
|
output to consume.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
||||||
|
|
||||||
for database in databases:
|
if len(database_config) != 1:
|
||||||
dump_filename = dump.make_database_dump_filename(
|
raise ValueError('The database configuration value is invalid')
|
||||||
make_dump_path(location_config), database['name'], database.get('hostname')
|
|
||||||
)
|
|
||||||
all_databases = bool(database['name'] == 'all')
|
|
||||||
analyze_command = (
|
|
||||||
('psql', '--no-password', '--quiet')
|
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
|
||||||
+ (('--username', database['username']) if 'username' in database else ())
|
|
||||||
+ (('--dbname', database['name']) if not all_databases else ())
|
|
||||||
+ ('--command', 'ANALYZE')
|
|
||||||
)
|
|
||||||
restore_command = (
|
|
||||||
('psql' if all_databases else 'pg_restore', '--no-password')
|
|
||||||
+ (
|
|
||||||
('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
|
|
||||||
if not all_databases
|
|
||||||
else ()
|
|
||||||
)
|
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
|
||||||
+ (('--username', database['username']) if 'username' in database else ())
|
|
||||||
+ (('-f', dump_filename) if all_databases else (dump_filename,))
|
|
||||||
)
|
|
||||||
extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
|
|
||||||
|
|
||||||
logger.debug(
|
database = database_config[0]
|
||||||
'{}: Restoring PostgreSQL database {}{}'.format(
|
all_databases = bool(database['name'] == 'all')
|
||||||
log_prefix, database['name'], dry_run_label
|
analyze_command = (
|
||||||
)
|
('psql', '--no-password', '--quiet')
|
||||||
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
|
+ (('--username', database['username']) if 'username' in database else ())
|
||||||
|
+ (('--dbname', database['name']) if not all_databases else ())
|
||||||
|
+ ('--command', 'ANALYZE')
|
||||||
|
)
|
||||||
|
restore_command = (
|
||||||
|
('psql' if all_databases else 'pg_restore', '--no-password')
|
||||||
|
+ (
|
||||||
|
('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
|
||||||
|
if not all_databases
|
||||||
|
else ()
|
||||||
)
|
)
|
||||||
if not dry_run:
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
execute_command(restore_command, extra_environment=extra_environment)
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
execute_command(analyze_command, extra_environment=extra_environment)
|
+ (('--username', database['username']) if 'username' in database else ())
|
||||||
|
)
|
||||||
|
extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
'{}: Restoring PostgreSQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
|
||||||
|
)
|
||||||
|
if dry_run:
|
||||||
|
return
|
||||||
|
|
||||||
|
execute_command_with_processes(
|
||||||
|
restore_command,
|
||||||
|
[extract_process],
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment=extra_environment,
|
||||||
|
)
|
||||||
|
execute_command(analyze_command, extra_environment=extra_environment)
|
||||||
|
|
|
@ -22,13 +22,13 @@ hooks:
|
||||||
- name: posts
|
- name: posts
|
||||||
```
|
```
|
||||||
|
|
||||||
Prior to each backup, borgmatic dumps each configured database to a file
|
As part of each backup, borgmatic streams a database dump for each configured
|
||||||
and includes it in the backup. After the backup completes, borgmatic removes
|
database directly to Borg, so it's included in the backup without consuming
|
||||||
the database dump files to recover disk space.
|
additional disk space.
|
||||||
|
|
||||||
borgmatic creates these temporary dump files in `~/.borgmatic` by default. To
|
To support this, borgmatic creates temporary named pipes in `~/.borgmatic` by
|
||||||
customize this path, set the `borgmatic_source_directory` option in the
|
default. To customize this path, set the `borgmatic_source_directory` option
|
||||||
`location` section of borgmatic's configuration.
|
in the `location` section of borgmatic's configuration.
|
||||||
|
|
||||||
Here's a more involved example that connects to remote databases:
|
Here's a more involved example that connects to remote databases:
|
||||||
|
|
||||||
|
|
|
@ -14,20 +14,12 @@ def test_log_output_logs_each_line_separately():
|
||||||
|
|
||||||
hi_process = subprocess.Popen(['echo', 'hi'], stdout=subprocess.PIPE)
|
hi_process = subprocess.Popen(['echo', 'hi'], stdout=subprocess.PIPE)
|
||||||
module.log_output(
|
module.log_output(
|
||||||
['echo', 'hi'],
|
hi_process, hi_process.stdout, output_log_level=logging.INFO, error_on_warnings=False
|
||||||
hi_process,
|
|
||||||
hi_process.stdout,
|
|
||||||
output_log_level=logging.INFO,
|
|
||||||
error_on_warnings=False,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
there_process = subprocess.Popen(['echo', 'there'], stdout=subprocess.PIPE)
|
there_process = subprocess.Popen(['echo', 'there'], stdout=subprocess.PIPE)
|
||||||
module.log_output(
|
module.log_output(
|
||||||
['echo', 'there'],
|
there_process, there_process.stdout, output_log_level=logging.INFO, error_on_warnings=False
|
||||||
there_process,
|
|
||||||
there_process.stdout,
|
|
||||||
output_log_level=logging.INFO,
|
|
||||||
error_on_warnings=False,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -39,11 +31,7 @@ def test_log_output_includes_error_output_in_exception():
|
||||||
|
|
||||||
with pytest.raises(subprocess.CalledProcessError) as error:
|
with pytest.raises(subprocess.CalledProcessError) as error:
|
||||||
module.log_output(
|
module.log_output(
|
||||||
['grep'],
|
process, process.stdout, output_log_level=logging.INFO, error_on_warnings=False
|
||||||
process,
|
|
||||||
process.stdout,
|
|
||||||
output_log_level=logging.INFO,
|
|
||||||
error_on_warnings=False,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
assert error.value.returncode == 2
|
assert error.value.returncode == 2
|
||||||
|
@ -59,11 +47,7 @@ def test_log_output_truncates_long_error_output():
|
||||||
|
|
||||||
with pytest.raises(subprocess.CalledProcessError) as error:
|
with pytest.raises(subprocess.CalledProcessError) as error:
|
||||||
module.log_output(
|
module.log_output(
|
||||||
['grep'],
|
process, process.stdout, output_log_level=logging.INFO, error_on_warnings=False
|
||||||
process,
|
|
||||||
process.stdout,
|
|
||||||
output_log_level=logging.INFO,
|
|
||||||
error_on_warnings=False,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
assert error.value.returncode == 2
|
assert error.value.returncode == 2
|
||||||
|
@ -76,5 +60,5 @@ def test_log_output_with_no_output_logs_nothing():
|
||||||
|
|
||||||
process = subprocess.Popen(['true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
process = subprocess.Popen(['true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
module.log_output(
|
module.log_output(
|
||||||
['true'], process, process.stdout, output_log_level=logging.INFO, error_on_warnings=False
|
process, process.stdout, output_log_level=logging.INFO, error_on_warnings=False
|
||||||
)
|
)
|
||||||
|
|
|
@ -34,29 +34,6 @@ def test_make_database_dump_filename_with_invalid_name_raises():
|
||||||
module.make_database_dump_filename('databases', 'invalid/name')
|
module.make_database_dump_filename('databases', 'invalid/name')
|
||||||
|
|
||||||
|
|
||||||
def test_flatten_dump_patterns_produces_list_of_all_patterns():
|
|
||||||
dump_patterns = {'postgresql_databases': ['*/glob', 'glob/*'], 'mysql_databases': ['*/*/*']}
|
|
||||||
expected_patterns = sorted(
|
|
||||||
dump_patterns['postgresql_databases'] + dump_patterns['mysql_databases']
|
|
||||||
)
|
|
||||||
|
|
||||||
assert sorted(module.flatten_dump_patterns(dump_patterns, ('bob',))) == expected_patterns
|
|
||||||
|
|
||||||
|
|
||||||
def test_flatten_dump_patterns_with_no_patterns_errors():
|
|
||||||
dump_patterns = {'postgresql_databases': [], 'mysql_databases': []}
|
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
assert module.flatten_dump_patterns(dump_patterns, ('bob',))
|
|
||||||
|
|
||||||
|
|
||||||
def test_flatten_dump_patterns_with_no_hooks_errors():
|
|
||||||
dump_patterns = {}
|
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
assert module.flatten_dump_patterns(dump_patterns, ('bob',))
|
|
||||||
|
|
||||||
|
|
||||||
def test_remove_database_dumps_removes_dump_for_each_database():
|
def test_remove_database_dumps_removes_dump_for_each_database():
|
||||||
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
||||||
flexmock(module).should_receive('make_database_dump_filename').with_args(
|
flexmock(module).should_receive('make_database_dump_filename').with_args(
|
||||||
|
@ -107,93 +84,3 @@ def test_remove_database_dumps_without_databases_does_not_raise():
|
||||||
|
|
||||||
def test_convert_glob_patterns_to_borg_patterns_removes_leading_slash():
|
def test_convert_glob_patterns_to_borg_patterns_removes_leading_slash():
|
||||||
assert module.convert_glob_patterns_to_borg_patterns(('/etc/foo/bar',)) == ['sh:etc/foo/bar']
|
assert module.convert_glob_patterns_to_borg_patterns(('/etc/foo/bar',)) == ['sh:etc/foo/bar']
|
||||||
|
|
||||||
|
|
||||||
def test_get_database_names_from_dumps_gets_names_from_filenames_matching_globs():
|
|
||||||
flexmock(module.glob).should_receive('glob').and_return(
|
|
||||||
('databases/localhost/foo',)
|
|
||||||
).and_return(('databases/localhost/bar',)).and_return(())
|
|
||||||
|
|
||||||
assert module.get_database_names_from_dumps(
|
|
||||||
('databases/*/foo', 'databases/*/bar', 'databases/*/baz')
|
|
||||||
) == ['foo', 'bar']
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_database_configurations_only_produces_named_databases():
|
|
||||||
databases = [
|
|
||||||
{'name': 'foo', 'hostname': 'example.org'},
|
|
||||||
{'name': 'bar', 'hostname': 'example.com'},
|
|
||||||
{'name': 'baz', 'hostname': 'example.org'},
|
|
||||||
]
|
|
||||||
|
|
||||||
assert list(module.get_database_configurations(databases, ('foo', 'baz'))) == [
|
|
||||||
{'name': 'foo', 'hostname': 'example.org'},
|
|
||||||
{'name': 'baz', 'hostname': 'example.org'},
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_database_configurations_matches_all_database():
|
|
||||||
databases = [
|
|
||||||
{'name': 'foo', 'hostname': 'example.org'},
|
|
||||||
{'name': 'all', 'hostname': 'example.com'},
|
|
||||||
]
|
|
||||||
|
|
||||||
assert list(module.get_database_configurations(databases, ('foo', 'bar', 'baz'))) == [
|
|
||||||
{'name': 'foo', 'hostname': 'example.org'},
|
|
||||||
{'name': 'bar', 'hostname': 'example.com'},
|
|
||||||
{'name': 'baz', 'hostname': 'example.com'},
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_per_hook_database_configurations_partitions_by_hook():
|
|
||||||
hooks = {'postgresql_databases': [flexmock()]}
|
|
||||||
names = ('foo', 'bar')
|
|
||||||
dump_patterns = flexmock()
|
|
||||||
expected_config = {'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]}
|
|
||||||
flexmock(module).should_receive('get_database_configurations').with_args(
|
|
||||||
hooks['postgresql_databases'], names
|
|
||||||
).and_return(expected_config['postgresql_databases'])
|
|
||||||
|
|
||||||
config = module.get_per_hook_database_configurations(hooks, names, dump_patterns)
|
|
||||||
|
|
||||||
assert config == expected_config
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_per_hook_database_configurations_defaults_to_detected_database_names():
|
|
||||||
hooks = {'postgresql_databases': [flexmock()]}
|
|
||||||
names = ()
|
|
||||||
detected_names = flexmock()
|
|
||||||
dump_patterns = {'postgresql_databases': [flexmock()]}
|
|
||||||
expected_config = {'postgresql_databases': [flexmock()]}
|
|
||||||
flexmock(module).should_receive('get_database_names_from_dumps').and_return(detected_names)
|
|
||||||
flexmock(module).should_receive('get_database_configurations').with_args(
|
|
||||||
hooks['postgresql_databases'], detected_names
|
|
||||||
).and_return(expected_config['postgresql_databases'])
|
|
||||||
|
|
||||||
config = module.get_per_hook_database_configurations(hooks, names, dump_patterns)
|
|
||||||
|
|
||||||
assert config == expected_config
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_per_hook_database_configurations_with_unknown_database_name_raises():
|
|
||||||
hooks = {'postgresql_databases': [flexmock()]}
|
|
||||||
names = ('foo', 'bar')
|
|
||||||
dump_patterns = flexmock()
|
|
||||||
flexmock(module).should_receive('get_database_configurations').with_args(
|
|
||||||
hooks['postgresql_databases'], names
|
|
||||||
).and_return([])
|
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
module.get_per_hook_database_configurations(hooks, names, dump_patterns)
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_per_hook_database_configurations_with_all_and_no_archive_dumps_raises():
|
|
||||||
hooks = {'postgresql_databases': [flexmock()]}
|
|
||||||
names = ('foo', 'all')
|
|
||||||
dump_patterns = flexmock()
|
|
||||||
flexmock(module).should_receive('get_database_configurations').with_args(
|
|
||||||
hooks['postgresql_databases'], names
|
|
||||||
).and_return([])
|
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
module.get_per_hook_database_configurations(hooks, names, dump_patterns)
|
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
import sys
|
import pytest
|
||||||
|
|
||||||
from flexmock import flexmock
|
from flexmock import flexmock
|
||||||
|
|
||||||
from borgmatic.hooks import mysql as module
|
from borgmatic.hooks import mysql as module
|
||||||
|
@ -36,7 +35,7 @@ def test_database_names_to_dump_queries_mysql_for_database_names():
|
||||||
|
|
||||||
def test_dump_databases_runs_mysqldump_for_each_database():
|
def test_dump_databases_runs_mysqldump_for_each_database():
|
||||||
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
||||||
output_file = flexmock()
|
processes = [flexmock(), flexmock()]
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/localhost/foo'
|
'databases/localhost/foo'
|
||||||
|
@ -44,17 +43,24 @@ def test_dump_databases_runs_mysqldump_for_each_database():
|
||||||
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
|
||||||
('bar',)
|
('bar',)
|
||||||
)
|
)
|
||||||
flexmock(module.os).should_receive('makedirs')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
|
|
||||||
|
|
||||||
for name in ('foo', 'bar'):
|
for name, process in zip(('foo', 'bar'), processes):
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
('mysqldump', '--add-drop-database', '--databases', name),
|
(
|
||||||
output_file=output_file,
|
'mysqldump',
|
||||||
|
'--add-drop-database',
|
||||||
|
'--databases',
|
||||||
|
name,
|
||||||
|
'>',
|
||||||
|
'databases/localhost/{}'.format(name),
|
||||||
|
),
|
||||||
|
shell=True,
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
).once()
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
|
||||||
|
|
||||||
|
|
||||||
def test_dump_databases_with_dry_run_skips_mysqldump():
|
def test_dump_databases_with_dry_run_skips_mysqldump():
|
||||||
|
@ -66,7 +72,7 @@ def test_dump_databases_with_dry_run_skips_mysqldump():
|
||||||
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
|
||||||
('bar',)
|
('bar',)
|
||||||
)
|
)
|
||||||
flexmock(module.os).should_receive('makedirs').never()
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
|
||||||
flexmock(module).should_receive('execute_command').never()
|
flexmock(module).should_receive('execute_command').never()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=True)
|
module.dump_databases(databases, 'test.yaml', {}, dry_run=True)
|
||||||
|
@ -74,14 +80,13 @@ def test_dump_databases_with_dry_run_skips_mysqldump():
|
||||||
|
|
||||||
def test_dump_databases_runs_mysqldump_with_hostname_and_port():
|
def test_dump_databases_runs_mysqldump_with_hostname_and_port():
|
||||||
databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
||||||
output_file = flexmock()
|
process = flexmock()
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/database.example.org/foo'
|
'databases/database.example.org/foo'
|
||||||
)
|
)
|
||||||
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
||||||
flexmock(module.os).should_receive('makedirs')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
|
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
(
|
(
|
||||||
|
@ -95,128 +100,135 @@ def test_dump_databases_runs_mysqldump_with_hostname_and_port():
|
||||||
'tcp',
|
'tcp',
|
||||||
'--databases',
|
'--databases',
|
||||||
'foo',
|
'foo',
|
||||||
|
'>',
|
||||||
|
'databases/database.example.org/foo',
|
||||||
),
|
),
|
||||||
output_file=output_file,
|
shell=True,
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
).once()
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
|
||||||
|
|
||||||
|
|
||||||
def test_dump_databases_runs_mysqldump_with_username_and_password():
|
def test_dump_databases_runs_mysqldump_with_username_and_password():
|
||||||
databases = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
|
databases = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
|
||||||
output_file = flexmock()
|
process = flexmock()
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/localhost/foo'
|
'databases/localhost/foo'
|
||||||
)
|
)
|
||||||
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
||||||
flexmock(module.os).should_receive('makedirs')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
|
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
('mysqldump', '--add-drop-database', '--user', 'root', '--databases', 'foo'),
|
(
|
||||||
output_file=output_file,
|
'mysqldump',
|
||||||
|
'--add-drop-database',
|
||||||
|
'--user',
|
||||||
|
'root',
|
||||||
|
'--databases',
|
||||||
|
'foo',
|
||||||
|
'>',
|
||||||
|
'databases/localhost/foo',
|
||||||
|
),
|
||||||
|
shell=True,
|
||||||
extra_environment={'MYSQL_PWD': 'trustsome1'},
|
extra_environment={'MYSQL_PWD': 'trustsome1'},
|
||||||
).once()
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
|
||||||
|
|
||||||
|
|
||||||
def test_dump_databases_runs_mysqldump_with_options():
|
def test_dump_databases_runs_mysqldump_with_options():
|
||||||
databases = [{'name': 'foo', 'options': '--stuff=such'}]
|
databases = [{'name': 'foo', 'options': '--stuff=such'}]
|
||||||
output_file = flexmock()
|
process = flexmock()
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/localhost/foo'
|
'databases/localhost/foo'
|
||||||
)
|
)
|
||||||
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
||||||
flexmock(module.os).should_receive('makedirs')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
|
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
('mysqldump', '--add-drop-database', '--stuff=such', '--databases', 'foo'),
|
(
|
||||||
output_file=output_file,
|
'mysqldump',
|
||||||
|
'--add-drop-database',
|
||||||
|
'--stuff=such',
|
||||||
|
'--databases',
|
||||||
|
'foo',
|
||||||
|
'>',
|
||||||
|
'databases/localhost/foo',
|
||||||
|
),
|
||||||
|
shell=True,
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
).once()
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
|
||||||
|
|
||||||
|
|
||||||
def test_dump_databases_runs_mysqldump_for_all_databases():
|
def test_dump_databases_runs_mysqldump_for_all_databases():
|
||||||
databases = [{'name': 'all'}]
|
databases = [{'name': 'all'}]
|
||||||
output_file = flexmock()
|
process = flexmock()
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/localhost/all'
|
'databases/localhost/all'
|
||||||
)
|
)
|
||||||
flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar'))
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar'))
|
||||||
flexmock(module.os).should_receive('makedirs')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
|
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
('mysqldump', '--add-drop-database', '--databases', 'foo', 'bar'),
|
(
|
||||||
output_file=output_file,
|
'mysqldump',
|
||||||
|
'--add-drop-database',
|
||||||
|
'--databases',
|
||||||
|
'foo',
|
||||||
|
'bar',
|
||||||
|
'>',
|
||||||
|
'databases/localhost/all',
|
||||||
|
),
|
||||||
|
shell=True,
|
||||||
|
extra_environment=None,
|
||||||
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_runs_mysql_to_restore():
|
||||||
|
database_config = [{'name': 'foo'}]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
('mysql', '--batch'),
|
||||||
|
processes=[extract_process],
|
||||||
|
input_file=extract_process.stdout,
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||||
|
|
||||||
def test_make_database_dump_patterns_converts_names_to_glob_paths():
|
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
|
||||||
'databases/*/foo'
|
|
||||||
).and_return('databases/*/bar')
|
|
||||||
|
|
||||||
assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ('foo', 'bar')) == [
|
|
||||||
'databases/*/foo',
|
|
||||||
'databases/*/bar',
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_make_database_dump_patterns_treats_empty_names_as_matching_all_databases():
|
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('/dump/path')
|
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').with_args(
|
|
||||||
'/dump/path', '*', '*'
|
|
||||||
).and_return('databases/*/*')
|
|
||||||
|
|
||||||
assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ()) == ['databases/*/*']
|
|
||||||
|
|
||||||
|
|
||||||
def test_restore_database_dumps_restores_each_database():
|
|
||||||
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
|
||||||
'databases/localhost/foo'
|
|
||||||
).and_return('databases/localhost/bar')
|
|
||||||
|
|
||||||
for name in ('foo', 'bar'):
|
|
||||||
dump_filename = 'databases/localhost/{}'.format(name)
|
|
||||||
input_file = flexmock()
|
|
||||||
flexmock(sys.modules['builtins']).should_receive('open').with_args(
|
|
||||||
dump_filename
|
|
||||||
).and_return(input_file)
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
|
||||||
('mysql', '--batch'), input_file=input_file, extra_environment=None
|
|
||||||
).once()
|
|
||||||
|
|
||||||
module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
|
|
||||||
|
|
||||||
|
|
||||||
def test_restore_database_dumps_runs_mysql_with_hostname_and_port():
|
|
||||||
databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
|
||||||
'databases/localhost/foo'
|
|
||||||
)
|
|
||||||
dump_filename = 'databases/localhost/foo'
|
|
||||||
input_file = flexmock()
|
|
||||||
flexmock(sys.modules['builtins']).should_receive('open').with_args(dump_filename).and_return(
|
|
||||||
input_file
|
|
||||||
)
|
)
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
|
||||||
|
def test_restore_database_dump_errors_on_multiple_database_config():
|
||||||
|
database_config = [{'name': 'foo'}, {'name': 'bar'}]
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').never()
|
||||||
|
flexmock(module).should_receive('execute_command').never()
|
||||||
|
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_runs_mysql_with_hostname_and_port():
|
||||||
|
database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
(
|
(
|
||||||
'mysql',
|
'mysql',
|
||||||
'--batch',
|
'--batch',
|
||||||
|
@ -227,29 +239,27 @@ def test_restore_database_dumps_runs_mysql_with_hostname_and_port():
|
||||||
'--protocol',
|
'--protocol',
|
||||||
'tcp',
|
'tcp',
|
||||||
),
|
),
|
||||||
input_file=input_file,
|
processes=[extract_process],
|
||||||
|
input_file=extract_process.stdout,
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||||
|
|
||||||
def test_restore_database_dumps_runs_mysql_with_username_and_password():
|
|
||||||
databases = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
|
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
|
||||||
'databases/localhost/foo'
|
|
||||||
)
|
|
||||||
dump_filename = 'databases/localhost/foo'
|
|
||||||
input_file = flexmock()
|
|
||||||
flexmock(sys.modules['builtins']).should_receive('open').with_args(dump_filename).and_return(
|
|
||||||
input_file
|
|
||||||
)
|
)
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
|
||||||
|
def test_restore_database_dump_runs_mysql_with_username_and_password():
|
||||||
|
database_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
('mysql', '--batch', '--user', 'root'),
|
('mysql', '--batch', '--user', 'root'),
|
||||||
input_file=input_file,
|
processes=[extract_process],
|
||||||
|
input_file=extract_process.stdout,
|
||||||
extra_environment={'MYSQL_PWD': 'trustsome1'},
|
extra_environment={'MYSQL_PWD': 'trustsome1'},
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||||
|
)
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import pytest
|
||||||
from flexmock import flexmock
|
from flexmock import flexmock
|
||||||
|
|
||||||
from borgmatic.hooks import postgresql as module
|
from borgmatic.hooks import postgresql as module
|
||||||
|
@ -5,19 +6,21 @@ from borgmatic.hooks import postgresql as module
|
||||||
|
|
||||||
def test_dump_databases_runs_pg_dump_for_each_database():
|
def test_dump_databases_runs_pg_dump_for_each_database():
|
||||||
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
||||||
|
processes = [flexmock(), flexmock()]
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/localhost/foo'
|
'databases/localhost/foo'
|
||||||
).and_return('databases/localhost/bar')
|
).and_return('databases/localhost/bar')
|
||||||
flexmock(module.os).should_receive('makedirs')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
for name in ('foo', 'bar'):
|
for name, process in zip(('foo', 'bar'), processes):
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
(
|
(
|
||||||
'pg_dump',
|
'pg_dump',
|
||||||
'--no-password',
|
'--no-password',
|
||||||
'--clean',
|
'--clean',
|
||||||
'--if-exists',
|
'--if-exists',
|
||||||
|
'--no-sync',
|
||||||
'--file',
|
'--file',
|
||||||
'databases/localhost/{}'.format(name),
|
'databases/localhost/{}'.format(name),
|
||||||
'--format',
|
'--format',
|
||||||
|
@ -25,9 +28,10 @@ def test_dump_databases_runs_pg_dump_for_each_database():
|
||||||
name,
|
name,
|
||||||
),
|
),
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
).once()
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
|
||||||
|
|
||||||
|
|
||||||
def test_dump_databases_with_dry_run_skips_pg_dump():
|
def test_dump_databases_with_dry_run_skips_pg_dump():
|
||||||
|
@ -36,19 +40,20 @@ def test_dump_databases_with_dry_run_skips_pg_dump():
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/localhost/foo'
|
'databases/localhost/foo'
|
||||||
).and_return('databases/localhost/bar')
|
).and_return('databases/localhost/bar')
|
||||||
flexmock(module.os).should_receive('makedirs').never()
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
|
||||||
flexmock(module).should_receive('execute_command').never()
|
flexmock(module).should_receive('execute_command').never()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=True)
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == []
|
||||||
|
|
||||||
|
|
||||||
def test_dump_databases_runs_pg_dump_with_hostname_and_port():
|
def test_dump_databases_runs_pg_dump_with_hostname_and_port():
|
||||||
databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
||||||
|
process = flexmock()
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/database.example.org/foo'
|
'databases/database.example.org/foo'
|
||||||
)
|
)
|
||||||
flexmock(module.os).should_receive('makedirs')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
(
|
(
|
||||||
|
@ -56,6 +61,7 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port():
|
||||||
'--no-password',
|
'--no-password',
|
||||||
'--clean',
|
'--clean',
|
||||||
'--if-exists',
|
'--if-exists',
|
||||||
|
'--no-sync',
|
||||||
'--file',
|
'--file',
|
||||||
'databases/database.example.org/foo',
|
'databases/database.example.org/foo',
|
||||||
'--host',
|
'--host',
|
||||||
|
@ -67,18 +73,20 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port():
|
||||||
'foo',
|
'foo',
|
||||||
),
|
),
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
).once()
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
|
||||||
|
|
||||||
|
|
||||||
def test_dump_databases_runs_pg_dump_with_username_and_password():
|
def test_dump_databases_runs_pg_dump_with_username_and_password():
|
||||||
databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
|
databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
|
||||||
|
process = flexmock()
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/localhost/foo'
|
'databases/localhost/foo'
|
||||||
)
|
)
|
||||||
flexmock(module.os).should_receive('makedirs')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
(
|
(
|
||||||
|
@ -86,6 +94,7 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
|
||||||
'--no-password',
|
'--no-password',
|
||||||
'--clean',
|
'--clean',
|
||||||
'--if-exists',
|
'--if-exists',
|
||||||
|
'--no-sync',
|
||||||
'--file',
|
'--file',
|
||||||
'databases/localhost/foo',
|
'databases/localhost/foo',
|
||||||
'--username',
|
'--username',
|
||||||
|
@ -95,18 +104,20 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
|
||||||
'foo',
|
'foo',
|
||||||
),
|
),
|
||||||
extra_environment={'PGPASSWORD': 'trustsome1'},
|
extra_environment={'PGPASSWORD': 'trustsome1'},
|
||||||
).once()
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
|
||||||
|
|
||||||
|
|
||||||
def test_dump_databases_runs_pg_dump_with_format():
|
def test_dump_databases_runs_pg_dump_with_format():
|
||||||
databases = [{'name': 'foo', 'format': 'tar'}]
|
databases = [{'name': 'foo', 'format': 'tar'}]
|
||||||
|
process = flexmock()
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/localhost/foo'
|
'databases/localhost/foo'
|
||||||
)
|
)
|
||||||
flexmock(module.os).should_receive('makedirs')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
(
|
(
|
||||||
|
@ -114,6 +125,7 @@ def test_dump_databases_runs_pg_dump_with_format():
|
||||||
'--no-password',
|
'--no-password',
|
||||||
'--clean',
|
'--clean',
|
||||||
'--if-exists',
|
'--if-exists',
|
||||||
|
'--no-sync',
|
||||||
'--file',
|
'--file',
|
||||||
'databases/localhost/foo',
|
'databases/localhost/foo',
|
||||||
'--format',
|
'--format',
|
||||||
|
@ -121,18 +133,20 @@ def test_dump_databases_runs_pg_dump_with_format():
|
||||||
'foo',
|
'foo',
|
||||||
),
|
),
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
).once()
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
|
||||||
|
|
||||||
|
|
||||||
def test_dump_databases_runs_pg_dump_with_options():
|
def test_dump_databases_runs_pg_dump_with_options():
|
||||||
databases = [{'name': 'foo', 'options': '--stuff=such'}]
|
databases = [{'name': 'foo', 'options': '--stuff=such'}]
|
||||||
|
process = flexmock()
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/localhost/foo'
|
'databases/localhost/foo'
|
||||||
)
|
)
|
||||||
flexmock(module.os).should_receive('makedirs')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
(
|
(
|
||||||
|
@ -140,6 +154,7 @@ def test_dump_databases_runs_pg_dump_with_options():
|
||||||
'--no-password',
|
'--no-password',
|
||||||
'--clean',
|
'--clean',
|
||||||
'--if-exists',
|
'--if-exists',
|
||||||
|
'--no-sync',
|
||||||
'--file',
|
'--file',
|
||||||
'databases/localhost/foo',
|
'databases/localhost/foo',
|
||||||
'--format',
|
'--format',
|
||||||
|
@ -148,18 +163,20 @@ def test_dump_databases_runs_pg_dump_with_options():
|
||||||
'foo',
|
'foo',
|
||||||
),
|
),
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
).once()
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
|
||||||
|
|
||||||
|
|
||||||
def test_dump_databases_runs_pg_dumpall_for_all_databases():
|
def test_dump_databases_runs_pg_dumpall_for_all_databases():
|
||||||
databases = [{'name': 'all'}]
|
databases = [{'name': 'all'}]
|
||||||
|
process = flexmock()
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
||||||
'databases/localhost/all'
|
'databases/localhost/all'
|
||||||
)
|
)
|
||||||
flexmock(module.os).should_receive('makedirs')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
(
|
(
|
||||||
|
@ -167,73 +184,62 @@ def test_dump_databases_runs_pg_dumpall_for_all_databases():
|
||||||
'--no-password',
|
'--no-password',
|
||||||
'--clean',
|
'--clean',
|
||||||
'--if-exists',
|
'--if-exists',
|
||||||
|
'--no-sync',
|
||||||
'--file',
|
'--file',
|
||||||
'databases/localhost/all',
|
'databases/localhost/all',
|
||||||
),
|
),
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_runs_pg_restore():
|
||||||
|
database_config = [{'name': 'foo'}]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
(
|
||||||
|
'pg_restore',
|
||||||
|
'--no-password',
|
||||||
|
'--if-exists',
|
||||||
|
'--exit-on-error',
|
||||||
|
'--clean',
|
||||||
|
'--dbname',
|
||||||
|
'foo',
|
||||||
|
),
|
||||||
|
processes=[extract_process],
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment=None,
|
||||||
|
).once()
|
||||||
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
|
('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
|
||||||
|
extra_environment=None,
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||||
|
|
||||||
def test_make_database_dump_patterns_converts_names_to_glob_paths():
|
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
|
||||||
'databases/*/foo'
|
|
||||||
).and_return('databases/*/bar')
|
|
||||||
|
|
||||||
assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ('foo', 'bar')) == [
|
|
||||||
'databases/*/foo',
|
|
||||||
'databases/*/bar',
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_make_database_dump_patterns_treats_empty_names_as_matching_all_databases():
|
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('/dump/path')
|
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').with_args(
|
|
||||||
'/dump/path', '*', '*'
|
|
||||||
).and_return('databases/*/*')
|
|
||||||
|
|
||||||
assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ()) == ['databases/*/*']
|
|
||||||
|
|
||||||
|
|
||||||
def test_restore_database_dumps_restores_each_database():
|
|
||||||
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
|
||||||
'databases/localhost/foo'
|
|
||||||
).and_return('databases/localhost/bar')
|
|
||||||
|
|
||||||
for name in ('foo', 'bar'):
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
|
||||||
(
|
|
||||||
'pg_restore',
|
|
||||||
'--no-password',
|
|
||||||
'--if-exists',
|
|
||||||
'--exit-on-error',
|
|
||||||
'--clean',
|
|
||||||
'--dbname',
|
|
||||||
name,
|
|
||||||
'databases/localhost/{}'.format(name),
|
|
||||||
),
|
|
||||||
extra_environment=None,
|
|
||||||
).once()
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
|
||||||
('psql', '--no-password', '--quiet', '--dbname', name, '--command', 'ANALYZE'),
|
|
||||||
extra_environment=None,
|
|
||||||
).once()
|
|
||||||
|
|
||||||
module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
|
|
||||||
|
|
||||||
|
|
||||||
def test_restore_database_dumps_runs_pg_restore_with_hostname_and_port():
|
|
||||||
databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
|
||||||
'databases/localhost/foo'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
|
||||||
|
def test_restore_database_dump_errors_on_multiple_database_config():
|
||||||
|
database_config = [{'name': 'foo'}, {'name': 'bar'}]
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').never()
|
||||||
|
flexmock(module).should_receive('execute_command').never()
|
||||||
|
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
|
||||||
|
database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
(
|
(
|
||||||
'pg_restore',
|
'pg_restore',
|
||||||
'--no-password',
|
'--no-password',
|
||||||
|
@ -246,8 +252,9 @@ def test_restore_database_dumps_runs_pg_restore_with_hostname_and_port():
|
||||||
'database.example.org',
|
'database.example.org',
|
||||||
'--port',
|
'--port',
|
||||||
'5433',
|
'5433',
|
||||||
'databases/localhost/foo',
|
|
||||||
),
|
),
|
||||||
|
processes=[extract_process],
|
||||||
|
input_file=extract_process.stdout,
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
).once()
|
).once()
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
|
@ -267,17 +274,16 @@ def test_restore_database_dumps_runs_pg_restore_with_hostname_and_port():
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||||
|
|
||||||
def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
|
|
||||||
databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
|
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
|
||||||
'databases/localhost/foo'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
|
||||||
|
def test_restore_database_dump_runs_pg_restore_with_username_and_password():
|
||||||
|
database_config = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
(
|
(
|
||||||
'pg_restore',
|
'pg_restore',
|
||||||
'--no-password',
|
'--no-password',
|
||||||
|
@ -288,8 +294,9 @@ def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
|
||||||
'foo',
|
'foo',
|
||||||
'--username',
|
'--username',
|
||||||
'postgres',
|
'postgres',
|
||||||
'databases/localhost/foo',
|
|
||||||
),
|
),
|
||||||
|
processes=[extract_process],
|
||||||
|
input_file=extract_process.stdout,
|
||||||
extra_environment={'PGPASSWORD': 'trustsome1'},
|
extra_environment={'PGPASSWORD': 'trustsome1'},
|
||||||
).once()
|
).once()
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
|
@ -307,21 +314,25 @@ def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
|
||||||
extra_environment={'PGPASSWORD': 'trustsome1'},
|
extra_environment={'PGPASSWORD': 'trustsome1'},
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||||
|
|
||||||
def test_restore_database_dumps_runs_psql_for_all_database_dump():
|
|
||||||
databases = [{'name': 'all'}]
|
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
||||||
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
|
|
||||||
'databases/localhost/all'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
|
||||||
('psql', '--no-password', '-f', 'databases/localhost/all'), extra_environment=None
|
def test_restore_database_dump_runs_psql_for_all_database_dump():
|
||||||
|
database_config = [{'name': 'all'}]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
('psql', '--no-password'),
|
||||||
|
processes=[extract_process],
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment=None,
|
||||||
).once()
|
).once()
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), extra_environment=None
|
('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), extra_environment=None
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||||
|
)
|
||||||
|
|
|
@ -87,7 +87,7 @@ def test_execute_command_calls_full_command_with_shell():
|
||||||
full_command = ['foo', 'bar']
|
full_command = ['foo', 'bar']
|
||||||
flexmock(module.os, environ={'a': 'b'})
|
flexmock(module.os, environ={'a': 'b'})
|
||||||
flexmock(module.subprocess).should_receive('Popen').with_args(
|
flexmock(module.subprocess).should_receive('Popen').with_args(
|
||||||
full_command,
|
' '.join(full_command),
|
||||||
stdin=None,
|
stdin=None,
|
||||||
stdout=module.subprocess.PIPE,
|
stdout=module.subprocess.PIPE,
|
||||||
stderr=module.subprocess.STDOUT,
|
stderr=module.subprocess.STDOUT,
|
||||||
|
@ -140,6 +140,24 @@ def test_execute_command_calls_full_command_with_working_directory():
|
||||||
assert output is None
|
assert output is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_execute_command_without_run_to_completion_returns_process():
|
||||||
|
full_command = ['foo', 'bar']
|
||||||
|
process = flexmock()
|
||||||
|
flexmock(module.os, environ={'a': 'b'})
|
||||||
|
flexmock(module.subprocess).should_receive('Popen').with_args(
|
||||||
|
full_command,
|
||||||
|
stdin=None,
|
||||||
|
stdout=module.subprocess.PIPE,
|
||||||
|
stderr=module.subprocess.STDOUT,
|
||||||
|
shell=False,
|
||||||
|
env=None,
|
||||||
|
cwd=None,
|
||||||
|
).and_return(process).once()
|
||||||
|
flexmock(module).should_receive('log_output')
|
||||||
|
|
||||||
|
assert module.execute_command(full_command, run_to_completion=False) == process
|
||||||
|
|
||||||
|
|
||||||
def test_execute_command_captures_output():
|
def test_execute_command_captures_output():
|
||||||
full_command = ['foo', 'bar']
|
full_command = ['foo', 'bar']
|
||||||
expected_output = '[]'
|
expected_output = '[]'
|
||||||
|
|
Loading…
Reference in a new issue