Merge branch 'main' of github.com:borgmatic-collective/borgmatic

This commit is contained in:
Dan Helfman 2023-06-22 12:29:32 -07:00
commit 9e0df595c8
14 changed files with 1222 additions and 65 deletions

View file

@ -68,9 +68,11 @@ def restore_single_database(
archive_name, archive_name,
hook_name, hook_name,
database, database,
connection_params,
): # pragma: no cover ): # pragma: no cover
''' '''
Given (among other things) an archive name, a database hook name, and a configured database Given (among other things) an archive name, a database hook name, the hostname,
port, username and password as connection params, and a configured database
configuration dict, restore that database from the archive. configuration dict, restore that database from the archive.
''' '''
logger.info( logger.info(
@ -113,6 +115,7 @@ def restore_single_database(
location, location,
global_arguments.dry_run, global_arguments.dry_run,
extract_process, extract_process,
connection_params,
) )
@ -301,6 +304,13 @@ def run_restore(
restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names) restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names)
found_names = set() found_names = set()
remaining_restore_names = {} remaining_restore_names = {}
connection_params = {
'hostname': restore_arguments.hostname,
'port': restore_arguments.port,
'username': restore_arguments.username,
'password': restore_arguments.password,
'restore_path': restore_arguments.restore_path,
}
for hook_name, database_names in restore_names.items(): for hook_name, database_names in restore_names.items():
for database_name in database_names: for database_name in database_names:
@ -327,6 +337,7 @@ def run_restore(
archive_name, archive_name,
found_hook_name or hook_name, found_hook_name or hook_name,
dict(found_database, **{'schemas': restore_arguments.schemas}), dict(found_database, **{'schemas': restore_arguments.schemas}),
connection_params,
) )
# For any database that weren't found via exact matches in the hooks configuration, try to # For any database that weren't found via exact matches in the hooks configuration, try to
@ -356,6 +367,7 @@ def run_restore(
archive_name, archive_name,
found_hook_name or hook_name, found_hook_name or hook_name,
dict(database, **{'schemas': restore_arguments.schemas}), dict(database, **{'schemas': restore_arguments.schemas}),
connection_params,
) )
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(

View file

@ -931,6 +931,26 @@ def make_parsers():
dest='schemas', dest='schemas',
help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases', help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases',
) )
restore_group.add_argument(
'--hostname',
help='Database hostname to restore to. Defaults to the "restore_hostname" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--port',
help='Port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--username',
help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--password',
help='Password with which to connect to the restore database. Defaults to the "restore_password" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--restore-path',
help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
)
restore_group.add_argument( restore_group.add_argument(
'-h', '--help', action='help', help='Show this help message and exit' '-h', '--help', action='help', help='Show this help message and exit'
) )

View file

@ -763,10 +763,21 @@ properties:
Database hostname to connect to. Defaults to Database hostname to connect to. Defaults to
connecting via local Unix socket. connecting via local Unix socket.
example: database.example.org example: database.example.org
restore_hostname:
type: string
description: |
Database hostname to restore to. Defaults to
the "hostname" option.
example: database.example.org
port: port:
type: integer type: integer
description: Port to connect to. Defaults to 5432. description: Port to connect to. Defaults to 5432.
example: 5433 example: 5433
restore_port:
type: integer
description: Port to restore to. Defaults to the
"port" option.
example: 5433
username: username:
type: string type: string
description: | description: |
@ -775,6 +786,12 @@ properties:
You probably want to specify the "postgres" You probably want to specify the "postgres"
superuser here when the database name is "all". superuser here when the database name is "all".
example: dbuser example: dbuser
restore_username:
type: string
description: |
Username with which to restore the database.
Defaults to the "username" option.
example: dbuser
password: password:
type: string type: string
description: | description: |
@ -784,6 +801,24 @@ properties:
without a password or you create a ~/.pgpass without a password or you create a ~/.pgpass
file. file.
example: trustsome1 example: trustsome1
restore_password:
type: string
description: |
Password with which to connect to the restore
database. Defaults to the "password" option.
example: trustsome1
no_owner:
type: boolean
description: |
Do not output commands to set ownership of
objects to match the original database. By
default, pg_dump and pg_restore issue ALTER
OWNER or SET SESSION AUTHORIZATION statements
to set ownership of created schema elements.
These statements will fail unless the initial
connection to the database is made by a
superuser.
example: true
format: format:
type: string type: string
enum: ['plain', 'custom', 'directory', 'tar'] enum: ['plain', 'custom', 'directory', 'tar']
@ -919,16 +954,33 @@ properties:
Database hostname to connect to. Defaults to Database hostname to connect to. Defaults to
connecting via local Unix socket. connecting via local Unix socket.
example: database.example.org example: database.example.org
restore_hostname:
type: string
description: |
Database hostname to restore to. Defaults to
the "hostname" option.
example: database.example.org
port: port:
type: integer type: integer
description: Port to connect to. Defaults to 3306. description: Port to connect to. Defaults to 3306.
example: 3307 example: 3307
restore_port:
type: integer
description: Port to restore to. Defaults to the
"port" option.
example: 5433
username: username:
type: string type: string
description: | description: |
Username with which to connect to the database. Username with which to connect to the database.
Defaults to the username of the current user. Defaults to the username of the current user.
example: dbuser example: dbuser
restore_username:
type: string
description: |
Username with which to restore the database.
Defaults to the "username" option.
example: dbuser
password: password:
type: string type: string
description: | description: |
@ -937,6 +989,12 @@ properties:
configured to trust the configured username configured to trust the configured username
without a password. without a password.
example: trustsome1 example: trustsome1
restore_password:
type: string
description: |
Password with which to connect to the restore
database. Defaults to the "password" option.
example: trustsome1
format: format:
type: string type: string
enum: ['sql'] enum: ['sql']
@ -1014,6 +1072,12 @@ properties:
read_special and one_file_system (see above) to read_special and one_file_system (see above) to
support dump and restore streaming. support dump and restore streaming.
example: /var/lib/sqlite/users.db example: /var/lib/sqlite/users.db
restore_path:
type: string
description: |
Path to the SQLite database file to restore to.
Defaults to the "path" option.
example: /var/lib/sqlite/users.db
mongodb_databases: mongodb_databases:
type: array type: array
items: items:
@ -1036,22 +1100,45 @@ properties:
Database hostname to connect to. Defaults to Database hostname to connect to. Defaults to
connecting to localhost. connecting to localhost.
example: database.example.org example: database.example.org
restore_hostname:
type: string
description: |
Database hostname to restore to. Defaults to
the "hostname" option.
example: database.example.org
port: port:
type: integer type: integer
description: Port to connect to. Defaults to 27017. description: Port to connect to. Defaults to 27017.
example: 27018 example: 27018
restore_port:
type: integer
description: Port to restore to. Defaults to the
"port" option.
example: 5433
username: username:
type: string type: string
description: | description: |
Username with which to connect to the database. Username with which to connect to the database.
Skip it if no authentication is needed. Skip it if no authentication is needed.
example: dbuser example: dbuser
restore_username:
type: string
description: |
Username with which to restore the database.
Defaults to the "username" option.
example: dbuser
password: password:
type: string type: string
description: | description: |
Password with which to connect to the database. Password with which to connect to the database.
Skip it if no authentication is needed. Skip it if no authentication is needed.
example: trustsome1 example: trustsome1
restore_password:
type: string
description: |
Password with which to connect to the restore
database. Defaults to the "password" option.
example: trustsome1
authentication_database: authentication_database:
type: string type: string
description: | description: |

View file

@ -102,7 +102,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process): def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
''' '''
Restore the given MongoDB database from an extract stream. The database is supplied as a Restore the given MongoDB database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema. one-element sequence containing a dict describing the database, as per the configuration schema.
@ -122,7 +124,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
dump_filename = dump.make_database_dump_filename( dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), database['name'], database.get('hostname') make_dump_path(location_config), database['name'], database.get('hostname')
) )
restore_command = build_restore_command(extract_process, database, dump_filename) restore_command = build_restore_command(
extract_process, database, dump_filename, connection_params
)
logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}") logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}")
if dry_run: if dry_run:
@ -138,10 +142,21 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
) )
def build_restore_command(extract_process, database, dump_filename): def build_restore_command(extract_process, database, dump_filename, connection_params):
''' '''
Return the mongorestore command from a single database configuration. Return the mongorestore command from a single database configuration.
''' '''
hostname = connection_params['hostname'] or database.get(
'restore_hostname', database.get('hostname')
)
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
username = connection_params['username'] or database.get(
'restore_username', database.get('username')
)
password = connection_params['password'] or database.get(
'restore_password', database.get('password')
)
command = ['mongorestore'] command = ['mongorestore']
if extract_process: if extract_process:
command.append('--archive') command.append('--archive')
@ -149,14 +164,14 @@ def build_restore_command(extract_process, database, dump_filename):
command.extend(('--dir', dump_filename)) command.extend(('--dir', dump_filename))
if database['name'] != 'all': if database['name'] != 'all':
command.extend(('--drop', '--db', database['name'])) command.extend(('--drop', '--db', database['name']))
if 'hostname' in database: if hostname:
command.extend(('--host', database['hostname'])) command.extend(('--host', hostname))
if 'port' in database: if port:
command.extend(('--port', str(database['port']))) command.extend(('--port', str(port)))
if 'username' in database: if username:
command.extend(('--username', database['username'])) command.extend(('--username', username))
if 'password' in database: if password:
command.extend(('--password', database['password'])) command.extend(('--password', password))
if 'authentication_database' in database: if 'authentication_database' in database:
command.extend(('--authenticationDatabase', database['authentication_database'])) command.extend(('--authenticationDatabase', database['authentication_database']))
if 'restore_options' in database: if 'restore_options' in database:

View file

@ -185,7 +185,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process): def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
''' '''
Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema. one-element sequence containing a dict describing the database, as per the configuration schema.
@ -199,15 +201,27 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
raise ValueError('The database configuration value is invalid') raise ValueError('The database configuration value is invalid')
database = database_config[0] database = database_config[0]
hostname = connection_params['hostname'] or database.get(
'restore_hostname', database.get('hostname')
)
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
username = connection_params['username'] or database.get(
'restore_username', database.get('username')
)
password = connection_params['password'] or database.get(
'restore_password', database.get('password')
)
restore_command = ( restore_command = (
('mysql', '--batch') ('mysql', '--batch')
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
+ (('--host', database['hostname']) if 'hostname' in database else ()) + (('--host', hostname) if hostname else ())
+ (('--port', str(database['port'])) if 'port' in database else ()) + (('--port', str(port)) if port else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + (('--protocol', 'tcp') if hostname or port else ())
+ (('--user', database['username']) if 'username' in database else ()) + (('--user', username) if username else ())
) )
extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None extra_environment = {'MYSQL_PWD': password} if password else None
logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}") logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}")
if dry_run: if dry_run:

View file

@ -23,13 +23,23 @@ def make_dump_path(location_config): # pragma: no cover
) )
def make_extra_environment(database): def make_extra_environment(database, restore_connection_params=None):
''' '''
Make the extra_environment dict from the given database configuration. Make the extra_environment dict from the given database configuration.
If restore connection params are given, this is for a restore operation.
''' '''
extra = dict() extra = dict()
if 'password' in database:
try:
if restore_connection_params:
extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get(
'restore_password', database['password']
)
else:
extra['PGPASSWORD'] = database['password'] extra['PGPASSWORD'] = database['password']
except (AttributeError, KeyError):
pass
extra['PGSSLMODE'] = database.get('ssl_mode', 'disable') extra['PGSSLMODE'] = database.get('ssl_mode', 'disable')
if 'ssl_cert' in database: if 'ssl_cert' in database:
extra['PGSSLCERT'] = database['ssl_cert'] extra['PGSSLCERT'] = database['ssl_cert']
@ -135,6 +145,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
+ (('--host', database['hostname']) if 'hostname' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ()) + (('--username', database['username']) if 'username' in database else ())
+ (('--no-owner',) if database.get('no_owner', False) else ())
+ (('--format', dump_format) if dump_format else ()) + (('--format', dump_format) if dump_format else ())
+ (('--file', dump_filename) if dump_format == 'directory' else ()) + (('--file', dump_filename) if dump_format == 'directory' else ())
+ (tuple(database['options'].split(' ')) if 'options' in database else ()) + (tuple(database['options'].split(' ')) if 'options' in database else ())
@ -192,7 +203,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process): def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
''' '''
Restore the given PostgreSQL database from an extract stream. The database is supplied as a Restore the given PostgreSQL database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema. one-element sequence containing a dict describing the database, as per the configuration schema.
@ -202,6 +215,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
If the extract process is None, then restore the dump from the filesystem rather than from an If the extract process is None, then restore the dump from the filesystem rather than from an
extract stream. extract stream.
Use the given connection parameters to connect to the database. The connection parameters are
hostname, port, username, and password.
''' '''
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
@ -209,6 +225,15 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
raise ValueError('The database configuration value is invalid') raise ValueError('The database configuration value is invalid')
database = database_config[0] database = database_config[0]
hostname = connection_params['hostname'] or database.get(
'restore_hostname', database.get('hostname')
)
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
username = connection_params['username'] or database.get(
'restore_username', database.get('username')
)
all_databases = bool(database['name'] == 'all') all_databases = bool(database['name'] == 'all')
dump_filename = dump.make_database_dump_filename( dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), database['name'], database.get('hostname') make_dump_path(location_config), database['name'], database.get('hostname')
@ -217,9 +242,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
analyze_command = ( analyze_command = (
tuple(psql_command) tuple(psql_command)
+ ('--no-password', '--no-psqlrc', '--quiet') + ('--no-password', '--no-psqlrc', '--quiet')
+ (('--host', database['hostname']) if 'hostname' in database else ()) + (('--host', hostname) if hostname else ())
+ (('--port', str(database['port'])) if 'port' in database else ()) + (('--port', port) if port else ())
+ (('--username', database['username']) if 'username' in database else ()) + (('--username', username) if username else ())
+ (('--dbname', database['name']) if not all_databases else ()) + (('--dbname', database['name']) if not all_databases else ())
+ (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ()) + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
+ ('--command', 'ANALYZE') + ('--command', 'ANALYZE')
@ -231,9 +256,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
+ ('--no-password',) + ('--no-password',)
+ (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean')) + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
+ (('--dbname', database['name']) if not all_databases else ()) + (('--dbname', database['name']) if not all_databases else ())
+ (('--host', database['hostname']) if 'hostname' in database else ()) + (('--host', hostname) if hostname else ())
+ (('--port', str(database['port'])) if 'port' in database else ()) + (('--port', port) if port else ())
+ (('--username', database['username']) if 'username' in database else ()) + (('--username', username) if username else ())
+ (('--no-owner',) if database.get('no_owner', False) else ())
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
+ (() if extract_process else (dump_filename,)) + (() if extract_process else (dump_filename,))
+ tuple( + tuple(
@ -243,7 +269,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
) )
) )
extra_environment = make_extra_environment(database) extra_environment = make_extra_environment(
database, restore_connection_params=connection_params
)
logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}")
if dry_run: if dry_run:

View file

@ -85,7 +85,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name) return dump.make_database_dump_filename(make_dump_path(location_config), name)
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process): def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
''' '''
Restore the given SQLite3 database from an extract stream. The database is supplied as a Restore the given SQLite3 database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema. one-element sequence containing a dict describing the database, as per the configuration schema.
@ -98,7 +100,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
if len(database_config) != 1: if len(database_config) != 1:
raise ValueError('The database configuration value is invalid') raise ValueError('The database configuration value is invalid')
database_path = database_config[0]['path'] database_path = connection_params['restore_path'] or database_config[0].get(
'restore_path', database_config[0].get('path')
)
logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}') logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
if dry_run: if dry_run:

View file

@ -5,16 +5,35 @@ services:
environment: environment:
POSTGRES_PASSWORD: test POSTGRES_PASSWORD: test
POSTGRES_DB: test POSTGRES_DB: test
postgresql2:
image: docker.io/postgres:13.1-alpine
environment:
POSTGRES_PASSWORD: test2
POSTGRES_DB: test
POSTGRES_USER: postgres2
command: -p 5433
mysql: mysql:
image: docker.io/mariadb:10.5 image: docker.io/mariadb:10.5
environment: environment:
MYSQL_ROOT_PASSWORD: test MYSQL_ROOT_PASSWORD: test
MYSQL_DATABASE: test MYSQL_DATABASE: test
mysql2:
image: docker.io/mariadb:10.5
environment:
MYSQL_ROOT_PASSWORD: test2
MYSQL_DATABASE: test
command: --port=3307
mongodb: mongodb:
image: docker.io/mongo:5.0.5 image: docker.io/mongo:5.0.5
environment: environment:
MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: test MONGO_INITDB_ROOT_PASSWORD: test
mongodb2:
image: docker.io/mongo:5.0.5
environment:
MONGO_INITDB_ROOT_USERNAME: root2
MONGO_INITDB_ROOT_PASSWORD: test2
command: --port=27018
tests: tests:
image: docker.io/alpine:3.13 image: docker.io/alpine:3.13
environment: environment:
@ -30,5 +49,8 @@ services:
command: --end-to-end-only command: --end-to-end-only
depends_on: depends_on:
- postgresql - postgresql
- postgresql2
- mysql - mysql
- mysql2
- mongodb - mongodb
- mongodb2

View file

@ -82,6 +82,108 @@ hooks:
config_file.write(config) config_file.write(config)
def write_custom_restore_configuration(
source_directory,
config_path,
repository_path,
borgmatic_source_directory,
postgresql_dump_format='custom',
mongodb_dump_format='archive',
):
'''
Write out borgmatic configuration into a file at the config path. Set the options so as to work
for testing with custom restore options. This includes a custom restore_hostname, restore_port,
restore_username, restore_password and restore_path.
'''
config = f'''
location:
source_directories:
- {source_directory}
repositories:
- {repository_path}
borgmatic_source_directory: {borgmatic_source_directory}
storage:
encryption_passphrase: "test"
hooks:
postgresql_databases:
- name: test
hostname: postgresql
username: postgres
password: test
format: {postgresql_dump_format}
restore_hostname: postgresql2
restore_port: 5433
restore_username: postgres2
restore_password: test2
mysql_databases:
- name: test
hostname: mysql
username: root
password: test
restore_hostname: mysql2
restore_port: 3307
restore_username: root
restore_password: test2
mongodb_databases:
- name: test
hostname: mongodb
username: root
password: test
authentication_database: admin
format: {mongodb_dump_format}
restore_hostname: mongodb2
restore_port: 27018
restore_username: root2
restore_password: test2
sqlite_databases:
- name: sqlite_test
path: /tmp/sqlite_test.db
restore_path: /tmp/sqlite_test2.db
'''
with open(config_path, 'w') as config_file:
config_file.write(config)
def write_simple_custom_restore_configuration(
source_directory,
config_path,
repository_path,
borgmatic_source_directory,
postgresql_dump_format='custom',
):
'''
Write out borgmatic configuration into a file at the config path. Set the options so as to work
for testing with custom restore options, but this time using CLI arguments. This includes a
custom restore_hostname, restore_port, restore_username and restore_password as we only test
these options for PostgreSQL.
'''
config = f'''
location:
source_directories:
- {source_directory}
repositories:
- {repository_path}
borgmatic_source_directory: {borgmatic_source_directory}
storage:
encryption_passphrase: "test"
hooks:
postgresql_databases:
- name: test
hostname: postgresql
username: postgres
password: test
format: {postgresql_dump_format}
'''
with open(config_path, 'w') as config_file:
config_file.write(config)
def test_database_dump_and_restore(): def test_database_dump_and_restore():
# Create a Borg repository. # Create a Borg repository.
temporary_directory = tempfile.mkdtemp() temporary_directory = tempfile.mkdtemp()
@ -125,6 +227,103 @@ def test_database_dump_and_restore():
shutil.rmtree(temporary_directory) shutil.rmtree(temporary_directory)
def test_database_dump_and_restore_with_restore_cli_arguments():
# Create a Borg repository.
temporary_directory = tempfile.mkdtemp()
repository_path = os.path.join(temporary_directory, 'test.borg')
borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
original_working_directory = os.getcwd()
try:
config_path = os.path.join(temporary_directory, 'test.yaml')
write_simple_custom_restore_configuration(
temporary_directory, config_path, repository_path, borgmatic_source_directory
)
subprocess.check_call(
['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
)
# Run borgmatic to generate a backup archive including a database dump.
subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
# Get the created archive name.
output = subprocess.check_output(
['borgmatic', '--config', config_path, 'list', '--json']
).decode(sys.stdout.encoding)
parsed_output = json.loads(output)
assert len(parsed_output) == 1
assert len(parsed_output[0]['archives']) == 1
archive_name = parsed_output[0]['archives'][0]['archive']
# Restore the database from the archive.
subprocess.check_call(
[
'borgmatic',
'-v',
'2',
'--config',
config_path,
'restore',
'--archive',
archive_name,
'--hostname',
'postgresql2',
'--port',
'5433',
'--username',
'postgres2',
'--password',
'test2',
]
)
finally:
os.chdir(original_working_directory)
shutil.rmtree(temporary_directory)
def test_database_dump_and_restore_with_restore_configuration_options():
# Create a Borg repository.
temporary_directory = tempfile.mkdtemp()
repository_path = os.path.join(temporary_directory, 'test.borg')
borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
original_working_directory = os.getcwd()
try:
config_path = os.path.join(temporary_directory, 'test.yaml')
write_custom_restore_configuration(
temporary_directory, config_path, repository_path, borgmatic_source_directory
)
subprocess.check_call(
['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
)
# Run borgmatic to generate a backup archive including a database dump.
subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
# Get the created archive name.
output = subprocess.check_output(
['borgmatic', '--config', config_path, 'list', '--json']
).decode(sys.stdout.encoding)
parsed_output = json.loads(output)
assert len(parsed_output) == 1
assert len(parsed_output[0]['archives']) == 1
archive_name = parsed_output[0]['archives'][0]['archive']
# Restore the database from the archive.
subprocess.check_call(
['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name]
)
finally:
os.chdir(original_working_directory)
shutil.rmtree(temporary_directory)
def test_database_dump_and_restore_with_directory_format(): def test_database_dump_and_restore_with_directory_format():
# Create a Borg repository. # Create a Borg repository.
temporary_directory = tempfile.mkdtemp() temporary_directory = tempfile.mkdtemp()

View file

@ -241,6 +241,7 @@ def test_run_restore_restores_each_database():
archive_name=object, archive_name=object,
hook_name='postgresql_databases', hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None}, database={'name': 'foo', 'schemas': None},
connection_params=object,
).once() ).once()
flexmock(module).should_receive('restore_single_database').with_args( flexmock(module).should_receive('restore_single_database').with_args(
repository=object, repository=object,
@ -254,6 +255,7 @@ def test_run_restore_restores_each_database():
archive_name=object, archive_name=object,
hook_name='postgresql_databases', hook_name='postgresql_databases',
database={'name': 'bar', 'schemas': None}, database={'name': 'bar', 'schemas': None},
connection_params=object,
).once() ).once()
flexmock(module).should_receive('ensure_databases_found') flexmock(module).should_receive('ensure_databases_found')
@ -264,7 +266,15 @@ def test_run_restore_restores_each_database():
hooks=flexmock(), hooks=flexmock(),
local_borg_version=flexmock(), local_borg_version=flexmock(),
restore_arguments=flexmock( restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
), ),
global_arguments=flexmock(dry_run=False), global_arguments=flexmock(dry_run=False),
local_path=flexmock(), local_path=flexmock(),
@ -337,6 +347,7 @@ def test_run_restore_restores_database_configured_with_all_name():
archive_name=object, archive_name=object,
hook_name='postgresql_databases', hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None}, database={'name': 'foo', 'schemas': None},
connection_params=object,
).once() ).once()
flexmock(module).should_receive('restore_single_database').with_args( flexmock(module).should_receive('restore_single_database').with_args(
repository=object, repository=object,
@ -350,6 +361,7 @@ def test_run_restore_restores_database_configured_with_all_name():
archive_name=object, archive_name=object,
hook_name='postgresql_databases', hook_name='postgresql_databases',
database={'name': 'bar', 'schemas': None}, database={'name': 'bar', 'schemas': None},
connection_params=object,
).once() ).once()
flexmock(module).should_receive('ensure_databases_found') flexmock(module).should_receive('ensure_databases_found')
@ -360,7 +372,15 @@ def test_run_restore_restores_database_configured_with_all_name():
hooks=flexmock(), hooks=flexmock(),
local_borg_version=flexmock(), local_borg_version=flexmock(),
restore_arguments=flexmock( restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
), ),
global_arguments=flexmock(dry_run=False), global_arguments=flexmock(dry_run=False),
local_path=flexmock(), local_path=flexmock(),
@ -411,6 +431,7 @@ def test_run_restore_skips_missing_database():
archive_name=object, archive_name=object,
hook_name='postgresql_databases', hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None}, database={'name': 'foo', 'schemas': None},
connection_params=object,
).once() ).once()
flexmock(module).should_receive('restore_single_database').with_args( flexmock(module).should_receive('restore_single_database').with_args(
repository=object, repository=object,
@ -424,6 +445,7 @@ def test_run_restore_skips_missing_database():
archive_name=object, archive_name=object,
hook_name='postgresql_databases', hook_name='postgresql_databases',
database={'name': 'bar', 'schemas': None}, database={'name': 'bar', 'schemas': None},
connection_params=object,
).never() ).never()
flexmock(module).should_receive('ensure_databases_found') flexmock(module).should_receive('ensure_databases_found')
@ -434,7 +456,15 @@ def test_run_restore_skips_missing_database():
hooks=flexmock(), hooks=flexmock(),
local_borg_version=flexmock(), local_borg_version=flexmock(),
restore_arguments=flexmock( restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
), ),
global_arguments=flexmock(dry_run=False), global_arguments=flexmock(dry_run=False),
local_path=flexmock(), local_path=flexmock(),
@ -479,6 +509,7 @@ def test_run_restore_restores_databases_from_different_hooks():
archive_name=object, archive_name=object,
hook_name='postgresql_databases', hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None}, database={'name': 'foo', 'schemas': None},
connection_params=object,
).once() ).once()
flexmock(module).should_receive('restore_single_database').with_args( flexmock(module).should_receive('restore_single_database').with_args(
repository=object, repository=object,
@ -492,6 +523,7 @@ def test_run_restore_restores_databases_from_different_hooks():
archive_name=object, archive_name=object,
hook_name='mysql_databases', hook_name='mysql_databases',
database={'name': 'bar', 'schemas': None}, database={'name': 'bar', 'schemas': None},
connection_params=object,
).once() ).once()
flexmock(module).should_receive('ensure_databases_found') flexmock(module).should_receive('ensure_databases_found')
@ -502,7 +534,15 @@ def test_run_restore_restores_databases_from_different_hooks():
hooks=flexmock(), hooks=flexmock(),
local_borg_version=flexmock(), local_borg_version=flexmock(),
restore_arguments=flexmock( restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
), ),
global_arguments=flexmock(dry_run=False), global_arguments=flexmock(dry_run=False),
local_path=flexmock(), local_path=flexmock(),

View file

@ -171,7 +171,17 @@ def test_restore_database_dump_runs_mongorestore():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -185,7 +195,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
with pytest.raises(ValueError): with pytest.raises(ValueError):
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock() database_config,
'test.yaml',
{},
dry_run=False,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -215,7 +235,17 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -253,7 +283,129 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
database_config = [
{
'name': 'foo',
'username': 'mongo',
'password': 'trustsome1',
'authentication_database': 'admin',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_username': 'restoreusername',
'restore_password': 'restorepassword',
'schemas': None,
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
[
'mongorestore',
'--archive',
'--drop',
'--db',
'foo',
'--host',
'clihost',
'--port',
'cliport',
'--username',
'cliusername',
'--password',
'clipassword',
'--authenticationDatabase',
'admin',
],
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': 'clihost',
'port': 'cliport',
'username': 'cliusername',
'password': 'clipassword',
},
)
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
database_config = [
{
'name': 'foo',
'username': 'mongo',
'password': 'trustsome1',
'authentication_database': 'admin',
'schemas': None,
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_username': 'restoreuser',
'restore_password': 'restorepass',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
[
'mongorestore',
'--archive',
'--drop',
'--db',
'foo',
'--host',
'restorehost',
'--port',
'restoreport',
'--username',
'restoreuser',
'--password',
'restorepass',
'--authenticationDatabase',
'admin',
],
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -271,7 +423,17 @@ def test_restore_database_dump_runs_mongorestore_with_options():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -299,7 +461,17 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -317,7 +489,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -329,7 +511,17 @@ def test_restore_database_dump_with_dry_run_skips_restore():
flexmock(module).should_receive('execute_command_with_processes').never() flexmock(module).should_receive('execute_command_with_processes').never()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock() database_config,
'test.yaml',
{},
dry_run=True,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -346,5 +538,15 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=None database_config,
'test.yaml',
{},
dry_run=False,
extract_process=None,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )

View file

@ -392,7 +392,17 @@ def test_restore_database_dump_runs_mysql_to_restore():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -404,7 +414,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
with pytest.raises(ValueError): with pytest.raises(ValueError):
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock() database_config,
'test.yaml',
{},
dry_run=False,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -421,7 +441,17 @@ def test_restore_database_dump_runs_mysql_with_options():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -447,7 +477,17 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -464,7 +504,115 @@ def test_restore_database_dump_runs_mysql_with_username_and_password():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
database_config = [
{
'name': 'foo',
'username': 'root',
'password': 'trustsome1',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_username': 'restoreusername',
'restore_password': 'restorepassword',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'mysql',
'--batch',
'--host',
'clihost',
'--port',
'cliport',
'--protocol',
'tcp',
'--user',
'cliusername',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'MYSQL_PWD': 'clipassword'},
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': 'clihost',
'port': 'cliport',
'username': 'cliusername',
'password': 'clipassword',
},
)
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
database_config = [
{
'name': 'foo',
'username': 'root',
'password': 'trustsome1',
'hostname': 'dbhost',
'port': 'dbport',
'restore_username': 'restoreuser',
'restore_password': 'restorepass',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'mysql',
'--batch',
'--host',
'restorehost',
'--port',
'restoreport',
'--protocol',
'tcp',
'--user',
'restoreuser',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'MYSQL_PWD': 'restorepass'},
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -474,5 +622,15 @@ def test_restore_database_dump_with_dry_run_skips_restore():
flexmock(module).should_receive('execute_command_with_processes').never() flexmock(module).should_receive('execute_command_with_processes').never()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock() database_config,
'test.yaml',
{},
dry_run=True,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )

View file

@ -479,7 +479,17 @@ def test_restore_database_dump_runs_pg_restore():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -494,7 +504,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
with pytest.raises(ValueError): with pytest.raises(ValueError):
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock() database_config,
'test.yaml',
{},
dry_run=False,
extract_process=flexmock(),
connection_params={
'restore_hostname': None,
'restore_port': None,
'restore_username': None,
'restore_password': None,
},
) )
@ -545,7 +565,17 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -594,7 +624,183 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_make_extra_environment_with_cli_password_sets_correct_password():
database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'}
extra = module.make_extra_environment(
database, restore_connection_params={'password': 'clipassword'}
)
assert extra['PGPASSWORD'] == 'clipassword'
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
database_config = [
{
'name': 'foo',
'hostname': 'database.example.org',
'port': 5433,
'username': 'postgres',
'password': 'trustsome1',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_username': 'restoreusername',
'restore_password': 'restorepassword',
'schemas': None,
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_extra_environment').and_return(
{'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}
)
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'pg_restore',
'--no-password',
'--if-exists',
'--exit-on-error',
'--clean',
'--dbname',
'foo',
'--host',
'clihost',
'--port',
'cliport',
'--username',
'cliusername',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
).once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
'--no-password',
'--no-psqlrc',
'--quiet',
'--host',
'clihost',
'--port',
'cliport',
'--username',
'cliusername',
'--dbname',
'foo',
'--command',
'ANALYZE',
),
extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': 'clihost',
'port': 'cliport',
'username': 'cliusername',
'password': 'clipassword',
},
)
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
database_config = [
{
'name': 'foo',
'hostname': 'database.example.org',
'port': 5433,
'username': 'postgres',
'password': 'trustsome1',
'schemas': None,
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_username': 'restoreusername',
'restore_password': 'restorepassword',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_extra_environment').and_return(
{'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}
)
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'pg_restore',
'--no-password',
'--if-exists',
'--exit-on-error',
'--clean',
'--dbname',
'foo',
'--host',
'restorehost',
'--port',
'restoreport',
'--username',
'restoreusername',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
).once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
'--no-password',
'--no-psqlrc',
'--quiet',
'--host',
'restorehost',
'--port',
'restoreport',
'--username',
'restoreusername',
'--dbname',
'foo',
'--command',
'ANALYZE',
),
extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -644,7 +850,17 @@ def test_restore_database_dump_runs_pg_restore_with_options():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -672,7 +888,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -705,7 +931,17 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -759,7 +995,17 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -772,7 +1018,17 @@ def test_restore_database_dump_with_dry_run_skips_restore():
flexmock(module).should_receive('execute_command_with_processes').never() flexmock(module).should_receive('execute_command_with_processes').never()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock() database_config,
'test.yaml',
{},
dry_run=True,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -813,7 +1069,17 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=None database_config,
'test.yaml',
{},
dry_run=False,
extract_process=None,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )
@ -858,5 +1124,15 @@ def test_restore_database_dump_with_schemas_restores_schemas():
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=None database_config,
'test.yaml',
{},
dry_run=False,
extract_process=None,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
) )

View file

@ -1,3 +1,4 @@
import logging
import pytest import pytest
from flexmock import flexmock from flexmock import flexmock
@ -94,12 +95,81 @@ def test_restore_database_dump_restores_database():
database_config = [{'path': '/path/to/database', 'name': 'database'}] database_config = [{'path': '/path/to/database', 'name': 'database'}]
extract_process = flexmock(stdout=flexmock()) extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').once() flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'sqlite3',
'/path/to/database',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
flexmock(module.os).should_receive('remove').once() flexmock(module.os).should_receive('remove').once()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={'restore_path': None},
)
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
database_config = [
{'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'sqlite3',
'cli/path/to/database',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
flexmock(module.os).should_receive('remove').once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={'restore_path': 'cli/path/to/database'},
)
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
database_config = [
{'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'sqlite3',
'config/path/to/database',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
flexmock(module.os).should_receive('remove').once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={'restore_path': None},
) )
@ -111,7 +181,12 @@ def test_restore_database_dump_does_not_restore_database_if_dry_run():
flexmock(module.os).should_receive('remove').never() flexmock(module.os).should_receive('remove').never()
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=True,
extract_process=extract_process,
connection_params={'restore_path': None},
) )
@ -121,5 +196,10 @@ def test_restore_database_dump_raises_error_if_database_config_is_invalid():
with pytest.raises(ValueError): with pytest.raises(ValueError):
module.restore_database_dump( module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={'restore_path': None},
) )