psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'main' of github.com:borgmatic-collective/borgmatic

Browse source
This commit is contained in:
Dan Helfman 2023-06-22 12:29:32 -07:00
commit 9e0df595c8
14 changed files with 1222 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
borgmatic/actions/restore.py
View file

@ -68,9 +68,11 @@ def restore_single_database(
archive_name,
hook_name,
database,
connection_params,
): # pragma: no cover
'''
Given (among other things) an archive name, a database hook name, and a configured database
Given (among other things) an archive name, a database hook name, the hostname,
port, username and password as connection params, and a configured database
configuration dict, restore that database from the archive.
'''
logger.info(
@ -113,6 +115,7 @@ def restore_single_database(
location,
global_arguments.dry_run,
extract_process,
connection_params,
)
@ -301,6 +304,13 @@ def run_restore(
restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names)
found_names = set()
remaining_restore_names = {}
connection_params = {
'hostname': restore_arguments.hostname,
'port': restore_arguments.port,
'username': restore_arguments.username,
'password': restore_arguments.password,
'restore_path': restore_arguments.restore_path,
}
for hook_name, database_names in restore_names.items():
for database_name in database_names:
@ -327,6 +337,7 @@ def run_restore(
archive_name,
found_hook_name or hook_name,
dict(found_database, **{'schemas': restore_arguments.schemas}),
connection_params,
)
# For any database that weren't found via exact matches in the hooks configuration, try to
@ -356,6 +367,7 @@ def run_restore(
archive_name,
found_hook_name or hook_name,
dict(database, **{'schemas': restore_arguments.schemas}),
connection_params,
)
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(

20
borgmatic/commands/arguments.py
View file

@ -931,6 +931,26 @@ def make_parsers():
dest='schemas',
help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases',
)
restore_group.add_argument(
'--hostname',
help='Database hostname to restore to. Defaults to the "restore_hostname" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--port',
help='Port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--username',
help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--password',
help='Password with which to connect to the restore database. Defaults to the "restore_password" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--restore-path',
help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'-h', '--help', action='help', help='Show this help message and exit'
)

87
borgmatic/config/schema.yaml
View file

@ -763,10 +763,21 @@ properties:
Database hostname to connect to. Defaults to
connecting via local Unix socket.
example: database.example.org
restore_hostname:
type: string
description: |
Database hostname to restore to. Defaults to
the "hostname" option.
example: database.example.org
port:
type: integer
description: Port to connect to. Defaults to 5432.
example: 5433
restore_port:
type: integer
description: Port to restore to. Defaults to the
"port" option.
example: 5433
username:
type: string
description: |
@ -775,6 +786,12 @@ properties:
You probably want to specify the "postgres"
superuser here when the database name is "all".
example: dbuser
restore_username:
type: string
description: |
Username with which to restore the database.
Defaults to the "username" option.
example: dbuser
password:
type: string
description: |
@ -784,6 +801,24 @@ properties:
without a password or you create a ~/.pgpass
file.
example: trustsome1
restore_password:
type: string
description: |
Password with which to connect to the restore
database. Defaults to the "password" option.
example: trustsome1
no_owner:
type: boolean
description: |
Do not output commands to set ownership of
objects to match the original database. By
default, pg_dump and pg_restore issue ALTER
OWNER or SET SESSION AUTHORIZATION statements
to set ownership of created schema elements.
These statements will fail unless the initial
connection to the database is made by a
superuser.
example: true
format:
type: string
enum: ['plain', 'custom', 'directory', 'tar']
@ -919,16 +954,33 @@ properties:
Database hostname to connect to. Defaults to
connecting via local Unix socket.
example: database.example.org
restore_hostname:
type: string
description: |
Database hostname to restore to. Defaults to
the "hostname" option.
example: database.example.org
port:
type: integer
description: Port to connect to. Defaults to 3306.
example: 3307
restore_port:
type: integer
description: Port to restore to. Defaults to the
"port" option.
example: 5433
username:
type: string
description: |
Username with which to connect to the database.
Defaults to the username of the current user.
example: dbuser
restore_username:
type: string
description: |
Username with which to restore the database.
Defaults to the "username" option.
example: dbuser
password:
type: string
description: |
@ -937,6 +989,12 @@ properties:
configured to trust the configured username
without a password.
example: trustsome1
restore_password:
type: string
description: |
Password with which to connect to the restore
database. Defaults to the "password" option.
example: trustsome1
format:
type: string
enum: ['sql']
@ -1014,6 +1072,12 @@ properties:
read_special and one_file_system (see above) to
support dump and restore streaming.
example: /var/lib/sqlite/users.db
restore_path:
type: string
description: |
Path to the SQLite database file to restore to.
Defaults to the "path" option.
example: /var/lib/sqlite/users.db
mongodb_databases:
type: array
items:
@ -1036,22 +1100,45 @@ properties:
Database hostname to connect to. Defaults to
connecting to localhost.
example: database.example.org
restore_hostname:
type: string
description: |
Database hostname to restore to. Defaults to
the "hostname" option.
example: database.example.org
port:
type: integer
description: Port to connect to. Defaults to 27017.
example: 27018
restore_port:
type: integer
description: Port to restore to. Defaults to the
"port" option.
example: 5433
username:
type: string
description: |
Username with which to connect to the database.
Skip it if no authentication is needed.
example: dbuser
restore_username:
type: string
description: |
Username with which to restore the database.
Defaults to the "username" option.
example: dbuser
password:
type: string
description: |
Password with which to connect to the database.
Skip it if no authentication is needed.
example: trustsome1
restore_password:
type: string
description: |
Password with which to connect to the restore
database. Defaults to the "password" option.
example: trustsome1
authentication_database:
type: string
description: |

37
borgmatic/hooks/mongodb.py
View file

@ -102,7 +102,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
'''
Restore the given MongoDB database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
@ -122,7 +124,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), database['name'], database.get('hostname')
)
restore_command = build_restore_command(extract_process, database, dump_filename)
restore_command = build_restore_command(
extract_process, database, dump_filename, connection_params
)
logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}")
if dry_run:
@ -138,10 +142,21 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
)
def build_restore_command(extract_process, database, dump_filename):
def build_restore_command(extract_process, database, dump_filename, connection_params):
'''
Return the mongorestore command from a single database configuration.
'''
hostname = connection_params['hostname'] or database.get(
'restore_hostname', database.get('hostname')
)
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
username = connection_params['username'] or database.get(
'restore_username', database.get('username')
)
password = connection_params['password'] or database.get(
'restore_password', database.get('password')
)
command = ['mongorestore']
if extract_process:
command.append('--archive')
@ -149,14 +164,14 @@ def build_restore_command(extract_process, database, dump_filename):
command.extend(('--dir', dump_filename))
if database['name'] != 'all':
command.extend(('--drop', '--db', database['name']))
if 'hostname' in database:
command.extend(('--host', database['hostname']))
if 'port' in database:
command.extend(('--port', str(database['port'])))
if 'username' in database:
command.extend(('--username', database['username']))
if 'password' in database:
command.extend(('--password', database['password']))
if hostname:
command.extend(('--host', hostname))
if port:
command.extend(('--port', str(port)))
if username:
command.extend(('--username', username))
if password:
command.extend(('--password', password))
if 'authentication_database' in database:
command.extend(('--authenticationDatabase', database['authentication_database']))
if 'restore_options' in database:

26
borgmatic/hooks/mysql.py
View file

@ -185,7 +185,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
'''
Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
@ -199,15 +201,27 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
raise ValueError('The database configuration value is invalid')
database = database_config[0]
hostname = connection_params['hostname'] or database.get(
'restore_hostname', database.get('hostname')
)
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
username = connection_params['username'] or database.get(
'restore_username', database.get('username')
)
password = connection_params['password'] or database.get(
'restore_password', database.get('password')
)
restore_command = (
('mysql', '--batch')
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--user', database['username']) if 'username' in database else ())
+ (('--host', hostname) if hostname else ())
+ (('--port', str(port)) if port else ())
+ (('--protocol', 'tcp') if hostname or port else ())
+ (('--user', username) if username else ())
)
extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
extra_environment = {'MYSQL_PWD': password} if password else None
logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}")
if dry_run:

50
borgmatic/hooks/postgresql.py
View file

@ -23,13 +23,23 @@ def make_dump_path(location_config): # pragma: no cover
)
def make_extra_environment(database):
def make_extra_environment(database, restore_connection_params=None):
'''
Make the extra_environment dict from the given database configuration.
If restore connection params are given, this is for a restore operation.
'''
extra = dict()
if 'password' in database:
extra['PGPASSWORD'] = database['password']
try:
if restore_connection_params:
extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get(
'restore_password', database['password']
)
else:
extra['PGPASSWORD'] = database['password']
except (AttributeError, KeyError):
pass
extra['PGSSLMODE'] = database.get('ssl_mode', 'disable')
if 'ssl_cert' in database:
extra['PGSSLCERT'] = database['ssl_cert']
@ -135,6 +145,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ())
+ (('--no-owner',) if database.get('no_owner', False) else ())
+ (('--format', dump_format) if dump_format else ())
+ (('--file', dump_filename) if dump_format == 'directory' else ())
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
@ -192,7 +203,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
'''
Restore the given PostgreSQL database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
@ -202,6 +215,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
If the extract process is None, then restore the dump from the filesystem rather than from an
extract stream.
Use the given connection parameters to connect to the database. The connection parameters are
hostname, port, username, and password.
'''
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
@ -209,6 +225,15 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
raise ValueError('The database configuration value is invalid')
database = database_config[0]
hostname = connection_params['hostname'] or database.get(
'restore_hostname', database.get('hostname')
)
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
username = connection_params['username'] or database.get(
'restore_username', database.get('username')
)
all_databases = bool(database['name'] == 'all')
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), database['name'], database.get('hostname')
@ -217,9 +242,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
analyze_command = (
tuple(psql_command)
+ ('--no-password', '--no-psqlrc', '--quiet')
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ())
+ (('--host', hostname) if hostname else ())
+ (('--port', port) if port else ())
+ (('--username', username) if username else ())
+ (('--dbname', database['name']) if not all_databases else ())
+ (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
+ ('--command', 'ANALYZE')
@ -231,9 +256,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
+ ('--no-password',)
+ (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
+ (('--dbname', database['name']) if not all_databases else ())
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ())
+ (('--host', hostname) if hostname else ())
+ (('--port', port) if port else ())
+ (('--username', username) if username else ())
+ (('--no-owner',) if database.get('no_owner', False) else ())
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
+ (() if extract_process else (dump_filename,))
+ tuple(
@ -243,7 +269,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
)
)
extra_environment = make_extra_environment(database)
extra_environment = make_extra_environment(
database, restore_connection_params=connection_params
)
logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}")
if dry_run:

8
borgmatic/hooks/sqlite.py
View file

@ -85,7 +85,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name)
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
'''
Restore the given SQLite3 database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
@ -98,7 +100,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
if len(database_config) != 1:
raise ValueError('The database configuration value is invalid')
database_path = database_config[0]['path']
database_path = connection_params['restore_path'] or database_config[0].get(
'restore_path', database_config[0].get('path')
)
logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
if dry_run:

22
tests/end-to-end/docker-compose.yaml
View file

@ -5,16 +5,35 @@ services:
environment:
POSTGRES_PASSWORD: test
POSTGRES_DB: test
postgresql2:
image: docker.io/postgres:13.1-alpine
environment:
POSTGRES_PASSWORD: test2
POSTGRES_DB: test
POSTGRES_USER: postgres2
command: -p 5433
mysql:
image: docker.io/mariadb:10.5
environment:
MYSQL_ROOT_PASSWORD: test
MYSQL_DATABASE: test
mysql2:
image: docker.io/mariadb:10.5
environment:
MYSQL_ROOT_PASSWORD: test2
MYSQL_DATABASE: test
command: --port=3307
mongodb:
image: docker.io/mongo:5.0.5
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: test
mongodb2:
image: docker.io/mongo:5.0.5
environment:
MONGO_INITDB_ROOT_USERNAME: root2
MONGO_INITDB_ROOT_PASSWORD: test2
command: --port=27018
tests:
image: docker.io/alpine:3.13
environment:
@ -30,5 +49,8 @@ services:
command: --end-to-end-only
depends_on:
- postgresql
- postgresql2
- mysql
- mysql2
- mongodb
- mongodb2

199
tests/end-to-end/test_database.py
View file

@ -82,6 +82,108 @@ hooks:
config_file.write(config)
def write_custom_restore_configuration(
source_directory,
config_path,
repository_path,
borgmatic_source_directory,
postgresql_dump_format='custom',
mongodb_dump_format='archive',
):
'''
Write out borgmatic configuration into a file at the config path. Set the options so as to work
for testing with custom restore options. This includes a custom restore_hostname, restore_port,
restore_username, restore_password and restore_path.
'''
config = f'''
location:
source_directories:
- {source_directory}
repositories:
- {repository_path}
borgmatic_source_directory: {borgmatic_source_directory}
storage:
encryption_passphrase: "test"
hooks:
postgresql_databases:
- name: test
hostname: postgresql
username: postgres
password: test
format: {postgresql_dump_format}
restore_hostname: postgresql2
restore_port: 5433
restore_username: postgres2
restore_password: test2
mysql_databases:
- name: test
hostname: mysql
username: root
password: test
restore_hostname: mysql2
restore_port: 3307
restore_username: root
restore_password: test2
mongodb_databases:
- name: test
hostname: mongodb
username: root
password: test
authentication_database: admin
format: {mongodb_dump_format}
restore_hostname: mongodb2
restore_port: 27018
restore_username: root2
restore_password: test2
sqlite_databases:
- name: sqlite_test
path: /tmp/sqlite_test.db
restore_path: /tmp/sqlite_test2.db
'''
with open(config_path, 'w') as config_file:
config_file.write(config)
def write_simple_custom_restore_configuration(
source_directory,
config_path,
repository_path,
borgmatic_source_directory,
postgresql_dump_format='custom',
):
'''
Write out borgmatic configuration into a file at the config path. Set the options so as to work
for testing with custom restore options, but this time using CLI arguments. This includes a
custom restore_hostname, restore_port, restore_username and restore_password as we only test
these options for PostgreSQL.
'''
config = f'''
location:
source_directories:
- {source_directory}
repositories:
- {repository_path}
borgmatic_source_directory: {borgmatic_source_directory}
storage:
encryption_passphrase: "test"
hooks:
postgresql_databases:
- name: test
hostname: postgresql
username: postgres
password: test
format: {postgresql_dump_format}
'''
with open(config_path, 'w') as config_file:
config_file.write(config)
def test_database_dump_and_restore():
# Create a Borg repository.
temporary_directory = tempfile.mkdtemp()
@ -125,6 +227,103 @@ def test_database_dump_and_restore():
shutil.rmtree(temporary_directory)
def test_database_dump_and_restore_with_restore_cli_arguments():
# Create a Borg repository.
temporary_directory = tempfile.mkdtemp()
repository_path = os.path.join(temporary_directory, 'test.borg')
borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
original_working_directory = os.getcwd()
try:
config_path = os.path.join(temporary_directory, 'test.yaml')
write_simple_custom_restore_configuration(
temporary_directory, config_path, repository_path, borgmatic_source_directory
)
subprocess.check_call(
['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
)
# Run borgmatic to generate a backup archive including a database dump.
subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
# Get the created archive name.
output = subprocess.check_output(
['borgmatic', '--config', config_path, 'list', '--json']
).decode(sys.stdout.encoding)
parsed_output = json.loads(output)
assert len(parsed_output) == 1
assert len(parsed_output[0]['archives']) == 1
archive_name = parsed_output[0]['archives'][0]['archive']
# Restore the database from the archive.
subprocess.check_call(
[
'borgmatic',
'-v',
'2',
'--config',
config_path,
'restore',
'--archive',
archive_name,
'--hostname',
'postgresql2',
'--port',
'5433',
'--username',
'postgres2',
'--password',
'test2',
]
)
finally:
os.chdir(original_working_directory)
shutil.rmtree(temporary_directory)
def test_database_dump_and_restore_with_restore_configuration_options():
# Create a Borg repository.
temporary_directory = tempfile.mkdtemp()
repository_path = os.path.join(temporary_directory, 'test.borg')
borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
original_working_directory = os.getcwd()
try:
config_path = os.path.join(temporary_directory, 'test.yaml')
write_custom_restore_configuration(
temporary_directory, config_path, repository_path, borgmatic_source_directory
)
subprocess.check_call(
['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
)
# Run borgmatic to generate a backup archive including a database dump.
subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
# Get the created archive name.
output = subprocess.check_output(
['borgmatic', '--config', config_path, 'list', '--json']
).decode(sys.stdout.encoding)
parsed_output = json.loads(output)
assert len(parsed_output) == 1
assert len(parsed_output[0]['archives']) == 1
archive_name = parsed_output[0]['archives'][0]['archive']
# Restore the database from the archive.
subprocess.check_call(
['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name]
)
finally:
os.chdir(original_working_directory)
shutil.rmtree(temporary_directory)
def test_database_dump_and_restore_with_directory_format():
# Create a Borg repository.
temporary_directory = tempfile.mkdtemp()

48
tests/unit/actions/test_restore.py
View file

@ -241,6 +241,7 @@ def test_run_restore_restores_each_database():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
@ -254,6 +255,7 @@ def test_run_restore_restores_each_database():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'bar', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('ensure_databases_found')
@ -264,7 +266,15 @@ def test_run_restore_restores_each_database():
hooks=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None
repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),
@ -337,6 +347,7 @@ def test_run_restore_restores_database_configured_with_all_name():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
@ -350,6 +361,7 @@ def test_run_restore_restores_database_configured_with_all_name():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'bar', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('ensure_databases_found')
@ -360,7 +372,15 @@ def test_run_restore_restores_database_configured_with_all_name():
hooks=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None
repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),
@ -411,6 +431,7 @@ def test_run_restore_skips_missing_database():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
@ -424,6 +445,7 @@ def test_run_restore_skips_missing_database():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'bar', 'schemas': None},
connection_params=object,
).never()
flexmock(module).should_receive('ensure_databases_found')
@ -434,7 +456,15 @@ def test_run_restore_skips_missing_database():
hooks=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None
repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),
@ -479,6 +509,7 @@ def test_run_restore_restores_databases_from_different_hooks():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
@ -492,6 +523,7 @@ def test_run_restore_restores_databases_from_different_hooks():
archive_name=object,
hook_name='mysql_databases',
database={'name': 'bar', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('ensure_databases_found')
@ -502,7 +534,15 @@ def test_run_restore_restores_databases_from_different_hooks():
hooks=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None
repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),

220
tests/unit/hooks/test_mongodb.py
View file

@ -171,7 +171,17 @@ def test_restore_database_dump_runs_mongorestore():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -185,7 +195,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
with pytest.raises(ValueError):
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -215,7 +235,17 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -253,7 +283,129 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
database_config = [
{
'name': 'foo',
'username': 'mongo',
'password': 'trustsome1',
'authentication_database': 'admin',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_username': 'restoreusername',
'restore_password': 'restorepassword',
'schemas': None,
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
[
'mongorestore',
'--archive',
'--drop',
'--db',
'foo',
'--host',
'clihost',
'--port',
'cliport',
'--username',
'cliusername',
'--password',
'clipassword',
'--authenticationDatabase',
'admin',
],
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': 'clihost',
'port': 'cliport',
'username': 'cliusername',
'password': 'clipassword',
},
)
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
database_config = [
{
'name': 'foo',
'username': 'mongo',
'password': 'trustsome1',
'authentication_database': 'admin',
'schemas': None,
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_username': 'restoreuser',
'restore_password': 'restorepass',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
[
'mongorestore',
'--archive',
'--drop',
'--db',
'foo',
'--host',
'restorehost',
'--port',
'restoreport',
'--username',
'restoreuser',
'--password',
'restorepass',
'--authenticationDatabase',
'admin',
],
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -271,7 +423,17 @@ def test_restore_database_dump_runs_mongorestore_with_options():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -299,7 +461,17 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -317,7 +489,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -329,7 +511,17 @@ def test_restore_database_dump_with_dry_run_skips_restore():
flexmock(module).should_receive('execute_command_with_processes').never()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=True,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -346,5 +538,15 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=None
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=None,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)

170
tests/unit/hooks/test_mysql.py
View file

@ -392,7 +392,17 @@ def test_restore_database_dump_runs_mysql_to_restore():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -404,7 +414,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
with pytest.raises(ValueError):
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -421,7 +441,17 @@ def test_restore_database_dump_runs_mysql_with_options():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -447,7 +477,17 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -464,7 +504,115 @@ def test_restore_database_dump_runs_mysql_with_username_and_password():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
database_config = [
{
'name': 'foo',
'username': 'root',
'password': 'trustsome1',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_username': 'restoreusername',
'restore_password': 'restorepassword',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'mysql',
'--batch',
'--host',
'clihost',
'--port',
'cliport',
'--protocol',
'tcp',
'--user',
'cliusername',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'MYSQL_PWD': 'clipassword'},
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': 'clihost',
'port': 'cliport',
'username': 'cliusername',
'password': 'clipassword',
},
)
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
database_config = [
{
'name': 'foo',
'username': 'root',
'password': 'trustsome1',
'hostname': 'dbhost',
'port': 'dbport',
'restore_username': 'restoreuser',
'restore_password': 'restorepass',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'mysql',
'--batch',
'--host',
'restorehost',
'--port',
'restoreport',
'--protocol',
'tcp',
'--user',
'restoreuser',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'MYSQL_PWD': 'restorepass'},
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -474,5 +622,15 @@ def test_restore_database_dump_with_dry_run_skips_restore():
flexmock(module).should_receive('execute_command_with_processes').never()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=True,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)

298
tests/unit/hooks/test_postgresql.py
View file

@ -479,7 +479,17 @@ def test_restore_database_dump_runs_pg_restore():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -494,7 +504,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
with pytest.raises(ValueError):
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=flexmock(),
connection_params={
'restore_hostname': None,
'restore_port': None,
'restore_username': None,
'restore_password': None,
},
)
@ -545,7 +565,17 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -594,7 +624,183 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_make_extra_environment_with_cli_password_sets_correct_password():
database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'}
extra = module.make_extra_environment(
database, restore_connection_params={'password': 'clipassword'}
)
assert extra['PGPASSWORD'] == 'clipassword'
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
database_config = [
{
'name': 'foo',
'hostname': 'database.example.org',
'port': 5433,
'username': 'postgres',
'password': 'trustsome1',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_username': 'restoreusername',
'restore_password': 'restorepassword',
'schemas': None,
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_extra_environment').and_return(
{'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}
)
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'pg_restore',
'--no-password',
'--if-exists',
'--exit-on-error',
'--clean',
'--dbname',
'foo',
'--host',
'clihost',
'--port',
'cliport',
'--username',
'cliusername',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
).once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
'--no-password',
'--no-psqlrc',
'--quiet',
'--host',
'clihost',
'--port',
'cliport',
'--username',
'cliusername',
'--dbname',
'foo',
'--command',
'ANALYZE',
),
extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': 'clihost',
'port': 'cliport',
'username': 'cliusername',
'password': 'clipassword',
},
)
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
database_config = [
{
'name': 'foo',
'hostname': 'database.example.org',
'port': 5433,
'username': 'postgres',
'password': 'trustsome1',
'schemas': None,
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_username': 'restoreusername',
'restore_password': 'restorepassword',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_extra_environment').and_return(
{'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}
)
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'pg_restore',
'--no-password',
'--if-exists',
'--exit-on-error',
'--clean',
'--dbname',
'foo',
'--host',
'restorehost',
'--port',
'restoreport',
'--username',
'restoreusername',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
).once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
'--no-password',
'--no-psqlrc',
'--quiet',
'--host',
'restorehost',
'--port',
'restoreport',
'--username',
'restoreusername',
'--dbname',
'foo',
'--command',
'ANALYZE',
),
extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
).once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -644,7 +850,17 @@ def test_restore_database_dump_runs_pg_restore_with_options():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -672,7 +888,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -705,7 +931,17 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -759,7 +995,17 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -772,7 +1018,17 @@ def test_restore_database_dump_with_dry_run_skips_restore():
flexmock(module).should_receive('execute_command_with_processes').never()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=True,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -813,7 +1069,17 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=None
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=None,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -858,5 +1124,15 @@ def test_restore_database_dump_with_schemas_restores_schemas():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=None
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=None,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)

88
tests/unit/hooks/test_sqlite.py
View file

@ -1,3 +1,4 @@
import logging
import pytest
from flexmock import flexmock
@ -94,12 +95,81 @@ def test_restore_database_dump_restores_database():
database_config = [{'path': '/path/to/database', 'name': 'database'}]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').once()
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'sqlite3',
'/path/to/database',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
flexmock(module.os).should_receive('remove').once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={'restore_path': None},
)
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
database_config = [
{'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'sqlite3',
'cli/path/to/database',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
flexmock(module.os).should_receive('remove').once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={'restore_path': 'cli/path/to/database'},
)
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
database_config = [
{'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'sqlite3',
'config/path/to/database',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
flexmock(module.os).should_receive('remove').once()
module.restore_database_dump(
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={'restore_path': None},
)
@ -111,7 +181,12 @@ def test_restore_database_dump_does_not_restore_database_if_dry_run():
flexmock(module.os).should_receive('remove').never()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=True,
extract_process=extract_process,
connection_params={'restore_path': None},
)
@ -121,5 +196,10 @@ def test_restore_database_dump_raises_error_if_database_config_is_invalid():
with pytest.raises(ValueError):
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={'restore_path': None},
)