psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pass all existing tests (and formatting)

Browse source
This commit is contained in:
Divyansh Singh 2023-06-16 15:14:00 +05:30
parent 8389851f2f
commit 89602d1614
12 changed files with 411 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
borgmatic/actions/restore.py
View file

@ -304,13 +304,6 @@ def run_restore(
restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names)
found_names = set()
remaining_restore_names = {}
for hook_name, database_names in restore_names.items():
for database_name in database_names:
found_hook_name, found_database = get_configured_database(
hooks, archive_database_names, hook_name, database_name
)
connection_params = {
'hostname': restore_arguments.hostname,
'port': restore_arguments.port,
@ -319,6 +312,12 @@ def run_restore(
'restore_path': restore_arguments.restore_path,
}
for hook_name, database_names in restore_names.items():
for database_name in database_names:
found_hook_name, found_database = get_configured_database(
hooks, archive_database_names, hook_name, database_name
)
if not found_database:
remaining_restore_names.setdefault(found_hook_name or hook_name, []).append(
database_name
@ -368,6 +367,7 @@ def run_restore(
archive_name,
found_hook_name or hook_name,
dict(database, **{'schemas': restore_arguments.schemas}),
connection_params,
)
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(

3
borgmatic/commands/arguments.py
View file

@ -725,7 +725,8 @@ def make_parsers():
help='Database hostname to restore to. Defaults to the "restore_hostname" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--port', help='Port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration'
'--port',
help='Port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--username',

3
borgmatic/config/schema.yaml
View file

@ -816,7 +816,8 @@ properties:
OWNER or SET SESSION AUTHORIZATION statements
to set ownership of created schema elements.
These statements will fail unless the initial
connection to the database is made by a superuser.
connection to the database is made by a
superuser.
format:
type: string
enum: ['plain', 'custom', 'directory', 'tar']

20
borgmatic/hooks/mongodb.py
View file

@ -102,7 +102,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params):
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
'''
Restore the given MongoDB database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
@ -122,7 +124,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), database['name'], database.get('hostname')
)
restore_command = build_restore_command(extract_process, database, dump_filename, connection_params)
restore_command = build_restore_command(
extract_process, database, dump_filename, connection_params
)
logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}")
if dry_run:
@ -142,10 +146,16 @@ def build_restore_command(extract_process, database, dump_filename, connection_p
'''
Return the mongorestore command from a single database configuration.
'''
hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname'))
hostname = connection_params['hostname'] or database.get(
'restore_hostname', database.get('hostname')
)
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
username = connection_params['username'] or database.get('restore_username', database.get('username'))
password = connection_params['password'] or database.get('restore_password', database.get('password'))
username = connection_params['username'] or database.get(
'restore_username', database.get('username')
)
password = connection_params['password'] or database.get(
'restore_password', database.get('password')
)
command = ['mongorestore']
if extract_process:

16
borgmatic/hooks/mysql.py
View file

@ -185,7 +185,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params):
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
'''
Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
@ -200,10 +202,16 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
database = database_config[0]
hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname'))
hostname = connection_params['hostname'] or database.get(
'restore_hostname', database.get('hostname')
)
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
username = connection_params['username'] or database.get('restore_username', database.get('username'))
password = connection_params['password'] or database.get('restore_password', database.get('password'))
username = connection_params['username'] or database.get(
'restore_username', database.get('username')
)
password = connection_params['password'] or database.get(
'restore_password', database.get('password')
)
restore_command = (
('mysql', '--batch')

21
borgmatic/hooks/postgresql.py
View file

@ -22,6 +22,7 @@ def make_dump_path(location_config): # pragma: no cover
location_config.get('borgmatic_source_directory'), 'postgresql_databases'
)
def make_extra_environment(database, restore_connection_params=None):
'''
Make the extra_environment dict from the given database configuration.
@ -31,7 +32,9 @@ def make_extra_environment(database, restore_connection_params=None):
try:
if restore_connection_params:
extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get('restore_password', database['password'])
extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get(
'restore_password', database['password']
)
else:
extra['PGPASSWORD'] = database['password']
except (AttributeError, KeyError):
@ -200,7 +203,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params):
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
'''
Restore the given PostgreSQL database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
@ -221,9 +226,13 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
database = database_config[0]
hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname'))
hostname = connection_params['hostname'] or database.get(
'restore_hostname', database.get('hostname')
)
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
username = connection_params['username'] or database.get('restore_username', database.get('username'))
username = connection_params['username'] or database.get(
'restore_username', database.get('username')
)
all_databases = bool(database['name'] == 'all')
dump_filename = dump.make_database_dump_filename(
@ -260,7 +269,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
)
)
extra_environment = make_extra_environment(database, restore_connection_params=connection_params)
extra_environment = make_extra_environment(
database, restore_connection_params=connection_params
)
logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}")
if dry_run:

8
borgmatic/hooks/sqlite.py
View file

@ -85,7 +85,9 @@ def make_database_dump_pattern(
return dump.make_database_dump_filename(make_dump_path(location_config), name)
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params):
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
):
'''
Restore the given SQLite3 database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
@ -98,7 +100,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
if len(database_config) != 1:
raise ValueError('The database configuration value is invalid')
database_path = connection_params['restore_path'] or database_config[0].get('restore_path', database_config[0].get('path'))
database_path = connection_params['restore_path'] or database_config[0].get(
'restore_path', database_config[0].get('path')
)
logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
if dry_run:

48
tests/unit/actions/test_restore.py
View file

@ -241,6 +241,7 @@ def test_run_restore_restores_each_database():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
@ -254,6 +255,7 @@ def test_run_restore_restores_each_database():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'bar', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('ensure_databases_found')
@ -264,7 +266,15 @@ def test_run_restore_restores_each_database():
hooks=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None
repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),
@ -337,6 +347,7 @@ def test_run_restore_restores_database_configured_with_all_name():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
@ -350,6 +361,7 @@ def test_run_restore_restores_database_configured_with_all_name():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'bar', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('ensure_databases_found')
@ -360,7 +372,15 @@ def test_run_restore_restores_database_configured_with_all_name():
hooks=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None
repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),
@ -411,6 +431,7 @@ def test_run_restore_skips_missing_database():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
@ -424,6 +445,7 @@ def test_run_restore_skips_missing_database():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'bar', 'schemas': None},
connection_params=object,
).never()
flexmock(module).should_receive('ensure_databases_found')
@ -434,7 +456,15 @@ def test_run_restore_skips_missing_database():
hooks=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None
repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),
@ -479,6 +509,7 @@ def test_run_restore_restores_databases_from_different_hooks():
archive_name=object,
hook_name='postgresql_databases',
database={'name': 'foo', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
@ -492,6 +523,7 @@ def test_run_restore_restores_databases_from_different_hooks():
archive_name=object,
hook_name='mysql_databases',
database={'name': 'bar', 'schemas': None},
connection_params=object,
).once()
flexmock(module).should_receive('ensure_databases_found')
@ -502,7 +534,15 @@ def test_run_restore_restores_databases_from_different_hooks():
hooks=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo', archive='archive', databases=flexmock(), schemas=None
repository='repo',
archive='archive',
databases=flexmock(),
schemas=None,
hostname=None,
port=None,
username=None,
password=None,
restore_path=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),

108
tests/unit/hooks/test_mongodb.py
View file

@ -171,7 +171,17 @@ def test_restore_database_dump_runs_mongorestore():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -185,7 +195,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
with pytest.raises(ValueError):
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -215,7 +235,17 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -253,7 +283,17 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -271,7 +311,17 @@ def test_restore_database_dump_runs_mongorestore_with_options():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -299,7 +349,17 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -317,7 +377,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -329,7 +399,17 @@ def test_restore_database_dump_with_dry_run_skips_restore():
flexmock(module).should_receive('execute_command_with_processes').never()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=True,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -346,5 +426,15 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=None
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=None,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)

72
tests/unit/hooks/test_mysql.py
View file

@ -392,7 +392,17 @@ def test_restore_database_dump_runs_mysql_to_restore():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -404,7 +414,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
with pytest.raises(ValueError):
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -421,7 +441,17 @@ def test_restore_database_dump_runs_mysql_with_options():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -447,7 +477,17 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -464,7 +504,17 @@ def test_restore_database_dump_runs_mysql_with_username_and_password():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -474,5 +524,15 @@ def test_restore_database_dump_with_dry_run_skips_restore():
flexmock(module).should_receive('execute_command_with_processes').never()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=True,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)

132
tests/unit/hooks/test_postgresql.py
View file

@ -479,7 +479,17 @@ def test_restore_database_dump_runs_pg_restore():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -494,7 +504,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
with pytest.raises(ValueError):
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=flexmock(),
connection_params={
'restore_hostname': None,
'restore_port': None,
'restore_username': None,
'restore_password': None,
},
)
@ -545,7 +565,17 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -594,7 +624,17 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -644,7 +684,17 @@ def test_restore_database_dump_runs_pg_restore_with_options():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -672,7 +722,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -705,7 +765,17 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -759,7 +829,17 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -772,7 +852,17 @@ def test_restore_database_dump_with_dry_run_skips_restore():
flexmock(module).should_receive('execute_command_with_processes').never()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
database_config,
'test.yaml',
{},
dry_run=True,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -813,7 +903,17 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=None
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=None,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
@ -858,5 +958,15 @@ def test_restore_database_dump_with_schemas_restores_schemas():
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=None
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=None,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)

21
tests/unit/hooks/test_sqlite.py
View file

@ -99,7 +99,12 @@ def test_restore_database_dump_restores_database():
flexmock(module.os).should_receive('remove').once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={'restore_path': None},
)
@ -111,7 +116,12 @@ def test_restore_database_dump_does_not_restore_database_if_dry_run():
flexmock(module.os).should_receive('remove').never()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=True, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=True,
extract_process=extract_process,
connection_params={'restore_path': None},
)
@ -121,5 +131,10 @@ def test_restore_database_dump_raises_error_if_database_config_is_invalid():
with pytest.raises(ValueError):
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
database_config,
'test.yaml',
{},
dry_run=False,
extract_process=extract_process,
connection_params={'restore_path': None},
)