psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fork a MariaDB database hook from the MySQL database hook (#727).

Browse source
This commit is contained in:
Dan Helfman 2023-08-04 13:22:44 -07:00
parent 8a94b9e2f1
commit 193dd93de2
13 changed files with 1150 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
.drone.yml
View file

@ -16,16 +16,16 @@ services:
POSTGRES_USER: postgres2
commands:
- docker-entrypoint.sh -p 5433
- name: mysql
image: docker.io/mariadb:10.5
- name: mariadb
image: docker.io/mariadb:10.11.4
environment:
MYSQL_ROOT_PASSWORD: test
MYSQL_DATABASE: test
- name: mysql2
image: docker.io/mariadb:10.5
MARIADB_ROOT_PASSWORD: test
MARIADB_DATABASE: test
- name: mariadb2
image: docker.io/mariadb:10.11.4
environment:
MYSQL_ROOT_PASSWORD: test2
MYSQL_DATABASE: test
MARIADB_ROOT_PASSWORD: test2
MARIADB_DATABASE: test
commands:
- docker-entrypoint.sh --port=3307
- name: mongodb

5
NEWS
View file

@ -1,3 +1,8 @@
1.8.2.dev0
* #727: Add a MariaDB database hook that uses native MariaDB commands instead of the deprecated
MySQL ones. Be aware though that any existing backups made with the "mysql_databases:" hook are
only restorable with a "mysql_databases:" configuration.
1.8.1
* #326: Add documentation for restoring a database to an alternate host:
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-to-an-alternate-host

145
borgmatic/config/schema.yaml
View file

@ -841,10 +841,121 @@ properties:
description: |
List of one or more PostgreSQL databases to dump before creating a
backup, run once per configuration file. The database dumps are
added to your source directories at runtime, backed up, and removed
afterwards. Requires pg_dump/pg_dumpall/pg_restore commands. See
added to your source directories at runtime and streamed directly
to Borg. Requires pg_dump/pg_dumpall/pg_restore commands. See
https://www.postgresql.org/docs/current/app-pgdump.html and
https://www.postgresql.org/docs/current/libpq-ssl.html for details.
https://www.postgresql.org/docs/current/libpq-ssl.html for
details.
mariadb_databases:
type: array
items:
type: object
required: ['name']
additionalProperties: false
properties:
name:
type: string
description: |
Database name (required if using this hook). Or "all" to
dump all databases on the host. Note that using this
database hook implicitly enables both read_special and
one_file_system (see above) to support dump and restore
streaming.
example: users
hostname:
type: string
description: |
Database hostname to connect to. Defaults to connecting
via local Unix socket.
example: database.example.org
restore_hostname:
type: string
description: |
Database hostname to restore to. Defaults to the
"hostname" option.
example: database.example.org
port:
type: integer
description: Port to connect to. Defaults to 3306.
example: 3307
restore_port:
type: integer
description: |
Port to restore to. Defaults to the "port" option.
example: 5433
username:
type: string
description: |
Username with which to connect to the database. Defaults
to the username of the current user.
example: dbuser
restore_username:
type: string
description: |
Username with which to restore the database. Defaults to
the "username" option.
example: dbuser
password:
type: string
description: |
Password with which to connect to the database. Omitting
a password will only work if MariaDB is configured to
trust the configured username without a password.
example: trustsome1
restore_password:
type: string
description: |
Password with which to connect to the restore database.
Defaults to the "password" option.
example: trustsome1
format:
type: string
enum: ['sql']
description: |
Database dump output format. Currently only "sql" is
supported. Defaults to "sql" for a single database. Or,
when database name is "all" and format is blank, dumps
all databases to a single file. But if a format is
specified with an "all" database name, dumps each
database to a separate file of that format, allowing
more convenient restores of individual databases.
example: directory
add_drop_database:
type: boolean
description: |
Use the "--add-drop-database" flag with mariadb-dump,
causing the database to be dropped right before restore.
Defaults to true.
example: false
options:
type: string
description: |
Additional mariadb-dump options to pass directly to the
dump command, without performing any validation on them.
See mariadb-dump documentation for details.
example: --skip-comments
list_options:
type: string
description: |
Additional options to pass directly to the mariadb
command that lists available databases, without
performing any validation on them. See mariadb command
documentation for details.
example: --defaults-extra-file=mariadb.cnf
restore_options:
type: string
description: |
Additional options to pass directly to the mariadb
command that restores database dumps, without
performing any validation on them. See mariadb command
documentation for details.
example: --defaults-extra-file=mariadb.cnf
description: |
List of one or more MariaDB databases to dump before creating a
backup, run once per configuration file. The database dumps are
added to your source directories at runtime and streamed directly
to Borg. Requires mariadb-dump/mariadb commands. See
https://mariadb.com/kb/en/library/mysqldump/ for details.
mysql_databases:
type: array
items:
@ -936,26 +1047,26 @@ properties:
list_options:
type: string
description: |
Additional mysql options to pass directly to the mysql
Additional options to pass directly to the mysql
command that lists available databases, without
performing any validation on them. See mysql
performing any validation on them. See mysql command
documentation for details.
example: --defaults-extra-file=my.cnf
restore_options:
type: string
description: |
Additional mysql options to pass directly to the mysql
command that restores database dumps, without performing
any validation on them. See mysql documentation for
details.
Additional options to pass directly to the mysql
command that restores database dumps, without
performing any validation on them. See mysql command
documentation for details.
example: --defaults-extra-file=my.cnf
description: |
List of one or more MySQL/MariaDB databases to dump before creating
a backup, run once per configuration file. The database dumps are
added to your source directories at runtime, backed up, and removed
afterwards. Requires mysqldump/mysql commands (from either MySQL or
MariaDB). See https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html
or https://mariadb.com/kb/en/library/mysqldump/ for details.
List of one or more MySQL databases to dump before creating a
backup, run once per configuration file. The database dumps are
added to your source directories at runtime and streamed directly
to Borg. Requires mysqldump/mysql commands. See
https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html for
details.
sqlite_databases:
type: array
items:
@ -1080,8 +1191,8 @@ properties:
description: |
List of one or more MongoDB databases to dump before creating a
backup, run once per configuration file. The database dumps are
added to your source directories at runtime, backed up, and removed
afterwards. Requires mongodump/mongorestore commands. See
added to your source directories at runtime and streamed directly
to Borg. Requires mongodump/mongorestore commands. See
https://docs.mongodb.com/database-tools/mongodump/ and
https://docs.mongodb.com/database-tools/mongorestore/ for details.
ntfy:

2
borgmatic/hooks/dispatch.py
View file

@ -4,6 +4,7 @@ from borgmatic.hooks import (
cronhub,
cronitor,
healthchecks,
mariadb,
mongodb,
mysql,
ntfy,
@ -18,6 +19,7 @@ HOOK_NAME_TO_MODULE = {
'cronhub': cronhub,
'cronitor': cronitor,
'healthchecks': healthchecks,
'mariadb_databases': mariadb,
'mongodb_databases': mongodb,
'mysql_databases': mysql,
'ntfy': ntfy,

3
borgmatic/hooks/dump.py
View file

@ -7,9 +7,10 @@ from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
logger = logging.getLogger(__name__)
DATABASE_HOOK_NAMES = (
'postgresql_databases',
'mariadb_databases',
'mysql_databases',
'mongodb_databases',
'postgresql_databases',
'sqlite_databases',
)

242
borgmatic/hooks/mariadb.py Normal file
View file

@ -0,0 +1,242 @@
import copy
import logging
import os
from borgmatic.execute import (
execute_command,
execute_command_and_capture_output,
execute_command_with_processes,
)
from borgmatic.hooks import dump
logger = logging.getLogger(__name__)
def make_dump_path(config): # pragma: no cover
'''
Make the dump path from the given configuration dict and the name of this hook.
'''
return dump.make_database_dump_path(
config.get('borgmatic_source_directory'), 'mariadb_databases'
)
SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
'''
Given a requested database config, return the corresponding sequence of database names to dump.
In the case of "all", query for the names of databases on the configured host and return them,
excluding any system databases that will cause problems during restore.
'''
if database['name'] != 'all':
return (database['name'],)
if dry_run:
return ()
show_command = (
('mariadb',)
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--user', database['username']) if 'username' in database else ())
+ ('--skip-column-names', '--batch')
+ ('--execute', 'show schemas')
)
logger.debug(f'{log_prefix}: Querying for "all" MariaDB databases to dump')
show_output = execute_command_and_capture_output(
show_command, extra_environment=extra_environment
)
return tuple(
show_name
for show_name in show_output.strip().splitlines()
if show_name not in SYSTEM_DATABASE_NAMES
)
def execute_dump_command(
database, log_prefix, dump_path, database_names, extra_environment, dry_run, dry_run_label
):
'''
Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named
pipe constructed from the given dump path and database names. Use the given log prefix in any
log entries.
Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if
this is a dry run, then don't actually dump anything and return None.
'''
database_name = database['name']
dump_filename = dump.make_database_dump_filename(
dump_path, database['name'], database.get('hostname')
)
if os.path.exists(dump_filename):
logger.warning(
f'{log_prefix}: Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}'
)
return None
dump_command = (
('mariadb-dump',)
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--user', database['username']) if 'username' in database else ())
+ ('--databases',)
+ database_names
+ ('--result-file', dump_filename)
)
logger.debug(
f'{log_prefix}: Dumping MariaDB database "{database_name}" to {dump_filename}{dry_run_label}'
)
if dry_run:
return None
dump.create_named_pipe_for_dump(dump_filename)
return execute_command(
dump_command,
extra_environment=extra_environment,
run_to_completion=False,
)
def dump_databases(databases, config, log_prefix, dry_run):
'''
Dump the given MariaDB databases to a named pipe. The databases are supplied as a sequence of
dicts, one dict describing each database as per the configuration schema. Use the given
configuration dict to construct the destination path and the given log prefix in any log
entries.
Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
'''
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
processes = []
logger.info(f'{log_prefix}: Dumping MariaDB databases{dry_run_label}')
for database in databases:
dump_path = make_dump_path(config)
extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
dump_database_names = database_names_to_dump(
database, extra_environment, log_prefix, dry_run
)
if not dump_database_names:
if dry_run:
continue
raise ValueError('Cannot find any MariaDB databases to dump.')
if database['name'] == 'all' and database.get('format'):
for dump_name in dump_database_names:
renamed_database = copy.copy(database)
renamed_database['name'] = dump_name
processes.append(
execute_dump_command(
renamed_database,
log_prefix,
dump_path,
(dump_name,),
extra_environment,
dry_run,
dry_run_label,
)
)
else:
processes.append(
execute_dump_command(
database,
log_prefix,
dump_path,
dump_database_names,
extra_environment,
dry_run,
dry_run_label,
)
)
return [process for process in processes if process]
def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
'''
Remove all database dump files for this hook regardless of the given databases. Use the given
configuration dict to construct the destination path and the log prefix in any log entries. If
this is a dry run, then don't actually remove anything.
'''
dump.remove_database_dumps(make_dump_path(config), 'MariaDB', log_prefix, dry_run)
def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
'''
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
database name to match, return the corresponding glob patterns to match the database dump in an
archive.
'''
return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*')
def restore_database_dump(
databases_config, config, log_prefix, database_name, dry_run, extract_process, connection_params
):
'''
Restore the given MariaDB database from an extract stream. The databases are supplied as a
sequence containing one dict describing each database (as per the configuration schema), but
only the database corresponding to the given database name is restored. Use the given log prefix
in any log entries. If this is a dry run, then don't actually restore anything. Trigger the
given active extract process (an instance of subprocess.Popen) to produce output to consume.
'''
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
try:
database = next(
database_config
for database_config in databases_config
if database_config.get('name') == database_name
)
except StopIteration:
raise ValueError(
f'A database named "{database_name}" could not be found in the configuration'
)
hostname = connection_params['hostname'] or database.get(
'restore_hostname', database.get('hostname')
)
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
username = connection_params['username'] or database.get(
'restore_username', database.get('username')
)
password = connection_params['password'] or database.get(
'restore_password', database.get('password')
)
restore_command = (
('mariadb', '--batch')
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
+ (('--host', hostname) if hostname else ())
+ (('--port', str(port)) if port else ())
+ (('--protocol', 'tcp') if hostname or port else ())
+ (('--user', username) if username else ())
)
extra_environment = {'MYSQL_PWD': password} if password else None
logger.debug(f"{log_prefix}: Restoring MariaDB database {database['name']}{dry_run_label}")
if dry_run:
return
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
# if the restore paths don't exist in the archive.
execute_command_with_processes(
restore_command,
[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=extra_environment,
)

37
borgmatic/hooks/mongodb.py
View file

@ -59,26 +59,23 @@ def build_dump_command(database, dump_filename, dump_format):
Return the mongodump command from a single database configuration.
'''
all_databases = database['name'] == 'all'
command = ['mongodump']
if dump_format == 'directory':
command.extend(('--out', dump_filename))
if 'hostname' in database:
command.extend(('--host', database['hostname']))
if 'port' in database:
command.extend(('--port', str(database['port'])))
if 'username' in database:
command.extend(('--username', database['username']))
if 'password' in database:
command.extend(('--password', database['password']))
if 'authentication_database' in database:
command.extend(('--authenticationDatabase', database['authentication_database']))
if not all_databases:
command.extend(('--db', database['name']))
if 'options' in database:
command.extend(database['options'].split(' '))
if dump_format != 'directory':
command.extend(('--archive', '>', dump_filename))
return command
return (
('mongodump',)
+ (('--out', dump_filename) if dump_format == 'directory' else ())
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ())
+ (('--password', database['password']) if 'password' in database else ())
+ (
('--authenticationDatabase', database['authentication_database'])
if 'authentication_database' in database
else ()
)
+ (('--db', database['name']) if not all_databases else ())
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
+ (('--archive', '>', dump_filename) if dump_format != 'directory' else ())
)
def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover

71
docs/how-to/backup-your-databases.md
View file

@ -15,7 +15,7 @@ consistent snapshot that is more suited for backups.
Fortunately, borgmatic includes built-in support for creating database dumps
prior to running backups. For example, here is everything you need to dump and
backup a couple of local PostgreSQL databases and a MySQL/MariaDB database.
backup a couple of local PostgreSQL databases and a MySQL database.
```yaml
postgresql_databases:
@ -46,6 +46,16 @@ sqlite_databases:
path: /var/lib/sqlite3/mydb.sqlite
```
<span class="minilink minilink-addedin">New in version 1.8.2</span> If you're
using MariaDB, use the MariaDB database hook instead of `mysql_databases:` as
the MariaDB hook calls native MariaDB commands instead of the deprecated MySQL
ones. For instance:
```yaml
mariadb_databases:
- name: comments
```
As part of each backup, borgmatic streams a database dump for each configured
database directly to Borg, so it's included in the backup without consuming
additional disk space. (The exceptions are the PostgreSQL/MongoDB "directory"
@ -75,16 +85,23 @@ postgresql_databases:
password: trustsome1
format: tar
options: "--role=someone"
mariadb_databases:
- name: photos
hostname: database3.example.org
port: 3307
username: root
password: trustsome1
options: "--skip-comments"
mysql_databases:
- name: posts
hostname: database3.example.org
hostname: database4.example.org
port: 3307
username: root
password: trustsome1
options: "--skip-comments"
mongodb_databases:
- name: messages
hostname: database4.example.org
hostname: database5.example.org
port: 27018
username: dbuser
password: trustsome1
@ -108,6 +125,8 @@ If you want to dump all databases on a host, use `all` for the database name:
```yaml
postgresql_databases:
- name: all
mariadb_databases:
- name: all
mysql_databases:
- name: all
mongodb_databases:
@ -123,15 +142,18 @@ The SQLite hook in particular does not consider "all" a special database name.
these options in the `hooks:` section of your configuration.
<span class="minilink minilink-addedin">New in version 1.7.6</span> With
PostgreSQL and MySQL, you can optionally dump "all" databases to separate
files instead of one combined dump file, allowing more convenient restores of
individual databases. Enable this by specifying your desired database dump
`format`:
PostgreSQL, MariaDB, and MySQL, you can optionally dump "all" databases to
separate files instead of one combined dump file, allowing more convenient
restores of individual databases. Enable this by specifying your desired
database dump `format`:
```yaml
postgresql_databases:
- name: all
format: custom
mariadb_databases:
- name: all
format: sql
mysql_databases:
- name: all
format: sql
@ -222,10 +244,16 @@ to prepare for this situation, it's a good idea to include borgmatic's own
configuration files as part of your regular backups. That way, you can always
bring back any missing configuration files in order to restore a database.
<span class="minilink minilink-addedin">New in version 1.7.15</span> borgmatic
automatically includes configuration files in your backup. See [the
documentation on the `config bootstrap`
action](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive)
for more information.
## Supported databases
As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, MongoDB, and SQLite
As of now, borgmatic supports PostgreSQL, MariaDB, MySQL, MongoDB, and SQLite
databases directly. But see below about general-purpose preparation and
cleanup hooks as a work-around with other database systems. Also, please [file
a ticket](https://torsion.org/borgmatic/#issues) for additional database
@ -420,9 +448,9 @@ dumps with any database system.
## Troubleshooting
### PostgreSQL/MySQL authentication errors
### Authentication errors
With PostgreSQL and MySQL/MariaDB, if you're getting authentication errors
With PostgreSQL, MariaDB, and MySQL, if you're getting authentication errors
when borgmatic tries to connect to your database, a natural reaction is to
increase your borgmatic verbosity with `--verbosity 2` and go looking in the
logs. You'll notice though that your database password does not show up in the
@ -436,23 +464,24 @@ authenticated. For instance, with PostgreSQL, check your
[pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html)
file for that configuration.
Additionally, MySQL/MariaDB may be picking up some of your credentials from a
defaults file like `~/.my.cnf`. If that's the case, then it's possible
MySQL/MariaDB ends up using, say, a username from borgmatic's configuration
and a password from `~/.my.cnf`. This may result in authentication errors if
this combination of credentials is not what you intend.
Additionally, MariaDB or MySQL may be picking up some of your credentials from
a defaults file like `~/mariadb.cnf` or `~/.my.cnf`. If that's the case, then
it's possible MariaDB or MySQL end up using, say, a username from borgmatic's
configuration and a password from `~/mariadb.cnf` or `~/.my.cnf`. This may
result in authentication errors if this combination of credentials is not what
you intend.
### MySQL table lock errors
### MariaDB or MySQL table lock errors
If you encounter table lock errors during a database dump with MySQL/MariaDB,
you may need to [use a
transaction](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html#option_mysqldump_single-transaction).
If you encounter table lock errors during a database dump with MariaDB or
MySQL, you may need to [use a
transaction](https://mariadb.com/docs/skysql-dbaas/ref/mdb/cli/mariadb-dump/single-transaction/).
You can add any additional flags to the `options:` in your database
configuration. Here's an example:
configuration. Here's an example with MariaDB:
```yaml
mysql_databases:
mariadb_databases:
- name: posts
options: "--single-transaction --quick"
```

2
setup.py
View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup
VERSION = '1.8.1'
VERSION = '1.8.2.dev0'
setup(

20
tests/end-to-end/docker-compose.yaml
View file

@ -12,16 +12,16 @@ services:
POSTGRES_DB: test
POSTGRES_USER: postgres2
command: docker-entrypoint.sh -p 5433
mysql:
image: docker.io/mariadb:10.5
mariadb:
image: docker.io/mariadb:10.11.4
environment:
MYSQL_ROOT_PASSWORD: test
MYSQL_DATABASE: test
mysql2:
image: docker.io/mariadb:10.5
MARIADB_ROOT_PASSWORD: test
MARIADB_DATABASE: test
mariadb2:
image: docker.io/mariadb:10.11.4
environment:
MYSQL_ROOT_PASSWORD: test2
MYSQL_DATABASE: test
MARIADB_ROOT_PASSWORD: test2
MARIADB_DATABASE: test
command: docker-entrypoint.sh --port=3307
mongodb:
image: docker.io/mongo:5.0.5
@ -50,7 +50,7 @@ services:
depends_on:
- postgresql
- postgresql2
- mysql
- mysql2
- mariadb
- mariadb2
- mongodb
- mongodb2

37
tests/end-to-end/test_database.py
View file

@ -45,18 +45,32 @@ postgresql_databases:
hostname: postgresql
username: postgres
password: test
mysql_databases:
mariadb_databases:
- name: test
hostname: mysql
hostname: mariadb
username: root
password: test
- name: all
hostname: mysql
hostname: mariadb
username: root
password: test
- name: all
format: sql
hostname: mysql
hostname: mariadb
username: root
password: test
mysql_databases:
- name: test
hostname: mariadb
username: root
password: test
- name: all
hostname: mariadb
username: root
password: test
- name: all
format: sql
hostname: mariadb
username: root
password: test
mongodb_databases:
@ -111,12 +125,21 @@ postgresql_databases:
restore_port: 5433
restore_username: postgres2
restore_password: test2
mysql_databases:
mariadb_databases:
- name: test
hostname: mysql
hostname: mariadb
username: root
password: test
restore_hostname: mysql2
restore_hostname: mariadb2
restore_port: 3307
restore_username: root
restore_password: test2
mysql_databases:
- name: test
hostname: mariadb
username: root
password: test
restore_hostname: mariadb2
restore_port: 3307
restore_username: root
restore_password: test2

644
tests/unit/hooks/test_mariadb.py Normal file
View file

@ -0,0 +1,644 @@
import logging
import pytest
from flexmock import flexmock
from borgmatic.hooks import mariadb as module
def test_database_names_to_dump_passes_through_name():
extra_environment = flexmock()
log_prefix = ''
names = module.database_names_to_dump(
{'name': 'foo'}, extra_environment, log_prefix, dry_run=False
)
assert names == ('foo',)
def test_database_names_to_dump_bails_for_dry_run():
extra_environment = flexmock()
log_prefix = ''
flexmock(module).should_receive('execute_command_and_capture_output').never()
names = module.database_names_to_dump(
{'name': 'all'}, extra_environment, log_prefix, dry_run=True
)
assert names == ()
def test_database_names_to_dump_queries_mariadb_for_database_names():
extra_environment = flexmock()
log_prefix = ''
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
('mariadb', '--skip-column-names', '--batch', '--execute', 'show schemas'),
extra_environment=extra_environment,
).and_return('foo\nbar\nmysql\n').once()
names = module.database_names_to_dump(
{'name': 'all'}, extra_environment, log_prefix, dry_run=False
)
assert names == ('foo', 'bar')
def test_dump_databases_dumps_each_database():
databases = [{'name': 'foo'}, {'name': 'bar'}]
processes = [flexmock(), flexmock()]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
('bar',)
)
for name, process in zip(('foo', 'bar'), processes):
flexmock(module).should_receive('execute_dump_command').with_args(
database={'name': name},
log_prefix=object,
dump_path=object,
database_names=(name,),
extra_environment=object,
dry_run=object,
dry_run_label=object,
).and_return(process).once()
assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes
def test_dump_databases_dumps_with_password():
database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'}
process = flexmock()
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
('bar',)
)
flexmock(module).should_receive('execute_dump_command').with_args(
database=database,
log_prefix=object,
dump_path=object,
database_names=('foo',),
extra_environment={'MYSQL_PWD': 'trustsome1'},
dry_run=object,
dry_run_label=object,
).and_return(process).once()
assert module.dump_databases([database], {}, 'test.yaml', dry_run=False) == [process]
def test_dump_databases_dumps_all_databases_at_once():
databases = [{'name': 'all'}]
process = flexmock()
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar'))
flexmock(module).should_receive('execute_dump_command').with_args(
database={'name': 'all'},
log_prefix=object,
dump_path=object,
database_names=('foo', 'bar'),
extra_environment=object,
dry_run=object,
dry_run_label=object,
).and_return(process).once()
assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process]
def test_dump_databases_dumps_all_databases_separately_when_format_configured():
databases = [{'name': 'all', 'format': 'sql'}]
processes = [flexmock(), flexmock()]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar'))
for name, process in zip(('foo', 'bar'), processes):
flexmock(module).should_receive('execute_dump_command').with_args(
database={'name': name, 'format': 'sql'},
log_prefix=object,
dump_path=object,
database_names=(name,),
extra_environment=object,
dry_run=object,
dry_run_label=object,
).and_return(process).once()
assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes
def test_database_names_to_dump_runs_mariadb_with_list_options():
database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf'}
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
(
'mariadb',
'--defaults-extra-file=mariadb.cnf',
'--skip-column-names',
'--batch',
'--execute',
'show schemas',
),
extra_environment=None,
).and_return(('foo\nbar')).once()
assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
def test_execute_dump_command_runs_mariadb_dump():
process = flexmock()
flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'mariadb-dump',
'--add-drop-database',
'--databases',
'foo',
'--result-file',
'dump',
),
extra_environment=None,
run_to_completion=False,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'foo'},
log_prefix='log',
dump_path=flexmock(),
database_names=('foo',),
extra_environment=None,
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database():
process = flexmock()
flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'mariadb-dump',
'--databases',
'foo',
'--result-file',
'dump',
),
extra_environment=None,
run_to_completion=False,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'foo', 'add_drop_database': False},
log_prefix='log',
dump_path=flexmock(),
database_names=('foo',),
extra_environment=None,
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port():
process = flexmock()
flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'mariadb-dump',
'--add-drop-database',
'--host',
'database.example.org',
'--port',
'5433',
'--protocol',
'tcp',
'--databases',
'foo',
'--result-file',
'dump',
),
extra_environment=None,
run_to_completion=False,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433},
log_prefix='log',
dump_path=flexmock(),
database_names=('foo',),
extra_environment=None,
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_runs_mariadb_dump_with_username_and_password():
process = flexmock()
flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'mariadb-dump',
'--add-drop-database',
'--user',
'root',
'--databases',
'foo',
'--result-file',
'dump',
),
extra_environment={'MYSQL_PWD': 'trustsome1'},
run_to_completion=False,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'},
log_prefix='log',
dump_path=flexmock(),
database_names=('foo',),
extra_environment={'MYSQL_PWD': 'trustsome1'},
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_runs_mariadb_dump_with_options():
process = flexmock()
flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'mariadb-dump',
'--stuff=such',
'--add-drop-database',
'--databases',
'foo',
'--result-file',
'dump',
),
extra_environment=None,
run_to_completion=False,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'foo', 'options': '--stuff=such'},
log_prefix='log',
dump_path=flexmock(),
database_names=('foo',),
extra_environment=None,
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump():
flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
flexmock(module).should_receive('execute_command').never()
assert (
module.execute_dump_command(
database={'name': 'foo'},
log_prefix='log',
dump_path=flexmock(),
database_names=('foo',),
extra_environment=None,
dry_run=True,
dry_run_label='SO DRY',
)
is None
)
def test_execute_dump_command_with_dry_run_skips_mariadb_dump():
flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').never()
assert (
module.execute_dump_command(
database={'name': 'foo'},
log_prefix='log',
dump_path=flexmock(),
database_names=('foo',),
extra_environment=None,
dry_run=True,
dry_run_label='SO DRY',
)
is None
)
def test_dump_databases_errors_for_missing_all_databases():
databases = [{'name': 'all'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
'databases/localhost/all'
)
flexmock(module).should_receive('database_names_to_dump').and_return(())
with pytest.raises(ValueError):
assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False)
def test_dump_databases_does_not_error_for_missing_all_databases_with_dry_run():
databases = [{'name': 'all'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
'databases/localhost/all'
)
flexmock(module).should_receive('database_names_to_dump').and_return(())
assert module.dump_databases(databases, {}, 'test.yaml', dry_run=True) == []
def test_restore_database_dump_runs_mariadb_to_restore():
databases_config = [{'name': 'foo'}, {'name': 'bar'}]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
('mariadb', '--batch'),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=None,
).once()
module.restore_database_dump(
databases_config,
{},
'test.yaml',
database_name='foo',
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_restore_database_dump_errors_when_database_missing_from_configuration():
databases_config = [{'name': 'foo'}, {'name': 'bar'}]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').never()
with pytest.raises(ValueError):
module.restore_database_dump(
databases_config,
{},
'test.yaml',
database_name='other',
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_restore_database_dump_runs_mariadb_with_options():
databases_config = [{'name': 'foo', 'restore_options': '--harder'}]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
('mariadb', '--batch', '--harder'),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=None,
).once()
module.restore_database_dump(
databases_config,
{},
'test.yaml',
database_name='foo',
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_restore_database_dump_runs_mariadb_with_hostname_and_port():
databases_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'mariadb',
'--batch',
'--host',
'database.example.org',
'--port',
'5433',
'--protocol',
'tcp',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=None,
).once()
module.restore_database_dump(
databases_config,
{},
'test.yaml',
database_name='foo',
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_restore_database_dump_runs_mariadb_with_username_and_password():
databases_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
('mariadb', '--batch', '--user', 'root'),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'MYSQL_PWD': 'trustsome1'},
).once()
module.restore_database_dump(
databases_config,
{},
'test.yaml',
database_name='foo',
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
databases_config = [
{
'name': 'foo',
'username': 'root',
'password': 'trustsome1',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_username': 'restoreusername',
'restore_password': 'restorepassword',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'mariadb',
'--batch',
'--host',
'clihost',
'--port',
'cliport',
'--protocol',
'tcp',
'--user',
'cliusername',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'MYSQL_PWD': 'clipassword'},
).once()
module.restore_database_dump(
databases_config,
{},
'test.yaml',
database_name='foo',
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': 'clihost',
'port': 'cliport',
'username': 'cliusername',
'password': 'clipassword',
},
)
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
databases_config = [
{
'name': 'foo',
'username': 'root',
'password': 'trustsome1',
'hostname': 'dbhost',
'port': 'dbport',
'restore_username': 'restoreuser',
'restore_password': 'restorepass',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'mariadb',
'--batch',
'--host',
'restorehost',
'--port',
'restoreport',
'--protocol',
'tcp',
'--user',
'restoreuser',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'MYSQL_PWD': 'restorepass'},
).once()
module.restore_database_dump(
databases_config,
{},
'test.yaml',
database_name='foo',
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)
def test_restore_database_dump_with_dry_run_skips_restore():
databases_config = [{'name': 'foo'}]
flexmock(module).should_receive('execute_command_with_processes').never()
module.restore_database_dump(
databases_config,
{},
'test.yaml',
database_name='foo',
dry_run=True,
extract_process=flexmock(),
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
)

16
tests/unit/hooks/test_mongodb.py
View file

@ -17,7 +17,7 @@ def test_dump_databases_runs_mongodump_for_each_database():
for name, process in zip(('foo', 'bar'), processes):
flexmock(module).should_receive('execute_command').with_args(
['mongodump', '--db', name, '--archive', '>', f'databases/localhost/{name}'],
('mongodump', '--db', name, '--archive', '>', f'databases/localhost/{name}'),
shell=True,
run_to_completion=False,
).and_return(process).once()
@ -47,7 +47,7 @@ def test_dump_databases_runs_mongodump_with_hostname_and_port():
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
[
(
'mongodump',
'--host',
'database.example.org',
@ -58,7 +58,7 @@ def test_dump_databases_runs_mongodump_with_hostname_and_port():
'--archive',
'>',
'databases/database.example.org/foo',
],
),
shell=True,
run_to_completion=False,
).and_return(process).once()
@ -83,7 +83,7 @@ def test_dump_databases_runs_mongodump_with_username_and_password():
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
[
(
'mongodump',
'--username',
'mongo',
@ -96,7 +96,7 @@ def test_dump_databases_runs_mongodump_with_username_and_password():
'--archive',
'>',
'databases/localhost/foo',
],
),
shell=True,
run_to_completion=False,
).and_return(process).once()
@ -114,7 +114,7 @@ def test_dump_databases_runs_mongodump_with_directory_format():
flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
flexmock(module).should_receive('execute_command').with_args(
['mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'],
('mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'),
shell=True,
).and_return(flexmock()).once()
@ -131,7 +131,7 @@ def test_dump_databases_runs_mongodump_with_options():
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
['mongodump', '--db', 'foo', '--stuff=such', '--archive', '>', 'databases/localhost/foo'],
('mongodump', '--db', 'foo', '--stuff=such', '--archive', '>', 'databases/localhost/foo'),
shell=True,
run_to_completion=False,
).and_return(process).once()
@ -149,7 +149,7 @@ def test_dump_databases_runs_mongodumpall_for_all_databases():
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
['mongodump', '--archive', '>', 'databases/localhost/all'],
('mongodump', '--archive', '>', 'databases/localhost/all'),
shell=True,
run_to_completion=False,
).and_return(process).once()