psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add configuration options for database command customization (#630).

Browse source
This commit is contained in:
Dan Helfman 2023-01-26 14:59:17 -08:00
parent 113c0e7616
commit 30cca62d09
9 changed files with 192 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
NEWS
View file

@ -5,6 +5,9 @@
* #602: Fix logs that interfere with JSON output by making warnings go to stderr instead of stdout.
* #622: Fix traceback when include merging configuration files on ARM64.
* #629: Skip warning about excluded special files when no special files have been excluded.
* #630: Add configuration options for database command customization: "list_options",
"restore_options", and "analyze_options" for PostgreSQL, "restore_options" for MySQL, and
"restore_options" for MongoDB.
1.7.5
* #311: Override PostgreSQL dump/restore commands via configuration options.

58
borgmatic/config/schema.yaml
View file

@ -806,6 +806,30 @@ properties:
any validation on them. See pg_dump
documentation for details.
example: --role=someone
list_options:
type: string
description: |
Additional psql options to pass directly to the
psql command that lists available databases,
without performing any validation on them. See
psql documentation for details.
example: --role=someone
restore_options:
type: string
description: |
Additional pg_restore/psql options to pass
directly to the restore command, without
performing any validation on them. See
pg_restore/psql documentation for details.
example: --role=someone
analyze_options:
type: string
description: |
Additional psql options to pass directly to the
analyze command run after a restore, without
performing any validation on them. See psql
documentation for details.
example: --role=someone
description: |
List of one or more PostgreSQL databases to dump before
creating a backup, run once per configuration file. The
@ -868,14 +892,6 @@ properties:
file of that format, allowing more convenient
restores of individual databases.
example: directory
list_options:
type: string
description: |
Additional mysql options to pass directly to
the mysql command that lists available
databases, without performing any validation on
them. See mysql documentation for details.
example: --defaults-extra-file=my.cnf
options:
type: string
description: |
@ -884,6 +900,22 @@ properties:
validation on them. See mysqldump documentation
for details.
example: --skip-comments
list_options:
type: string
description: |
Additional mysql options to pass directly to
the mysql command that lists available
databases, without performing any validation on
them. See mysql documentation for details.
example: --defaults-extra-file=my.cnf
restore_options:
type: string
description: |
Additional mysql options to pass directly to
the mysql command that restores database dumps,
without performing any validation on them. See
mysql documentation for details.
example: --defaults-extra-file=my.cnf
description: |
List of one or more MySQL/MariaDB databases to dump before
creating a backup, run once per configuration file. The
@ -956,7 +988,15 @@ properties:
directly to the dump command, without performing
any validation on them. See mongodump
documentation for details.
example: --role=someone
example: --dumpDbUsersAndRoles
restore_options:
type: string
description: |
Additional mongorestore options to pass
directly to the dump command, without performing
any validation on them. See mongorestore
documentation for details.
example: --restoreDbUsersAndRoles
description: |
List of one or more MongoDB databases to dump before
creating a backup, run once per configuration file. The

2
borgmatic/hooks/mongodb.py
View file

@ -160,4 +160,6 @@ def build_restore_command(extract_process, database, dump_filename):
command.extend(('--password', database['password']))
if 'authentication_database' in database:
command.extend(('--authenticationDatabase', database['authentication_database']))
if 'restore_options' in database:
command.extend(database['restore_options'].split(' '))
return command

1
borgmatic/hooks/mysql.py
View file

@ -197,6 +197,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
database = database_config[0]
restore_command = (
('mysql', '--batch')
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())

4
borgmatic/hooks/postgresql.py
View file

@ -62,7 +62,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ())
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
)
logger.debug(
'{}: Querying for "all" PostgreSQL databases to dump{}'.format(log_prefix, dry_run_label)
@ -204,6 +204,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ())
+ (('--dbname', database['name']) if not all_databases else ())
+ (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
+ ('--command', 'ANALYZE')
)
pg_restore_command = database.get('pg_restore_command') or 'pg_restore'
@ -217,6 +218,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ())
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
+ (() if extract_process else (dump_filename,))
)
extra_environment = make_extra_environment(database)

5
docs/how-to/backup-your-databases.md
View file

@ -76,6 +76,11 @@ hooks:
options: "--ssl"
```
See your [borgmatic configuration
file](https://torsion.org/borgmatic/docs/reference/configuration/) for
additional customization of the options passed to database commands (when
listing databases, restoring databases, etc.).
### All databases

18
tests/unit/hooks/test_mongodb.py
View file

@ -256,6 +256,24 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
)
def test_restore_database_dump_runs_mongorestore_with_options():
database_config = [{'name': 'foo', 'restore_options': '--harder',}]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
['mongorestore', '--archive', '--drop', '--db', 'foo', '--harder',],
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
)
def test_restore_database_dump_runs_psql_for_all_database_dump():
database_config = [{'name': 'all'}]
extract_process = flexmock(stdout=flexmock())

17
tests/unit/hooks/test_mysql.py
View file

@ -336,6 +336,23 @@ def test_restore_database_dump_errors_on_multiple_database_config():
)
def test_restore_database_dump_runs_mysql_with_options():
database_config = [{'name': 'foo', 'restore_options': '--harder'}]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
('mysql', '--batch', '--harder'),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=None,
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
)
def test_restore_database_dump_runs_mysql_with_hostname_and_port():
database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
extract_process = flexmock(stdout=flexmock())

95
tests/unit/hooks/test_postgresql.py
View file

@ -36,6 +36,55 @@ def test_database_names_to_dump_with_all_and_format_lists_databases():
)
def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port():
database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234}
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
(
'psql',
'--list',
'--no-password',
'--csv',
'--tuples-only',
'--host',
'localhost',
'--port',
'1234',
),
extra_environment=object,
).and_return('foo,test,\nbar,test,"stuff and such"')
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
'foo',
'bar',
)
def test_database_names_to_dump_with_all_and_format_lists_databases_with_username():
database = {'name': 'all', 'format': 'custom', 'username': 'postgres'}
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
('psql', '--list', '--no-password', '--csv', '--tuples-only', '--username', 'postgres'),
extra_environment=object,
).and_return('foo,test,\nbar,test,"stuff and such"')
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
'foo',
'bar',
)
def test_database_names_to_dump_with_all_and_format_lists_databases_with_options():
database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'}
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
('psql', '--list', '--no-password', '--csv', '--tuples-only', '--harder'),
extra_environment=object,
).and_return('foo,test,\nbar,test,"stuff and such"')
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
'foo',
'bar',
)
def test_database_names_to_dump_with_all_and_format_excludes_particular_databases():
database = {'name': 'all', 'format': 'custom'}
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
@ -90,7 +139,7 @@ def test_dump_databases_raises_when_no_database_names_to_dump():
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
def test_dump_databases_with_dupliate_dump_skips_pg_dump():
def test_dump_databases_with_duplicate_dump_skips_pg_dump():
databases = [{'name': 'foo'}, {'name': 'bar'}]
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
flexmock(module).should_receive('make_dump_path').and_return('')
@ -480,6 +529,50 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
)
def test_restore_database_dump_runs_pg_restore_with_options():
database_config = [
{'name': 'foo', 'restore_options': '--harder', 'analyze_options': '--smarter'}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'pg_restore',
'--no-password',
'--if-exists',
'--exit-on-error',
'--clean',
'--dbname',
'foo',
'--harder',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'PGSSLMODE': 'disable'},
).once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
'--no-password',
'--quiet',
'--dbname',
'foo',
'--smarter',
'--command',
'ANALYZE',
),
extra_environment={'PGSSLMODE': 'disable'},
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
)
def test_restore_database_dump_runs_psql_for_all_database_dump():
database_config = [{'name': 'all'}]
extract_process = flexmock(stdout=flexmock())