Add configuration options for database command customization (#630).

NEWS

@@ -5,6 +5,9 @@
  * #602: Fix logs that interfere with JSON output by making warnings go to stderr instead of stdout.
  * #622: Fix traceback when include merging configuration files on ARM64.
  * #629: Skip warning about excluded special files when no special files have been excluded.
+ * #630: Add configuration options for database command customization: "list_options",
+   "restore_options", and "analyze_options" for PostgreSQL, "restore_options" for MySQL, and
+   "restore_options" for MongoDB.
 
 1.7.5
  * #311: Override PostgreSQL dump/restore commands via configuration options.

borgmatic/config/schema.yaml

@@ -806,6 +806,30 @@ properties:
                                 any validation on them. See pg_dump
                                 documentation for details.
                             example: --role=someone
+                        list_options:
+                            type: string
+                            description: |
+                                Additional psql options to pass directly to the
+                                psql command that lists available databases,
+                                without performing any validation on them. See
+                                psql documentation for details.
+                            example: --role=someone
+                        restore_options:
+                            type: string
+                            description: |
+                                Additional pg_restore/psql options to pass
+                                directly to the restore command, without
+                                performing any validation on them. See
+                                pg_restore/psql documentation for details.
+                            example: --role=someone
+                        analyze_options:
+                            type: string
+                            description: |
+                                Additional psql options to pass directly to the
+                                analyze command run after a restore, without
+                                performing any validation on them. See psql
+                                documentation for details.
+                            example: --role=someone
                 description: |
                     List of one or more PostgreSQL databases to dump before
                     creating a backup, run once per configuration file. The
@@ -868,14 +892,6 @@ properties:
                                 file of that format, allowing more convenient
                                 restores of individual databases.
                             example: directory
-                        list_options:
-                            type: string
-                            description: |
-                                Additional mysql options to pass directly to
-                                the mysql command that lists available
-                                databases, without performing any validation on
-                                them. See mysql documentation for details.
-                            example: --defaults-extra-file=my.cnf
                         options:
                             type: string
                             description: |
@@ -884,6 +900,22 @@ properties:
                                 validation on them. See mysqldump documentation
                                 for details.
                             example: --skip-comments
+                        list_options:
+                            type: string
+                            description: |
+                                Additional mysql options to pass directly to
+                                the mysql command that lists available
+                                databases, without performing any validation on
+                                them. See mysql documentation for details.
+                            example: --defaults-extra-file=my.cnf
+                        restore_options:
+                            type: string
+                            description: |
+                                Additional mysql options to pass directly to
+                                the mysql command that restores database dumps,
+                                without performing any validation on them. See
+                                mysql documentation for details.
+                            example: --defaults-extra-file=my.cnf
                 description: |
                     List of one or more MySQL/MariaDB databases to dump before
                     creating a backup, run once per configuration file. The
@@ -956,7 +988,15 @@ properties:
                                 directly to the dump command, without performing
                                 any validation on them. See mongodump
                                 documentation for details.
-                            example: --role=someone
+                            example: --dumpDbUsersAndRoles
+                        restore_options:
+                            type: string
+                            description: |
+                                Additional mongorestore options to pass
+                                directly to the dump command, without performing
+                                any validation on them. See mongorestore
+                                documentation for details.
+                            example: --restoreDbUsersAndRoles
                 description: |
                     List of one or more MongoDB databases to dump before
                     creating a backup, run once per configuration file. The

borgmatic/hooks/mongodb.py

@@ -160,4 +160,6 @@ def build_restore_command(extract_process, database, dump_filename):
         command.extend(('--password', database['password']))
     if 'authentication_database' in database:
         command.extend(('--authenticationDatabase', database['authentication_database']))
+    if 'restore_options' in database:
+        command.extend(database['restore_options'].split(' '))
     return command

borgmatic/hooks/mysql.py

@@ -197,6 +197,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     database = database_config[0]
     restore_command = (
         ('mysql', '--batch')
+        + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())

borgmatic/hooks/postgresql.py

@@ -62,7 +62,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--username', database['username']) if 'username' in database else ())
-        + (tuple(database['options'].split(' ')) if 'options' in database else ())
+        + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
     )
     logger.debug(
         '{}: Querying for "all" PostgreSQL databases to dump{}'.format(log_prefix, dry_run_label)
@@ -204,6 +204,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--username', database['username']) if 'username' in database else ())
         + (('--dbname', database['name']) if not all_databases else ())
+        + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
         + ('--command', 'ANALYZE')
     )
     pg_restore_command = database.get('pg_restore_command') or 'pg_restore'
@@ -217,6 +218,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--username', database['username']) if 'username' in database else ())
+        + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
         + (() if extract_process else (dump_filename,))
     )
     extra_environment = make_extra_environment(database)

docs/how-to/backup-your-databases.md

@@ -76,6 +76,11 @@ hooks:
           options: "--ssl"
 ```
 
+See your [borgmatic configuration
+file](https://torsion.org/borgmatic/docs/reference/configuration/) for
+additional customization of the options passed to database commands (when
+listing databases, restoring databases, etc.).
+
 
 ### All databases
 

tests/unit/hooks/test_mongodb.py

@@ -256,6 +256,24 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
     )
 
 
+def test_restore_database_dump_runs_mongorestore_with_options():
+    database_config = [{'name': 'foo', 'restore_options': '--harder',}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ['mongorestore', '--archive', '--drop', '--db', 'foo', '--harder',],
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
 def test_restore_database_dump_runs_psql_for_all_database_dump():
     database_config = [{'name': 'all'}]
     extract_process = flexmock(stdout=flexmock())

tests/unit/hooks/test_mysql.py

@@ -336,6 +336,23 @@ def test_restore_database_dump_errors_on_multiple_database_config():
         )
 
 
+def test_restore_database_dump_runs_mysql_with_options():
+    database_config = [{'name': 'foo', 'restore_options': '--harder'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('mysql', '--batch', '--harder'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
 def test_restore_database_dump_runs_mysql_with_hostname_and_port():
     database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
     extract_process = flexmock(stdout=flexmock())

tests/unit/hooks/test_postgresql.py

@@ -36,6 +36,55 @@ def test_database_names_to_dump_with_all_and_format_lists_databases():
     )
 
 
+def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port():
+    database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234}
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        (
+            'psql',
+            '--list',
+            '--no-password',
+            '--csv',
+            '--tuples-only',
+            '--host',
+            'localhost',
+            '--port',
+            '1234',
+        ),
+        extra_environment=object,
+    ).and_return('foo,test,\nbar,test,"stuff and such"')
+
+    assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
+        'foo',
+        'bar',
+    )
+
+
+def test_database_names_to_dump_with_all_and_format_lists_databases_with_username():
+    database = {'name': 'all', 'format': 'custom', 'username': 'postgres'}
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('psql', '--list', '--no-password', '--csv', '--tuples-only', '--username', 'postgres'),
+        extra_environment=object,
+    ).and_return('foo,test,\nbar,test,"stuff and such"')
+
+    assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
+        'foo',
+        'bar',
+    )
+
+
+def test_database_names_to_dump_with_all_and_format_lists_databases_with_options():
+    database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'}
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('psql', '--list', '--no-password', '--csv', '--tuples-only', '--harder'),
+        extra_environment=object,
+    ).and_return('foo,test,\nbar,test,"stuff and such"')
+
+    assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
+        'foo',
+        'bar',
+    )
+
+
 def test_database_names_to_dump_with_all_and_format_excludes_particular_databases():
     database = {'name': 'all', 'format': 'custom'}
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
@@ -90,7 +139,7 @@ def test_dump_databases_raises_when_no_database_names_to_dump():
         module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
 
 
-def test_dump_databases_with_dupliate_dump_skips_pg_dump():
+def test_dump_databases_with_duplicate_dump_skips_pg_dump():
     databases = [{'name': 'foo'}, {'name': 'bar'}]
     flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
     flexmock(module).should_receive('make_dump_path').and_return('')
@@ -480,6 +529,50 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
     )
 
 
+def test_restore_database_dump_runs_pg_restore_with_options():
+    database_config = [
+        {'name': 'foo', 'restore_options': '--harder', 'analyze_options': '--smarter'}
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'pg_restore',
+            '--no-password',
+            '--if-exists',
+            '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
+            '--harder',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment={'PGSSLMODE': 'disable'},
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'psql',
+            '--no-password',
+            '--quiet',
+            '--dbname',
+            'foo',
+            '--smarter',
+            '--command',
+            'ANALYZE',
+        ),
+        extra_environment={'PGSSLMODE': 'disable'},
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
 def test_restore_database_dump_runs_psql_for_all_database_dump():
     database_config = [{'name': 'all'}]
     extract_process = flexmock(stdout=flexmock())