From 925f99cfef1684a07e7629ab62f669ac9bfd3b0e Mon Sep 17 00:00:00 2001 From: shivansh02 Date: Sun, 3 Mar 2024 03:47:02 +0530 Subject: [PATCH 1/4] custom dump command for mysql --- borgmatic/config/schema.yaml | 15 ++++++++ borgmatic/hooks/mysql.py | 7 ++-- tests/unit/hooks/test_mysql.py | 64 ++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 7d81c49..51278bf 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1087,6 +1087,21 @@ properties: Password with which to connect to the restore database. Defaults to the "password" option. example: trustsome1 + mysql_dump_command: + type: string + description: | + Command to use instead of "mysqldump". This can be used + to run a specific mysql_dump version (e.g., one inside + a running docker container). Defaults to "mysqldump". + example: docker exec mysql_container mysqldump + mysql_command: + type: string + description: | + Command to run instead of "mysql". This + can be used to run a specific mysql + version (e.g., one inside a running docker + container). Defaults to "mysql". + example: docker exec mysql_container mysql format: type: string enum: ['sql'] diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index 8f1b83b..086e798 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -79,8 +79,9 @@ def execute_dump_command( ) return None + mysql_dump_command = database.get('mysql_dump_command') or 'mysqldump' dump_command = ( - ('mysqldump',) + (mysql_dump_command,) + (tuple(database['options'].split(' ')) if 'options' in database else ()) + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) @@ -206,9 +207,9 @@ def restore_data_source_dump( password = connection_params['password'] or data_source.get( 'restore_password', data_source.get('password') ) - + mysql_restore_command = data_source.get('mysql_command') or 'mysql' restore_command = ( - ('mysql', '--batch') + (mysql_restore_command, '--batch') + ( tuple(data_source['restore_options'].split(' ')) if 'restore_options' in data_source diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 6f778b3..46d1777 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -315,6 +315,42 @@ def test_execute_dump_command_runs_mysqldump_with_options(): ) +def test_execute_dump_command_runs_non_default_mysqldump(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'custom_mysqldump', # Custom MySQL dump command + '--add-drop-database', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + extra_environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={ + 'name': 'foo', + 'mysql_dump_command': 'custom_mysqldump', + }, # Custom MySQL dump command specified + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(True) @@ -435,6 +471,34 @@ def test_restore_data_source_dump_runs_mysql_with_options(): ) +def test_restore_data_source_dump_runs_non_default_mysql_with_options(): + hook_config = [{'name': 'foo', 'mysql_command': 'custom_mysql', 'restore_options': '--harder'}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ('custom_mysql', '--batch', '--harder'), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment=None, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + 'test.yaml', + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) + + def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] extract_process = flexmock(stdout=flexmock()) From 2b755d8ade9717ea4c777a70da363b8fdec99c66 Mon Sep 17 00:00:00 2001 From: shivansh02 Date: Sun, 3 Mar 2024 23:15:07 +0530 Subject: [PATCH 2/4] custom show command for mysql and schema description --- borgmatic/config/schema.yaml | 6 +++--- borgmatic/hooks/mysql.py | 3 ++- tests/unit/hooks/test_mysql.py | 21 +++++++++++++++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 51278bf..c8288cf 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1092,15 +1092,15 @@ properties: description: | Command to use instead of "mysqldump". This can be used to run a specific mysql_dump version (e.g., one inside - a running docker container). Defaults to "mysqldump". + a running container). Defaults to "mysqldump". example: docker exec mysql_container mysqldump mysql_command: type: string description: | Command to run instead of "mysql". This can be used to run a specific mysql - version (e.g., one inside a running docker - container). Defaults to "mysql". + version (e.g., one inside a running container). + Defaults to "mysql". example: docker exec mysql_container mysql format: type: string diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index 086e798..2086efb 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -35,8 +35,9 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): if dry_run: return () + mysql_show_command = database.get('mysql_command') or 'mysql' show_command = ( - ('mysql',) + (mysql_show_command,) + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 46d1777..771fc2f 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -142,6 +142,27 @@ def test_database_names_to_dump_runs_mysql_with_list_options(): assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar') +def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): + database = { + 'name': 'all', + 'list_options': '--defaults-extra-file=my.cnf', + 'mysql_command': 'custom_mysql', + } + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + extra_environment=None, + full_command=( + 'custom_mysql', # Custom MySQL command + '--defaults-extra-file=my.cnf', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ) + ).and_return(('foo\nbar')).once() + + assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar') + + def test_execute_dump_command_runs_mysqldump(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') From 9e3d19a406364d422d7336fd47e1becd774d57ff Mon Sep 17 00:00:00 2001 From: shivansh02 Date: Sun, 3 Mar 2024 23:31:02 +0530 Subject: [PATCH 3/4] custom commands escaped --- borgmatic/hooks/mysql.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index 2086efb..8832723 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -1,6 +1,7 @@ import copy import logging import os +import shlex from borgmatic.execute import ( execute_command, @@ -34,10 +35,12 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): return (database['name'],) if dry_run: return () - - mysql_show_command = database.get('mysql_command') or 'mysql' + + mysql_show_command = tuple( + shlex.quote(part) for part in shlex.split(database.get('mysql_command') or 'mysql') + ) show_command = ( - (mysql_show_command,) + mysql_show_command + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) @@ -79,10 +82,12 @@ def execute_dump_command( f'{log_prefix}: Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}' ) return None - - mysql_dump_command = database.get('mysql_dump_command') or 'mysqldump' + + mysql_dump_command = tuple( + shlex.quote(part) for part in shlex.split(database.get('mysql_dump_command') or 'mysqldump') + ) dump_command = ( - (mysql_dump_command,) + mysql_dump_command + (tuple(database['options'].split(' ')) if 'options' in database else ()) + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) @@ -208,9 +213,13 @@ def restore_data_source_dump( password = connection_params['password'] or data_source.get( 'restore_password', data_source.get('password') ) - mysql_restore_command = data_source.get('mysql_command') or 'mysql' + + mysql_restore_command = tuple( + shlex.quote(part) for part in shlex.split(data_source.get('mysql_command') or 'mysql') + ) restore_command = ( - (mysql_restore_command, '--batch') + mysql_restore_command + + ('--batch',) + ( tuple(data_source['restore_options'].split(' ')) if 'restore_options' in data_source From b6cb7da98e44c52a02a91aa1184ccd9ad06365f8 Mon Sep 17 00:00:00 2001 From: shivansh02 Date: Mon, 4 Mar 2024 00:24:22 +0530 Subject: [PATCH 4/4] custom dump commands for mariadb --- borgmatic/config/schema.yaml | 16 ++++++ borgmatic/hooks/mariadb.py | 18 +++++-- borgmatic/hooks/mysql.py | 4 +- tests/unit/hooks/test_mariadb.py | 89 ++++++++++++++++++++++++++++++++ tests/unit/hooks/test_mysql.py | 2 +- 5 files changed, 123 insertions(+), 6 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index c8288cf..6aee336 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -971,6 +971,22 @@ properties: a password will only work if MariaDB is configured to trust the configured username without a password. example: trustsome1 + mariadb_dump_command: + type: string + description: | + Command to use instead of "mariadb-dump". This can + be used to run a specific mariadb_dump version + (e.g., one inside a running container). + Defaults to "mariadb-dump". + example: docker exec mariadb_container mariadb-dump + mariadb_command: + type: string + description: | + Command to run instead of "mariadb". This + can be used to run a specific mariadb + version (e.g., one inside a running container). + Defaults to "mariadb". + example: docker exec mariadb_container mariadb restore_password: type: string description: | diff --git a/borgmatic/hooks/mariadb.py b/borgmatic/hooks/mariadb.py index b3c9a3e..6740556 100644 --- a/borgmatic/hooks/mariadb.py +++ b/borgmatic/hooks/mariadb.py @@ -1,6 +1,7 @@ import copy import logging import os +import shlex from borgmatic.execute import ( execute_command, @@ -35,8 +36,11 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): if dry_run: return () + mariadb_show_command = tuple( + shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb') + ) show_command = ( - ('mariadb',) + mariadb_show_command + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) @@ -79,8 +83,12 @@ def execute_dump_command( ) return None + mariadb_dump_command = tuple( + shlex.quote(part) + for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump') + ) dump_command = ( - ('mariadb-dump',) + mariadb_dump_command + (tuple(database['options'].split(' ')) if 'options' in database else ()) + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) @@ -208,8 +216,12 @@ def restore_data_source_dump( 'restore_password', data_source.get('password') ) + mariadb_restore_command = tuple( + shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb') + ) restore_command = ( - ('mariadb', '--batch') + mariadb_restore_command + + ('--batch',) + ( tuple(data_source['restore_options'].split(' ')) if 'restore_options' in data_source diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index 8832723..46f7657 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -35,7 +35,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): return (database['name'],) if dry_run: return () - + mysql_show_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_command') or 'mysql') ) @@ -82,7 +82,7 @@ def execute_dump_command( f'{log_prefix}: Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}' ) return None - + mysql_dump_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_dump_command') or 'mysqldump') ) diff --git a/tests/unit/hooks/test_mariadb.py b/tests/unit/hooks/test_mariadb.py index 7fe0ab9..15394a5 100644 --- a/tests/unit/hooks/test_mariadb.py +++ b/tests/unit/hooks/test_mariadb.py @@ -142,6 +142,27 @@ def test_database_names_to_dump_runs_mariadb_with_list_options(): assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar') +def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): + database = { + 'name': 'all', + 'list_options': '--defaults-extra-file=mariadb.cnf', + 'mariadb_command': 'custom_mariadb', + } + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + extra_environment=None, + full_command=( + 'custom_mariadb', # Custom MariaDB command + '--defaults-extra-file=mariadb.cnf', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + ).and_return(('foo\nbar')).once() + + assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar') + + def test_execute_dump_command_runs_mariadb_dump(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') @@ -315,6 +336,44 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): ) +def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'custom_mariadb_dump', # Custom MariaDB dump command + '--stuff=such', + '--add-drop-database', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + extra_environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={ + 'name': 'foo', + 'mariadb_dump_command': 'custom_mariadb_dump', + 'options': '--stuff=such', + }, # Custom MariaDB dump command specified + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(True) @@ -435,6 +494,36 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): ) +def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): + hook_config = [ + {'name': 'foo', 'restore_options': '--harder', 'mariadb_command': 'custom_mariadb'} + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ('custom_mariadb', '--batch', '--harder'), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment=None, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + 'test.yaml', + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) + + def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] extract_process = flexmock(stdout=flexmock()) diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 771fc2f..3560a99 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -157,7 +157,7 @@ def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): '--batch', '--execute', 'show schemas', - ) + ), ).and_return(('foo\nbar')).once() assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')