Add SSL support to PostgreSQL database configuration (#331).

Reviewed-on: https://projects.torsion.org/witten/borgmatic/pulls/331
This commit is contained in:
Dan Helfman 2020-06-20 21:24:14 +00:00
commit f5ebca4907
3 changed files with 94 additions and 19 deletions

View file

@ -556,6 +556,37 @@ map:
documentation for details. Note that format is documentation for details. Note that format is
ignored when the database name is "all". ignored when the database name is "all".
example: directory example: directory
ssl_mode:
type: str
enum: ['disable', 'allow', 'prefer',
'require', 'verify-ca', 'verify-full']
desc: |
SSL mode to use to connect to the database
server. One of "disable", "allow", "prefer",
"require", "verify-ca" or "verify-full".
Defaults to "disable".
example: require
ssl_cert:
type: str
desc: |
Path to a client certificate.
example: "/root/.postgresql/postgresql.crt"
ssl_key:
type: str
desc: |
Path to a private client key.
example: "/root/.postgresql/postgresql.key"
ssl_root_cert:
type: str
desc: |
Path to a root certificate containing a list of
trusted certificate authorities.
example: "/root/.postgresql/root.crt"
ssl_crl:
type: str
desc: |
Path to a certificate revocation list.
example: "/root/.postgresql/root.crl"
options: options:
type: str type: str
desc: | desc: |
@ -570,7 +601,8 @@ map:
database dumps are added to your source directories at database dumps are added to your source directories at
runtime, backed up, and removed afterwards. Requires runtime, backed up, and removed afterwards. Requires
pg_dump/pg_dumpall/pg_restore commands. See pg_dump/pg_dumpall/pg_restore commands. See
https://www.postgresql.org/docs/current/app-pgdump.html for https://www.postgresql.org/docs/current/app-pgdump.html and
https://www.postgresql.org/docs/current/libpq-ssl.html for
details. details.
mysql_databases: mysql_databases:
seq: seq:

View file

@ -15,6 +15,25 @@ def make_dump_path(location_config): # pragma: no cover
) )
def make_extra_environment(database):
'''
Make the extra_environment dict from the given database configuration.
'''
extra = dict()
if 'password' in database:
extra['PGPASSWORD'] = database['password']
extra['PGSSLMODE'] = database.get('ssl_mode', 'disable')
if 'ssl_cert' in database:
extra['PGSSLCERT'] = database['ssl_cert']
if 'ssl_key' in database:
extra['PGSSLKEY'] = database['ssl_key']
if 'ssl_root_cert' in database:
extra['PGSSLROOTCERT'] = database['ssl_root_cert']
if 'ssl_crl' in database:
extra['PGSSLCRL'] = database['ssl_crl']
return extra
def dump_databases(databases, log_prefix, location_config, dry_run): def dump_databases(databases, log_prefix, location_config, dry_run):
''' '''
Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of
@ -56,7 +75,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
# format in a particular, a named destination is required, and redirection doesn't work. # format in a particular, a named destination is required, and redirection doesn't work.
+ (('>', dump_filename) if dump_format != 'directory' else ()) + (('>', dump_filename) if dump_format != 'directory' else ())
) )
extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None extra_environment = make_extra_environment(database)
logger.debug( logger.debug(
'{}: Dumping PostgreSQL database {} to {}{}'.format( '{}: Dumping PostgreSQL database {} to {}{}'.format(
@ -141,7 +160,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
+ (('--username', database['username']) if 'username' in database else ()) + (('--username', database['username']) if 'username' in database else ())
+ (() if extract_process else (dump_filename,)) + (() if extract_process else (dump_filename,))
) )
extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None extra_environment = make_extra_environment(database)
logger.debug( logger.debug(
'{}: Restoring PostgreSQL database {}{}'.format(log_prefix, database['name'], dry_run_label) '{}: Restoring PostgreSQL database {}{}'.format(log_prefix, database['name'], dry_run_label)

View file

@ -29,7 +29,7 @@ def test_dump_databases_runs_pg_dump_for_each_database():
'databases/localhost/{}'.format(name), 'databases/localhost/{}'.format(name),
), ),
shell=True, shell=True,
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
run_to_completion=False, run_to_completion=False,
).and_return(process).once() ).and_return(process).once()
@ -74,7 +74,7 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port():
'databases/database.example.org/foo', 'databases/database.example.org/foo',
), ),
shell=True, shell=True,
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
run_to_completion=False, run_to_completion=False,
).and_return(process).once() ).and_return(process).once()
@ -105,13 +105,34 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
'databases/localhost/foo', 'databases/localhost/foo',
), ),
shell=True, shell=True,
extra_environment={'PGPASSWORD': 'trustsome1'}, extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
run_to_completion=False, run_to_completion=False,
).and_return(process).once() ).and_return(process).once()
assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
def test_make_extra_environment():
database = {
'name': 'foo',
'ssl_mode': 'require',
'ssl_cert': 'cert.crt',
'ssl_key': 'key.key',
'ssl_root_cert': 'root.crt',
'ssl_crl': 'crl.crl',
}
expected = {
'PGSSLMODE': 'require',
'PGSSLCERT': 'cert.crt',
'PGSSLKEY': 'key.key',
'PGSSLROOTCERT': 'root.crt',
'PGSSLCRL': 'crl.crl',
}
extra_env = module.make_extra_environment(database)
assert extra_env == expected
def test_dump_databases_runs_pg_dump_with_directory_format(): def test_dump_databases_runs_pg_dump_with_directory_format():
databases = [{'name': 'foo', 'format': 'directory'}] databases = [{'name': 'foo', 'format': 'directory'}]
process = flexmock() process = flexmock()
@ -135,7 +156,7 @@ def test_dump_databases_runs_pg_dump_with_directory_format():
'foo', 'foo',
), ),
shell=True, shell=True,
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
run_to_completion=False, run_to_completion=False,
).and_return(process).once() ).and_return(process).once()
@ -151,6 +172,8 @@ def test_dump_databases_runs_pg_dump_with_options():
) )
flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
flexmock(module).should_receive('execute_command').with_args( flexmock(module).should_receive('execute_command').with_args(
( (
'pg_dump', 'pg_dump',
@ -165,7 +188,7 @@ def test_dump_databases_runs_pg_dump_with_options():
'databases/localhost/foo', 'databases/localhost/foo',
), ),
shell=True, shell=True,
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
run_to_completion=False, run_to_completion=False,
).and_return(process).once() ).and_return(process).once()
@ -184,7 +207,7 @@ def test_dump_databases_runs_pg_dumpall_for_all_databases():
flexmock(module).should_receive('execute_command').with_args( flexmock(module).should_receive('execute_command').with_args(
('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'), ('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'),
shell=True, shell=True,
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
run_to_completion=False, run_to_completion=False,
).and_return(process).once() ).and_return(process).once()
@ -210,12 +233,12 @@ def test_restore_database_dump_runs_pg_restore():
processes=[extract_process], processes=[extract_process],
output_log_level=logging.DEBUG, output_log_level=logging.DEBUG,
input_file=extract_process.stdout, input_file=extract_process.stdout,
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
borg_local_path='borg', borg_local_path='borg',
).once() ).once()
flexmock(module).should_receive('execute_command').with_args( flexmock(module).should_receive('execute_command').with_args(
('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
@ -260,7 +283,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
processes=[extract_process], processes=[extract_process],
output_log_level=logging.DEBUG, output_log_level=logging.DEBUG,
input_file=extract_process.stdout, input_file=extract_process.stdout,
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
borg_local_path='borg', borg_local_path='borg',
).once() ).once()
flexmock(module).should_receive('execute_command').with_args( flexmock(module).should_receive('execute_command').with_args(
@ -277,7 +300,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
'--command', '--command',
'ANALYZE', 'ANALYZE',
), ),
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
@ -306,7 +329,7 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
processes=[extract_process], processes=[extract_process],
output_log_level=logging.DEBUG, output_log_level=logging.DEBUG,
input_file=extract_process.stdout, input_file=extract_process.stdout,
extra_environment={'PGPASSWORD': 'trustsome1'}, extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
borg_local_path='borg', borg_local_path='borg',
).once() ).once()
flexmock(module).should_receive('execute_command').with_args( flexmock(module).should_receive('execute_command').with_args(
@ -321,7 +344,7 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
'--command', '--command',
'ANALYZE', 'ANALYZE',
), ),
extra_environment={'PGPASSWORD': 'trustsome1'}, extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
@ -340,11 +363,12 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
processes=[extract_process], processes=[extract_process],
output_log_level=logging.DEBUG, output_log_level=logging.DEBUG,
input_file=extract_process.stdout, input_file=extract_process.stdout,
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
borg_local_path='borg', borg_local_path='borg',
).once() ).once()
flexmock(module).should_receive('execute_command').with_args( flexmock(module).should_receive('execute_command').with_args(
('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), extra_environment=None ('psql', '--no-password', '--quiet', '--command', 'ANALYZE'),
extra_environment={'PGSSLMODE': 'disable'},
).once() ).once()
module.restore_database_dump( module.restore_database_dump(
@ -383,12 +407,12 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
processes=[], processes=[],
output_log_level=logging.DEBUG, output_log_level=logging.DEBUG,
input_file=None, input_file=None,
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
borg_local_path='borg', borg_local_path='borg',
).once() ).once()
flexmock(module).should_receive('execute_command').with_args( flexmock(module).should_receive('execute_command').with_args(
('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
extra_environment=None, extra_environment={'PGSSLMODE': 'disable'},
).once() ).once()
module.restore_database_dump( module.restore_database_dump(