Fix PostgreSQL restore error on "all" database dump.

This commit is contained in:
Dan Helfman 2020-04-24 15:50:33 -07:00 committed by GitHub
commit dce1928dc4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 62 additions and 19 deletions

View file

@ -34,7 +34,12 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
) )
all_databases = bool(name == 'all') all_databases = bool(name == 'all')
command = ( command = (
('pg_dumpall' if all_databases else 'pg_dump', '--no-password', '--clean') (
'pg_dumpall' if all_databases else 'pg_dump',
'--no-password',
'--clean',
'--if-exists',
)
+ ('--file', dump_filename) + ('--file', dump_filename)
+ (('--host', database['hostname']) if 'hostname' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ())
@ -93,23 +98,28 @@ def restore_database_dumps(databases, log_prefix, location_config, dry_run):
dump_filename = dump.make_database_dump_filename( dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), database['name'], database.get('hostname') make_dump_path(location_config), database['name'], database.get('hostname')
) )
restore_command = ( all_databases = bool(database['name'] == 'all')
('pg_restore', '--no-password', '--clean', '--if-exists', '--exit-on-error')
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ())
+ ('--dbname', database['name'])
+ (dump_filename,)
)
extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
analyze_command = ( analyze_command = (
('psql', '--no-password', '--quiet') ('psql', '--no-password', '--quiet')
+ (('--host', database['hostname']) if 'hostname' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ()) + (('--username', database['username']) if 'username' in database else ())
+ ('--dbname', database['name']) + (('--dbname', database['name']) if not all_databases else ())
+ ('--command', 'ANALYZE') + ('--command', 'ANALYZE')
) )
restore_command = (
('psql' if all_databases else 'pg_restore', '--no-password')
+ (
('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
if not all_databases
else ()
)
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ())
+ (('-f', dump_filename) if all_databases else (dump_filename,))
)
extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
logger.debug( logger.debug(
'{}: Restoring PostgreSQL database {}{}'.format( '{}: Restoring PostgreSQL database {}{}'.format(

View file

@ -29,6 +29,10 @@ hooks:
hostname: postgresql hostname: postgresql
username: postgres username: postgres
password: test password: test
- name: all
hostname: postgresql
username: postgres
password: test
mysql_databases: mysql_databases:
- name: test - name: test
hostname: mysql hostname: mysql

View file

@ -17,6 +17,7 @@ def test_dump_databases_runs_pg_dump_for_each_database():
'pg_dump', 'pg_dump',
'--no-password', '--no-password',
'--clean', '--clean',
'--if-exists',
'--file', '--file',
'databases/localhost/{}'.format(name), 'databases/localhost/{}'.format(name),
'--format', '--format',
@ -54,6 +55,7 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port():
'pg_dump', 'pg_dump',
'--no-password', '--no-password',
'--clean', '--clean',
'--if-exists',
'--file', '--file',
'databases/database.example.org/foo', 'databases/database.example.org/foo',
'--host', '--host',
@ -83,6 +85,7 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
'pg_dump', 'pg_dump',
'--no-password', '--no-password',
'--clean', '--clean',
'--if-exists',
'--file', '--file',
'databases/localhost/foo', 'databases/localhost/foo',
'--username', '--username',
@ -110,6 +113,7 @@ def test_dump_databases_runs_pg_dump_with_format():
'pg_dump', 'pg_dump',
'--no-password', '--no-password',
'--clean', '--clean',
'--if-exists',
'--file', '--file',
'databases/localhost/foo', 'databases/localhost/foo',
'--format', '--format',
@ -135,6 +139,7 @@ def test_dump_databases_runs_pg_dump_with_options():
'pg_dump', 'pg_dump',
'--no-password', '--no-password',
'--clean', '--clean',
'--if-exists',
'--file', '--file',
'databases/localhost/foo', 'databases/localhost/foo',
'--format', '--format',
@ -157,7 +162,14 @@ def test_dump_databases_runs_pg_dumpall_for_all_databases():
flexmock(module.os).should_receive('makedirs') flexmock(module.os).should_receive('makedirs')
flexmock(module).should_receive('execute_command').with_args( flexmock(module).should_receive('execute_command').with_args(
('pg_dumpall', '--no-password', '--clean', '--file', 'databases/localhost/all'), (
'pg_dumpall',
'--no-password',
'--clean',
'--if-exists',
'--file',
'databases/localhost/all',
),
extra_environment=None, extra_environment=None,
).once() ).once()
@ -197,9 +209,9 @@ def test_restore_database_dumps_restores_each_database():
( (
'pg_restore', 'pg_restore',
'--no-password', '--no-password',
'--clean',
'--if-exists', '--if-exists',
'--exit-on-error', '--exit-on-error',
'--clean',
'--dbname', '--dbname',
name, name,
'databases/localhost/{}'.format(name), 'databases/localhost/{}'.format(name),
@ -225,15 +237,15 @@ def test_restore_database_dumps_runs_pg_restore_with_hostname_and_port():
( (
'pg_restore', 'pg_restore',
'--no-password', '--no-password',
'--clean',
'--if-exists', '--if-exists',
'--exit-on-error', '--exit-on-error',
'--clean',
'--dbname',
'foo',
'--host', '--host',
'database.example.org', 'database.example.org',
'--port', '--port',
'5433', '5433',
'--dbname',
'foo',
'databases/localhost/foo', 'databases/localhost/foo',
), ),
extra_environment=None, extra_environment=None,
@ -269,13 +281,13 @@ def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
( (
'pg_restore', 'pg_restore',
'--no-password', '--no-password',
'--clean',
'--if-exists', '--if-exists',
'--exit-on-error', '--exit-on-error',
'--username', '--clean',
'postgres',
'--dbname', '--dbname',
'foo', 'foo',
'--username',
'postgres',
'databases/localhost/foo', 'databases/localhost/foo',
), ),
extra_environment={'PGPASSWORD': 'trustsome1'}, extra_environment={'PGPASSWORD': 'trustsome1'},
@ -296,3 +308,20 @@ def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
).once() ).once()
module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False) module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
def test_restore_all_database_dump():
databases = [{'name': 'all'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
'databases/localhost/all'
)
flexmock(module).should_receive('execute_command').with_args(
('psql', '--no-password', '-f', 'databases/localhost/all'), extra_environment=None
).once()
flexmock(module).should_receive('execute_command').with_args(
('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), extra_environment=None
).once()
module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)