fix integration tests and mongodb auth

This commit is contained in:
Andrea Ghensi 2021-12-29 22:18:50 +01:00
parent 6b7653484b
commit 7c6ce9399c
7 changed files with 57 additions and 18 deletions

View file

@ -13,6 +13,11 @@ services:
environment: environment:
MYSQL_ROOT_PASSWORD: test MYSQL_ROOT_PASSWORD: test
MYSQL_DATABASE: test MYSQL_DATABASE: test
- name: mongodb
image: mongo:5.0.5
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: test
clone: clone:
skip_verify: true skip_verify: true
@ -38,6 +43,11 @@ services:
environment: environment:
MYSQL_ROOT_PASSWORD: test MYSQL_ROOT_PASSWORD: test
MYSQL_DATABASE: test MYSQL_DATABASE: test
- name: mongodb
image: mongo:5.0.5
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: test
clone: clone:
skip_verify: true skip_verify: true
@ -63,6 +73,11 @@ services:
environment: environment:
MYSQL_ROOT_PASSWORD: test MYSQL_ROOT_PASSWORD: test
MYSQL_DATABASE: test MYSQL_DATABASE: test
- name: mongodb
image: mongo:5.0.5
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: test
clone: clone:
skip_verify: true skip_verify: true

View file

@ -811,15 +811,25 @@ properties:
Password with which to connect to the database. Password with which to connect to the database.
Skip it if no authentication is needed. Skip it if no authentication is needed.
example: trustsome1 example: trustsome1
auth_db:
type: string
description: |
Authentication database where the specified
username has been created.
If no authentication database is specified,
the databse provided in "name" will be used.
If "name" is "all", the "admin" database will
be used.
example: admin
format: format:
type: string type: string
enum: ['archive', 'directory'] enum: ['archive', 'directory']
description: | description: |
Database dump output format. One of "archive", Database dump output format. One of "archive",
or "directory". Defaults to "archive" (unlike or "directory". Defaults to "archive". See
raw pg_dump). See pg_dump documentation for mongodump documentation for details. Note that
details. Note that format is ignored when the format is ignored when the database name is
database name is "all". "all".
example: directory example: directory
options: options:
type: string type: string

View file

@ -35,7 +35,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
dump_filename = dump.make_database_dump_filename( dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), name, database.get('hostname') make_dump_path(location_config), name, database.get('hostname')
) )
dump_format = database.get('format', 'custom') dump_format = database.get('format', 'archive')
logger.debug( logger.debug(
'{}: Dumping MongoDB database {} to {}{}'.format( '{}: Dumping MongoDB database {} to {}{}'.format(
@ -72,6 +72,8 @@ def build_dump_command(database, dump_filename, dump_format):
command.extend(('--username', database['username'])) command.extend(('--username', database['username']))
if 'password' in database: if 'password' in database:
command.extend(('--password', database['password'])) command.extend(('--password', database['password']))
if 'auth_db' in database:
command.extend(('--authenticationDatabase', database['auth_db']))
if not all_databases: if not all_databases:
command.extend(('--db', database['name'])) command.extend(('--db', database['name']))
if 'options' in database: if 'options' in database:
@ -155,4 +157,6 @@ def build_restore_command(extract_process, database, dump_filename):
command.extend(('--username', database['username'])) command.extend(('--username', database['username']))
if 'password' in database: if 'password' in database:
command.extend(('--password', database['password'])) command.extend(('--password', database['password']))
if 'auth_db' in database:
command.extend(('--authenticationDatabase', database['auth_db']))
return command return command

View file

@ -10,7 +10,7 @@
set -e set -e
apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools
# If certain dependencies of black are available in this version of Alpine, install them. # If certain dependencies of black are available in this version of Alpine, install them.
apk add --no-cache py3-typed-ast py3-regex || true apk add --no-cache py3-typed-ast py3-regex || true
python3 -m pip install --upgrade pip==21.3.1 setuptools==58.2.0 python3 -m pip install --upgrade pip==21.3.1 setuptools==58.2.0

View file

@ -10,6 +10,11 @@ services:
environment: environment:
MYSQL_ROOT_PASSWORD: test MYSQL_ROOT_PASSWORD: test
MYSQL_DATABASE: test MYSQL_DATABASE: test
mongodb:
image: mongo:5.0.5
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: test
tests: tests:
image: alpine:3.13 image: alpine:3.13
volumes: volumes:

View file

@ -52,6 +52,7 @@ hooks:
hostname: mongodb hostname: mongodb
username: root username: root
password: test password: test
auth_db: admin
- name: all - name: all
hostname: mongodb hostname: mongodb
username: root username: root
@ -77,15 +78,15 @@ def test_database_dump_and_restore():
write_configuration(config_path, repository_path, borgmatic_source_directory) write_configuration(config_path, repository_path, borgmatic_source_directory)
subprocess.check_call( subprocess.check_call(
'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ') ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
) )
# Run borgmatic to generate a backup archive including a database dump. # Run borgmatic to generate a backup archive including a database dump.
subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' ')) subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
# Get the created archive name. # Get the created archive name.
output = subprocess.check_output( output = subprocess.check_output(
'borgmatic --config {} list --json'.format(config_path).split(' ') ['borgmatic', '--config', config_path, 'list', '--json']
).decode(sys.stdout.encoding) ).decode(sys.stdout.encoding)
parsed_output = json.loads(output) parsed_output = json.loads(output)
@ -95,9 +96,7 @@ def test_database_dump_and_restore():
# Restore the database from the archive. # Restore the database from the archive.
subprocess.check_call( subprocess.check_call(
'borgmatic --config {} restore --archive {}'.format(config_path, archive_name).split( ['borgmatic', '--config', config_path, 'restore', '--archive', archive_name]
' '
)
) )
finally: finally:
os.chdir(original_working_directory) os.chdir(original_working_directory)
@ -122,15 +121,15 @@ def test_database_dump_and_restore_with_directory_format():
) )
subprocess.check_call( subprocess.check_call(
'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ') ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
) )
# Run borgmatic to generate a backup archive including a database dump. # Run borgmatic to generate a backup archive including a database dump.
subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' ')) subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
# Restore the database from the archive. # Restore the database from the archive.
subprocess.check_call( subprocess.check_call(
'borgmatic --config {} restore --archive latest'.format(config_path).split(' ') ['borgmatic', '--config', config_path, 'restore', '--archive', 'latest']
) )
finally: finally:
os.chdir(original_working_directory) os.chdir(original_working_directory)
@ -150,7 +149,7 @@ def test_database_dump_with_error_causes_borgmatic_to_exit():
write_configuration(config_path, repository_path, borgmatic_source_directory) write_configuration(config_path, repository_path, borgmatic_source_directory)
subprocess.check_call( subprocess.check_call(
'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ') ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
) )
# Run borgmatic with a config override such that the database dump fails. # Run borgmatic with a config override such that the database dump fails.

View file

@ -67,7 +67,7 @@ def test_dump_databases_runs_mongodump_with_hostname_and_port():
def test_dump_databases_runs_mongodump_with_username_and_password(): def test_dump_databases_runs_mongodump_with_username_and_password():
databases = [{'name': 'foo', 'username': 'mongo', 'password': 'trustsome1'}] databases = [{'name': 'foo', 'username': 'mongo', 'password': 'trustsome1', 'auth_db': "admin"}]
process = flexmock() process = flexmock()
flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.dump).should_receive('make_database_dump_filename').and_return( flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
@ -83,6 +83,8 @@ def test_dump_databases_runs_mongodump_with_username_and_password():
'mongo', 'mongo',
'--password', '--password',
'trustsome1', 'trustsome1',
'--authenticationDatabase',
'admin',
'--db', '--db',
'foo', 'foo',
'>', '>',
@ -213,7 +215,9 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
def test_restore_database_dump_runs_pg_restore_with_username_and_password(): def test_restore_database_dump_runs_pg_restore_with_username_and_password():
database_config = [{'name': 'foo', 'username': 'mongo', 'password': 'trustsome1'}] database_config = [
{'name': 'foo', 'username': 'mongo', 'password': 'trustsome1', 'auth_db': 'admin'}
]
extract_process = flexmock(stdout=flexmock()) extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_dump_path') flexmock(module).should_receive('make_dump_path')
@ -229,6 +233,8 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
'mongo', 'mongo',
'--password', '--password',
'trustsome1', 'trustsome1',
'--authenticationDatabase',
'admin',
], ],
processes=[extract_process], processes=[extract_process],
output_log_level=logging.DEBUG, output_log_level=logging.DEBUG,