From af1cc27988f16bde56b4b07487b72002a8788236 Mon Sep 17 00:00:00 2001
From: Divyansh Singh <divyanshanandsingh@gmail.com>
Date: Thu, 2 Mar 2023 23:55:16 +0530
Subject: [PATCH 1/8] feat: add dump-restore support for sqlite databases

---
 borgmatic/config/schema.yaml         |  25 ++++++
 borgmatic/hooks/dispatch.py          |   2 +
 borgmatic/hooks/dump.py              |   7 +-
 borgmatic/hooks/sqlite.py            | 122 +++++++++++++++++++++++++++
 docs/how-to/backup-your-databases.md |  15 +++-
 tests/end-to-end/test_database.py    |   3 +
 tests/unit/hooks/test_sqlite.py      |  75 ++++++++++++++++
 7 files changed, 244 insertions(+), 5 deletions(-)
 create mode 100644 borgmatic/hooks/sqlite.py
 create mode 100644 tests/unit/hooks/test_sqlite.py

diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index d7acc1c..6f399ed 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -931,6 +931,31 @@ properties:
                     mysqldump/mysql commands (from either MySQL or MariaDB). See
                     https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
                     https://mariadb.com/kb/en/library/mysqldump/ for details.
+            sqlite_databases:
+                type: array
+                items:
+                    type: object
+                    required: ['path','name']
+                    additionalProperties: false
+                    properties:
+                        path:
+                            type: string
+                            description: |
+                                Path to the SQLite database file to dump. If
+                                relative, it is relative to the current working
+                                directory. If absolute, it is relative to the
+                                root of the filesystem. Note that using this
+                                database hook implicitly enables both
+                                read_special and one_file_system (see above) to
+                                support dump and restore streaming.
+                            example: /var/lib/sqlite/users.db
+                        name:
+                            type: string
+                            description: |
+                                This is used to tag the database dump file with
+                                a name. It is not used to identify the database
+                                file itself.
+                            example: users
             mongodb_databases:
                 type: array
                 items:
diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py
index 41dcee0..88a99eb 100644
--- a/borgmatic/hooks/dispatch.py
+++ b/borgmatic/hooks/dispatch.py
@@ -9,6 +9,7 @@ from borgmatic.hooks import (
     ntfy,
     pagerduty,
     postgresql,
+    sqlite,
 )
 
 logger = logging.getLogger(__name__)
@@ -22,6 +23,7 @@ HOOK_NAME_TO_MODULE = {
     'ntfy': ntfy,
     'pagerduty': pagerduty,
     'postgresql_databases': postgresql,
+    'sqlite_databases': sqlite,
 }
 
 
diff --git a/borgmatic/hooks/dump.py b/borgmatic/hooks/dump.py
index d4c8e3f..43686d3 100644
--- a/borgmatic/hooks/dump.py
+++ b/borgmatic/hooks/dump.py
@@ -6,7 +6,12 @@ from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
 
 logger = logging.getLogger(__name__)
 
-DATABASE_HOOK_NAMES = ('postgresql_databases', 'mysql_databases', 'mongodb_databases')
+DATABASE_HOOK_NAMES = (
+    'postgresql_databases',
+    'mysql_databases',
+    'mongodb_databases',
+    'sqlite_databases',
+)
 
 
 def make_database_dump_path(borgmatic_source_directory, database_hook_name):
diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py
new file mode 100644
index 0000000..f9b83ae
--- /dev/null
+++ b/borgmatic/hooks/sqlite.py
@@ -0,0 +1,122 @@
+import logging
+import os
+from subprocess import CalledProcessError
+
+from borgmatic.execute import execute_command, execute_command_with_processes
+from borgmatic.hooks import dump
+
+logger = logging.getLogger(__name__)
+
+
+def make_dump_path(location_config):  # pragma: no cover
+    '''
+    Make the dump path from the given location configuration and the name of this hook.
+    '''
+    return dump.make_database_dump_path(
+        location_config.get('borgmatic_source_directory'), 'sqlite_databases'
+    )
+
+
+def dump_databases(databases, log_prefix, location_config, dry_run):  # pragma: no cover
+    '''
+    Dump the given SQLite3 databases to a file. The databases are supplied as a sequence of
+    configuration dicts, as per the configuration schema. Use the given log prefix in any log
+    entries. Use the given location configuration dict to construct the destination path. If this
+    is a dry run, then don't actually dump anything.
+    '''
+    dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+    processes = []
+
+    logger.info('{}: Dumping SQLite databases{}'.format(log_prefix, dry_run_label))
+
+    for database in databases:
+        database_path = database['path']
+        database_filename = database['name']
+        dump_path = make_dump_path(location_config)
+        dump_filename = dump.make_database_dump_filename(dump_path, database_filename)
+        if os.path.exists(dump_filename):
+            logger.warning(
+                f'{log_prefix}: Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}'
+            )
+            continue
+
+        command = (
+            'sqlite3',
+            database_path,
+            '.dump',
+            '>',
+            dump_filename,
+        )
+        logger.debug(
+            f'{log_prefix}: Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}'
+        )
+        if dry_run:
+            continue
+
+        dump.create_parent_directory_for_dump(dump_filename)
+        processes.append(execute_command(command, shell=True, run_to_completion=False))
+
+    return processes
+
+
+def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # pragma: no cover
+    '''
+    Remove the given SQLite3 database dumps from the filesystem. The databases are supplied as a
+    sequence of configuration dicts, as per the configuration schema. Use the given log prefix in
+    any log entries. Use the given location configuration dict to construct the destination path.
+    If this is a dry run, then don't actually remove anything.
+    '''
+    dump.remove_database_dumps(make_dump_path(location_config), 'SQLite', log_prefix, dry_run)
+
+
+def make_database_dump_pattern(
+    databases, log_prefix, location_config, name=None
+):  # pragma: no cover
+    '''
+    Make a pattern that matches the given SQLite3 databases. The databases are supplied as a
+    sequence of configuration dicts, as per the configuration schema.
+    '''
+    return dump.make_database_dump_filename(make_dump_path(location_config), name)
+
+
+def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+    '''
+    Restore the given SQLite3 database from an extract stream. The database is supplied as a
+    one-element sequence containing a dict describing the database, as per the configuration schema.
+    Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
+    anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
+    output to consume.
+    '''
+    dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
+
+    if len(database_config) != 1:
+        raise ValueError('The database configuration value is invalid')
+
+    database_path = database_config[0]['path']
+
+    logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
+    if dry_run:
+        return
+
+    remove_command = (
+        'rm',
+        database_path,
+    )
+    try:
+        execute_command(remove_command, shell=True)
+    except CalledProcessError:
+        logger.info(f'{log_prefix}: Database does not exist at {database_path}, skipping removal')
+
+    restore_command = (
+        'sqlite3',
+        database_path,
+    )
+
+    # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
+    # if the restore paths don't exist in the archive.
+    execute_command_with_processes(
+        restore_command,
+        [extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+    )
diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md
index 1c7397b..5bf2002 100644
--- a/docs/how-to/backup-your-databases.md
+++ b/docs/how-to/backup-your-databases.md
@@ -15,8 +15,8 @@ consistent snapshot that is more suited for backups.
 
 Fortunately, borgmatic includes built-in support for creating database dumps
 prior to running backups. For example, here is everything you need to dump and
-backup a couple of local PostgreSQL databases, a MySQL/MariaDB database, and a
-MongoDB database:
+backup a couple of local PostgreSQL databases, a MySQL/MariaDB database, a
+MongoDB database and a SQLite database:
 
 ```yaml
 hooks:
@@ -27,6 +27,9 @@ hooks:
         - name: posts
     mongodb_databases:
         - name: messages
+    sqlite_databases:
+        - path: /var/lib/sqlite3/mydb.sqlite
+        - name: mydb
 ```
 
 As part of each backup, borgmatic streams a database dump for each configured
@@ -74,6 +77,9 @@ hooks:
           password: trustsome1
           authentication_database: mongousers
           options: "--ssl"
+    sqlite_databases:
+        - path: /var/lib/sqlite3/mydb.sqlite
+        - name: mydb
 ```
 
 See your [borgmatic configuration
@@ -97,7 +103,8 @@ hooks:
 ```
 
 Note that you may need to use a `username` of the `postgres` superuser for
-this to work with PostgreSQL.
+this to work with PostgreSQL. Also, the `all` database name is not supported
+for SQLite databases.
 
 <span class="minilink minilink-addedin">New in version 1.7.6</span> With
 PostgreSQL and MySQL, you can optionally dump "all" databases to separate
@@ -154,7 +161,7 @@ bring back any missing configuration files in order to restore a database.
 
 ## Supported databases
 
-As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, and MongoDB databases
+As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, MongoDB databases and SQLite databases
 directly. But see below about general-purpose preparation and cleanup hooks as
 a work-around with other database systems. Also, please [file a
 ticket](https://torsion.org/borgmatic/#issues) for additional database systems
diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py
index f981b40..3dcb82a 100644
--- a/tests/end-to-end/test_database.py
+++ b/tests/end-to-end/test_database.py
@@ -73,6 +73,9 @@ hooks:
           hostname: mongodb
           username: root
           password: test
+    sqlite_databases:
+        - path: /tmp/sqlite_test.db
+        - name: sqlite_test
 '''
 
     with open(config_path, 'w') as config_file:
diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py
new file mode 100644
index 0000000..15d5911
--- /dev/null
+++ b/tests/unit/hooks/test_sqlite.py
@@ -0,0 +1,75 @@
+import logging
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic.hooks import sqlite as module
+
+
+def test_dump_databases_logs_and_skips_if_dump_already_exists():
+    databases = [{'path': '/path/to/database', 'name': 'database'}]
+
+    flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
+    flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
+        '/path/to/dump/database'
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(logging).should_receive('info')
+    flexmock(logging).should_receive('warning')
+
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run = False) == []
+
+def test_dump_databases_dumps_each_database():
+    databases = [
+        {'path': '/path/to/database1', 'name': 'database1'},
+        {'path': '/path/to/database2', 'name': 'database2'},
+    ]
+
+    flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
+    flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
+        '/path/to/dump/database'
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(logging).should_receive('info')
+    flexmock(logging).should_receive('warning')
+    flexmock(module).should_receive('dump.create_parent_directory_for_dump')
+    flexmock(module).should_receive('execute_command').and_return('process')
+
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run = False) == ['process', 'process']
+
+def test_dump_databases_does_not_dump_if_dry_run():
+    databases = [{'path': '/path/to/database', 'name': 'database'}]
+
+    flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
+    flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
+        '/path/to/dump/database'
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(logging).should_receive('info')
+    flexmock(logging).should_receive('warning')
+    flexmock(module).should_receive('dump.create_parent_directory_for_dump')
+
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run = True) == []
+
+def test_restore_database_dump_restores_database():
+    database_config = [{'path': '/path/to/database', 'name': 'database'}]
+    extract_process = flexmock(stdout = flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').and_return('process')
+
+    module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = False, extract_process = extract_process)
+
+def test_restore_database_dump_does_not_restore_database_if_dry_run():
+    database_config = [{'path': '/path/to/database', 'name': 'database'}]
+    extract_process = flexmock(stdout = flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').never()
+
+    module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = True, extract_process = extract_process)
+
+def test_restore_database_dump_raises_error_if_database_config_is_invalid():
+    database_config = []
+    extract_process = flexmock(stdout = flexmock())
+
+    with pytest.raises(ValueError):
+        module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = False, extract_process = extract_process)
\ No newline at end of file

From 3aa88085ed7310b43380f3e4ec48449fb73f7692 Mon Sep 17 00:00:00 2001
From: Divyansh Singh <divyanshanandsingh@gmail.com>
Date: Fri, 3 Mar 2023 00:01:52 +0530
Subject: [PATCH 2/8] formatting fix

---
 tests/unit/hooks/test_sqlite.py | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py
index 15d5911..d679cad 100644
--- a/tests/unit/hooks/test_sqlite.py
+++ b/tests/unit/hooks/test_sqlite.py
@@ -17,7 +17,8 @@ def test_dump_databases_logs_and_skips_if_dump_already_exists():
     flexmock(logging).should_receive('info')
     flexmock(logging).should_receive('warning')
 
-    assert module.dump_databases(databases, 'test.yaml', {}, dry_run = False) == []
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == []
+
 
 def test_dump_databases_dumps_each_database():
     databases = [
@@ -35,7 +36,11 @@ def test_dump_databases_dumps_each_database():
     flexmock(module).should_receive('dump.create_parent_directory_for_dump')
     flexmock(module).should_receive('execute_command').and_return('process')
 
-    assert module.dump_databases(databases, 'test.yaml', {}, dry_run = False) == ['process', 'process']
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [
+        'process',
+        'process',
+    ]
+
 
 def test_dump_databases_does_not_dump_if_dry_run():
     databases = [{'path': '/path/to/database', 'name': 'database'}]
@@ -49,27 +54,36 @@ def test_dump_databases_does_not_dump_if_dry_run():
     flexmock(logging).should_receive('warning')
     flexmock(module).should_receive('dump.create_parent_directory_for_dump')
 
-    assert module.dump_databases(databases, 'test.yaml', {}, dry_run = True) == []
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == []
+
 
 def test_restore_database_dump_restores_database():
     database_config = [{'path': '/path/to/database', 'name': 'database'}]
-    extract_process = flexmock(stdout = flexmock())
+    extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('execute_command_with_processes').and_return('process')
 
-    module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = False, extract_process = extract_process)
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
 
 def test_restore_database_dump_does_not_restore_database_if_dry_run():
     database_config = [{'path': '/path/to/database', 'name': 'database'}]
-    extract_process = flexmock(stdout = flexmock())
+    extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('execute_command_with_processes').never()
 
-    module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = True, extract_process = extract_process)
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=True, extract_process=extract_process
+    )
+
 
 def test_restore_database_dump_raises_error_if_database_config_is_invalid():
     database_config = []
-    extract_process = flexmock(stdout = flexmock())
+    extract_process = flexmock(stdout=flexmock())
 
     with pytest.raises(ValueError):
-        module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = False, extract_process = extract_process)
\ No newline at end of file
+        module.restore_database_dump(
+            database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        )

From 903507bd03aed7942fd8434b7ba8f01005c7e97a Mon Sep 17 00:00:00 2001
From: Divyansh Singh <divyanshanandsingh@gmail.com>
Date: Sat, 4 Mar 2023 01:27:07 +0530
Subject: [PATCH 3/8] code review

---
 borgmatic/config/schema.yaml         | 18 +++++------
 borgmatic/hooks/sqlite.py            | 16 +++++-----
 docs/how-to/backup-your-databases.md |  9 +++---
 tests/unit/hooks/test_sqlite.py      | 46 ++++++++++++++++++----------
 4 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index 6f399ed..ac8cd28 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -938,24 +938,24 @@ properties:
                     required: ['path','name']
                     additionalProperties: false
                     properties:
+                        name:
+                            type: string
+                            description: |
+                                This is used to tag the database dump file 
+                                with a name. It is not the path to the database 
+                                file itself. The name "all" has no special 
+                                meaning for SQLite databases.
+                            example: users
                         path:
                             type: string
                             description: |
                                 Path to the SQLite database file to dump. If
                                 relative, it is relative to the current working
-                                directory. If absolute, it is relative to the
-                                root of the filesystem. Note that using this
+                                directory. Note that using this
                                 database hook implicitly enables both
                                 read_special and one_file_system (see above) to
                                 support dump and restore streaming.
                             example: /var/lib/sqlite/users.db
-                        name:
-                            type: string
-                            description: |
-                                This is used to tag the database dump file with
-                                a name. It is not used to identify the database
-                                file itself.
-                            example: users
             mongodb_databases:
                 type: array
                 items:
diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py
index f9b83ae..d31e331 100644
--- a/borgmatic/hooks/sqlite.py
+++ b/borgmatic/hooks/sqlite.py
@@ -17,7 +17,7 @@ def make_dump_path(location_config):  # pragma: no cover
     )
 
 
-def dump_databases(databases, log_prefix, location_config, dry_run):  # pragma: no cover
+def dump_databases(databases, log_prefix, location_config, dry_run):
     '''
     Dump the given SQLite3 databases to a file. The databases are supplied as a sequence of
     configuration dicts, as per the configuration schema. Use the given log prefix in any log
@@ -29,11 +29,13 @@ def dump_databases(databases, log_prefix, location_config, dry_run):  # pragma:
 
     logger.info('{}: Dumping SQLite databases{}'.format(log_prefix, dry_run_label))
 
+    if databases[0]['name'] == 'all':
+        logger.warning('The "all" database name has no meaning for SQLite3 databases')
+
     for database in databases:
         database_path = database['path']
-        database_filename = database['name']
         dump_path = make_dump_path(location_config)
-        dump_filename = dump.make_database_dump_filename(dump_path, database_filename)
+        dump_filename = dump.make_database_dump_filename(dump_path, database['name'])
         if os.path.exists(dump_filename):
             logger.warning(
                 f'{log_prefix}: Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}'
@@ -98,13 +100,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     if dry_run:
         return
 
-    remove_command = (
-        'rm',
-        database_path,
-    )
     try:
-        execute_command(remove_command, shell=True)
-    except CalledProcessError:
+        execute_command(('rm', database_path), shell=True)
+    except CalledProcessError:  # pragma: no cover
         logger.info(f'{log_prefix}: Database does not exist at {database_path}, skipping removal')
 
     restore_command = (
diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md
index 5bf2002..3e1cf2a 100644
--- a/docs/how-to/backup-your-databases.md
+++ b/docs/how-to/backup-your-databases.md
@@ -28,8 +28,8 @@ hooks:
     mongodb_databases:
         - name: messages
     sqlite_databases:
-        - path: /var/lib/sqlite3/mydb.sqlite
         - name: mydb
+          path: /var/lib/sqlite3/mydb.sqlite          
 ```
 
 As part of each backup, borgmatic streams a database dump for each configured
@@ -78,8 +78,8 @@ hooks:
           authentication_database: mongousers
           options: "--ssl"
     sqlite_databases:
-        - path: /var/lib/sqlite3/mydb.sqlite
         - name: mydb
+          path: /var/lib/sqlite3/mydb.sqlite
 ```
 
 See your [borgmatic configuration
@@ -103,8 +103,7 @@ hooks:
 ```
 
 Note that you may need to use a `username` of the `postgres` superuser for
-this to work with PostgreSQL. Also, the `all` database name is not supported
-for SQLite databases.
+this to work with PostgreSQL.
 
 <span class="minilink minilink-addedin">New in version 1.7.6</span> With
 PostgreSQL and MySQL, you can optionally dump "all" databases to separate
@@ -161,7 +160,7 @@ bring back any missing configuration files in order to restore a database.
 
 ## Supported databases
 
-As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, MongoDB databases and SQLite databases
+As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, MongoDB and SQLite databases
 directly. But see below about general-purpose preparation and cleanup hooks as
 a work-around with other database systems. Also, please [file a
 ticket](https://torsion.org/borgmatic/#issues) for additional database systems
diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py
index d679cad..f9e09e6 100644
--- a/tests/unit/hooks/test_sqlite.py
+++ b/tests/unit/hooks/test_sqlite.py
@@ -10,12 +10,10 @@ def test_dump_databases_logs_and_skips_if_dump_already_exists():
     databases = [{'path': '/path/to/database', 'name': 'database'}]
 
     flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
-    flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         '/path/to/dump/database'
     )
     flexmock(module.os.path).should_receive('exists').and_return(True)
-    flexmock(logging).should_receive('info')
-    flexmock(logging).should_receive('warning')
 
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == []
 
@@ -25,34 +23,49 @@ def test_dump_databases_dumps_each_database():
         {'path': '/path/to/database1', 'name': 'database1'},
         {'path': '/path/to/database2', 'name': 'database2'},
     ]
+    processes = [flexmock(), flexmock()]
 
     flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
-    flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         '/path/to/dump/database'
     )
     flexmock(module.os.path).should_receive('exists').and_return(False)
-    flexmock(logging).should_receive('info')
-    flexmock(logging).should_receive('warning')
-    flexmock(module).should_receive('dump.create_parent_directory_for_dump')
-    flexmock(module).should_receive('execute_command').and_return('process')
+    flexmock(module.dump).should_receive('create_parent_directory_for_dump')
+    flexmock(module).should_receive('execute_command').and_return(processes[0]).and_return(
+        processes[1]
+    )
 
-    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [
-        'process',
-        'process',
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
+
+
+def test_dumping_database_with_name_all_warns_and_dumps_all_databases():
+    databases = [
+        {'path': '/path/to/database1', 'name': 'all'},
     ]
+    processes = [flexmock()]
+
+    flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
+    flexmock(logging).should_receive('warning')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+        '/path/to/dump/database'
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.dump).should_receive('create_parent_directory_for_dump')
+    flexmock(module).should_receive('execute_command').and_return(processes[0])
+
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
 
 
 def test_dump_databases_does_not_dump_if_dry_run():
     databases = [{'path': '/path/to/database', 'name': 'database'}]
 
     flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
-    flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         '/path/to/dump/database'
     )
     flexmock(module.os.path).should_receive('exists').and_return(False)
-    flexmock(logging).should_receive('info')
-    flexmock(logging).should_receive('warning')
-    flexmock(module).should_receive('dump.create_parent_directory_for_dump')
+    flexmock(module.dump).should_receive('create_parent_directory_for_dump').never()
+    flexmock(module).should_receive('execute_command').never()
 
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == []
 
@@ -61,7 +74,8 @@ def test_restore_database_dump_restores_database():
     database_config = [{'path': '/path/to/database', 'name': 'database'}]
     extract_process = flexmock(stdout=flexmock())
 
-    flexmock(module).should_receive('execute_command_with_processes').and_return('process')
+    flexmock(module).should_receive('execute_command_with_processes').once()
+    flexmock(module).should_receive('execute_command').once()
 
     module.restore_database_dump(
         database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process

From 767a7d900b35ada478867d6aafb76ee6abda248f Mon Sep 17 00:00:00 2001
From: Divyansh Singh <divyanshanandsingh@gmail.com>
Date: Sat, 4 Mar 2023 01:29:01 +0530
Subject: [PATCH 4/8] e2e tests schema update

---
 tests/end-to-end/test_database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py
index 3dcb82a..8849b3c 100644
--- a/tests/end-to-end/test_database.py
+++ b/tests/end-to-end/test_database.py
@@ -74,8 +74,8 @@ hooks:
           username: root
           password: test
     sqlite_databases:
-        - path: /tmp/sqlite_test.db
         - name: sqlite_test
+          path: /tmp/sqlite_test.db
 '''
 
     with open(config_path, 'w') as config_file:

From 1793ad74bdf4ce485b9aeb7c10e1995dee54cbeb Mon Sep 17 00:00:00 2001
From: Divyansh Singh <divyanshanandsingh@gmail.com>
Date: Sat, 4 Mar 2023 02:41:14 +0530
Subject: [PATCH 5/8] add sqlite for e2e tests

---
 scripts/run-full-tests | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run-full-tests b/scripts/run-full-tests
index 993c5bd..58e5ac6 100755
--- a/scripts/run-full-tests
+++ b/scripts/run-full-tests
@@ -11,7 +11,7 @@
 set -e
 
 apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \
-    py3-ruamel.yaml py3-ruamel.yaml.clib bash
+    py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite
 # If certain dependencies of black are available in this version of Alpine, install them.
 apk add --no-cache py3-typed-ast py3-regex || true
 python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1

From 675e54ba9f2b8001c11550aa6f46a7ab48f2de43 Mon Sep 17 00:00:00 2001
From: Divyansh Singh <divyanshanandsingh@gmail.com>
Date: Sat, 4 Mar 2023 12:43:07 +0530
Subject: [PATCH 6/8] use os.remove and improve tests

---
 borgmatic/hooks/sqlite.py       | 18 +++++++++++-------
 tests/unit/hooks/test_sqlite.py | 27 +++++++++++++++++++++++----
 2 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py
index d31e331..b4937e1 100644
--- a/borgmatic/hooks/sqlite.py
+++ b/borgmatic/hooks/sqlite.py
@@ -1,6 +1,5 @@
 import logging
 import os
-from subprocess import CalledProcessError
 
 from borgmatic.execute import execute_command, execute_command_with_processes
 from borgmatic.hooks import dump
@@ -29,11 +28,16 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
 
     logger.info('{}: Dumping SQLite databases{}'.format(log_prefix, dry_run_label))
 
-    if databases[0]['name'] == 'all':
-        logger.warning('The "all" database name has no meaning for SQLite3 databases')
-
     for database in databases:
         database_path = database['path']
+
+        if database['name'] == 'all':
+            logger.warning('The "all" database name has no meaning for SQLite3 databases')
+        if not os.path.exists(database_path):
+            logger.warning(
+                f'{log_prefix}: No SQLite database at {database_path}; An empty database will be created and dumped'
+            )
+
         dump_path = make_dump_path(location_config)
         dump_filename = dump.make_database_dump_filename(dump_path, database['name'])
         if os.path.exists(dump_filename):
@@ -101,9 +105,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         return
 
     try:
-        execute_command(('rm', database_path), shell=True)
-    except CalledProcessError:  # pragma: no cover
-        logger.info(f'{log_prefix}: Database does not exist at {database_path}, skipping removal')
+        os.remove(database_path)
+    except FileNotFoundError:  # pragma: no cover
+        pass
 
     restore_command = (
         'sqlite3',
diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py
index f9e09e6..d5050fe 100644
--- a/tests/unit/hooks/test_sqlite.py
+++ b/tests/unit/hooks/test_sqlite.py
@@ -1,5 +1,3 @@
-import logging
-
 import pytest
 from flexmock import flexmock
 
@@ -14,6 +12,8 @@ def test_dump_databases_logs_and_skips_if_dump_already_exists():
         '/path/to/dump/database'
     )
     flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.dump).should_receive('create_parent_directory_for_dump').never()
+    flexmock(module).should_receive('execute_command').never()
 
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == []
 
@@ -38,6 +38,24 @@ def test_dump_databases_dumps_each_database():
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
 
 
+def test_dumping_database_with_non_existent_path_warns_and_dumps_database():
+    databases = [
+        {'path': '/path/to/database1', 'name': 'database1'},
+    ]
+    processes = [flexmock()]
+
+    flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
+    flexmock(module.logger).should_receive('warning').once()
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+        '/path/to/dump/database'
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.dump).should_receive('create_parent_directory_for_dump')
+    flexmock(module).should_receive('execute_command').and_return(processes[0])
+
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
+
+
 def test_dumping_database_with_name_all_warns_and_dumps_all_databases():
     databases = [
         {'path': '/path/to/database1', 'name': 'all'},
@@ -45,7 +63,9 @@ def test_dumping_database_with_name_all_warns_and_dumps_all_databases():
     processes = [flexmock()]
 
     flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
-    flexmock(logging).should_receive('warning')
+    flexmock(module.logger).should_receive(
+        'warning'
+    ).twice()  # once for the name=all, once for the non-existent path
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         '/path/to/dump/database'
     )
@@ -75,7 +95,6 @@ def test_restore_database_dump_restores_database():
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('execute_command_with_processes').once()
-    flexmock(module).should_receive('execute_command').once()
 
     module.restore_database_dump(
         database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process

From c71eb60cd253d9d1ce70ad1ffefb86610ee0231d Mon Sep 17 00:00:00 2001
From: Divyansh Singh <divyanshanandsingh@gmail.com>
Date: Sat, 4 Mar 2023 13:08:30 +0530
Subject: [PATCH 7/8] mock os.remove instead of actually removing a file

---
 borgmatic/hooks/sqlite.py       | 3 ++-
 tests/unit/hooks/test_sqlite.py | 3 +++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py
index b4937e1..3217f33 100644
--- a/borgmatic/hooks/sqlite.py
+++ b/borgmatic/hooks/sqlite.py
@@ -105,7 +105,8 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         return
 
     try:
-        os.remove(database_path)
+        os.remove('/home/divyansh/Desktop/hello.txt')
+        logger.warn(f'{log_prefix}: Removed existing SQLite database at {database_path}')
     except FileNotFoundError:  # pragma: no cover
         pass
 
diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py
index d5050fe..a660d81 100644
--- a/tests/unit/hooks/test_sqlite.py
+++ b/tests/unit/hooks/test_sqlite.py
@@ -96,6 +96,8 @@ def test_restore_database_dump_restores_database():
 
     flexmock(module).should_receive('execute_command_with_processes').once()
 
+    flexmock(module.os).should_receive('remove').once()
+
     module.restore_database_dump(
         database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
     )
@@ -106,6 +108,7 @@ def test_restore_database_dump_does_not_restore_database_if_dry_run():
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('execute_command_with_processes').never()
+    flexmock(module.os).should_receive('remove').never()
 
     module.restore_database_dump(
         database_config, 'test.yaml', {}, dry_run=True, extract_process=extract_process

From cf0275a3ed4c524444731f573b2c1976ad840548 Mon Sep 17 00:00:00 2001
From: Divyansh Singh <divyanshanandsingh@gmail.com>
Date: Sat, 4 Mar 2023 23:00:57 +0530
Subject: [PATCH 8/8] remove test path

---
 borgmatic/hooks/sqlite.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py
index 3217f33..637e7e9 100644
--- a/borgmatic/hooks/sqlite.py
+++ b/borgmatic/hooks/sqlite.py
@@ -105,7 +105,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         return
 
     try:
-        os.remove('/home/divyansh/Desktop/hello.txt')
+        os.remove(database_path)
         logger.warn(f'{log_prefix}: Removed existing SQLite database at {database_path}')
     except FileNotFoundError:  # pragma: no cover
         pass