Merge pull request #2726 from pi-hole/development

Pi-hole FTL v6.4
2025-12-19 20:58:24 +00:00 · 2025-11-27 18:02:19 +00:00
parent a0186fd468 3a2647d1f3
commit 8d1add8d99
45 changed files with 6142 additions and 2914 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -32,7 +32,7 @@ jobs:
    steps:
      -
        name: Checkout code
-        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0
+        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0
      -
        name: "Calculate required variables"
        id: variables
@@ -100,11 +100,11 @@ jobs:
    steps:
      -
        name: Checkout code
-        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0
+        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0
      # QEMU should come before Buildx
      -
        name: Set up QEMU
-        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 #v3.6.0
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 #v3.7.0
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 #v3.11.1
@@ -265,7 +265,7 @@ jobs:
      -
        name: Attach binaries to release
        if: github.event_name == 'release'
-        uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 #v2.4.1
+        uses: softprops/action-gh-release@5be0e66d93ac7ed76da52eca8bb058f665c3a5fe #v2.4.2
        with:
          tag_name: ${{ github.event.release.tag_name }}
          files: |
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -58,7 +58,7 @@ jobs:
        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
    steps:
    - name: Checkout repository
-      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0
+      uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0

    - name: Install dependencies
      run: |
@@ -85,7 +85,7 @@ jobs:

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
-      uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee #v4.31.2
+      uses: github/codeql-action/init@e12f0178983d466f2f6028f5cc7a6d786fd97f4b #v4.31.4
      with:
        languages: ${{ matrix.language }}
        build-mode: ${{ matrix.build-mode }}
@@ -108,7 +108,7 @@ jobs:
        ./build.sh

    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee #v4.31.2
+      uses: github/codeql-action/analyze@e12f0178983d466f2f6028f5cc7a6d786fd97f4b #v4.31.4
      with:
        category: "/language:${{matrix.language}}"
        upload: failure-only # upload only in case of failure, otherwise upload later after filtering
@@ -134,7 +134,7 @@ jobs:
        output: codeql-results/cpp.sarif

    - name: Upload SARIF
-      uses: github/codeql-action/upload-sarif@0499de31b99561a6d14a36a5f662c2a54f91beee #v4.31.2
+      uses: github/codeql-action/upload-sarif@e12f0178983d466f2f6028f5cc7a6d786fd97f4b #v4.31.4
      with:
        sarif_file: codeql-results/cpp.sarif

--- a/.github/workflows/codespell.yml
+++ b/.github/workflows/codespell.yml
@@ -13,10 +13,10 @@ jobs:
    steps:
    -
      name: Checkout repository
-      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0
+      uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0
    -
      name: Spell-Checking
-      uses: codespell-project/actions-codespell@406322ec52dd7b488e48c1c4b82e2a8b3a1bf630 #v2.1
+      uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 #v2.2
      with:
        ignore_words_file: .github/.codespellignore
        skip: ./src/database/sqlite3.c,./src/database/sqlite3.h,./src/database/shell.c,./src/lua,./src/dnsmasq,./src/tre-regex,./.git,./test/libs,./src/webserver/civetweb,./src/zip/miniz,./src/api/docs/content/external,./src/database/sqlite3_rsync.c,./package-lock.json,./src/config/tomlc17
--- a/.github/workflows/openapi-validator.yml
+++ b/.github/workflows/openapi-validator.yml
@@ -12,7 +12,7 @@ jobs:

    steps:
      - name: Clone repository
-        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0
+        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0

      - name: Set up Node.js
        uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 #v6.0.0
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -40,7 +40,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
-        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0
+        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0
      - name: Remove 'stale' label
        run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }}
        env:
--- a/.github/workflows/sync-back-to-dev.yml
+++ b/.github/workflows/sync-back-to-dev.yml
@@ -11,7 +11,7 @@ jobs:
    name: Syncing branches
    steps:
      - name: Checkout
-        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0
+        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0
      - name: Opening pull request
        run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal'
        env:
--- a/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch
+++ b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch
@@ -20,10 +20,10 @@ index 6280ebf6..a5e82f70 100644
 #else
 int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   char **argv;
-@@ -33467,6 +33469,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
+@@ -33656,6 +33658,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
+     if( stdin_is_interactive ){
       char *zHome;
       char *zHistory;
-       int nHistory;
 +      print_FTL_version();
       sqlite3_fprintf(stdout,
             "SQLite version %s %.19s\n" /*extra-version-info*/
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -40,7 +40,9 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
 # HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk.
 # SQLITE_DEFAULT_WORKER_THREADS=0: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. Do not increase this value as it, ironically, can cause performance degradation and definitely increases total memory usage.
 # SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL.
-set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=1 -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_H -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=0 -DSQLITE_MAX_PREPARE_RETRY=200")
+# SQLITE_ENABLE_CARRAY: Enable the carray virtual table module
+# SQLITE_ENABLE_PERCENTILE: Enable the percentile aggregate function
+set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=1 -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_H -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=0 -DSQLITE_MAX_PREPARE_RETRY=200 -DSQLITE_ENABLE_CARRAY=1 -DSQLITE_ENABLE_PERCENTILE=1")

 # Code hardening and debugging improvements
 # -fstack-protector-strong: The program will be resistant to having its stack overflowed
@@ -216,7 +218,7 @@ endif()
 set(CMAKE_C_FLAGS "-std=c17 -pipe ${WARN_FLAGS} -D_FILE_OFFSET_BITS=64 ${HARDENING_FLAGS} ${DEBUG_FLAGS} ${CMAKE_C_FLAGS} -DHAVE_POLL_H ${SQLITE_DEFINES}")

 set(CMAKE_C_FLAGS_DEBUG "-O0 -g3")
-set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG")
+set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG -funroll-loops")
 set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELEASE} -g3")
 set(CMAKE_C_FLAGS_MINSIZEREL "-Os -DNDEBUG")

--- a/src/api/2fa.c
+++ b/src/api/2fa.c
@@ -195,17 +195,21 @@ static bool encode_uint8_t_array_to_base32(const uint8_t *in, const size_t in_le
 	return true;
 }

+static time_t last_attempt = 0;
 static uint32_t last_code = 0;
 enum totp_status verifyTOTP(const uint32_t incode)
 {
+	// Only one attempt per second is allowed
+	const time_t now = time(NULL);
+	if(now == last_attempt)
+		return TOTP_RATE_LIMIT;
+	last_attempt = now;
+
 	// Decode base32 secret
 	uint8_t decoded_secret[RFC6238_SECRET_LEN];
 	if(!decode_base32_to_uint8_array(config.webserver.api.totp_secret.v.s, decoded_secret, sizeof(decoded_secret)))
 		return false;

-	// Get current time
-	const time_t now = time(NULL);
-
 	// Verify code for the previous, the current and the next time step
 	for(int i = -1; i <= 1; i++)
 	{
--- a/src/api/action.c
+++ b/src/api/action.c
@@ -121,7 +121,15 @@ static int run_and_stream_command(struct ftl_conn *api, const char *path, const

 int api_action_gravity(struct ftl_conn *api)
 {
-	return run_and_stream_command(api, "/usr/local/bin/pihole", (const char *const []){ "pihole", "-g", NULL }, "FORCE_COLOR");
+	// Only set FORCE_COLOR if the client explicitly requests it via "color=true" query parameter
+	// This prevents ANSI escape codes from being included in the output for API consumers that don't need them
+	bool color = false;
+	const char *query = api->request != NULL ? api->request->query_string : "";
+	if(query != NULL)
+		get_bool_var(query, "color", &color);
+
+	const char *extra_env = color ? "FORCE_COLOR" : NULL;
+	return run_and_stream_command(api, "/usr/local/bin/pihole", (const char *const []){ "pihole", "-g", NULL }, extra_env);
 }

 int api_action_restartDNS(struct ftl_conn *api)
--- a/src/api/api.h
+++ b/src/api/api.h
@@ -114,6 +114,7 @@ enum totp_status {
 	TOTP_INVALID,
 	TOTP_CORRECT,
 	TOTP_REUSED,
+	TOTP_RATE_LIMIT
 } __attribute__ ((packed));
 enum totp_status verifyTOTP(const uint32_t code);
 int generateTOTP(struct ftl_conn *api);
--- a/src/api/auth.c
+++ b/src/api/auth.c
@@ -559,6 +559,14 @@ int api_auth(struct ftl_conn *api)
 				                       "Reused 2FA token",
 				                       "wait for new token");
 			}
+			else if(totp == TOTP_RATE_LIMIT)
+			{
+				// 2FA validation requested too often
+				return send_json_error(api, 429,
+				                       "rate_limiting",
+				                       "Rate-limiting 2FA token requests, try again later",
+				                       NULL);
+			}
 			else if(totp != TOTP_CORRECT)
 			{
 				// 2FA token is invalid
--- a/src/api/config.c
+++ b/src/api/config.c
@@ -22,7 +22,7 @@
 #include "config/toml_writer.h"
 // write_dnsmasq_config()
 #include "config/dnsmasq_config.h"
-// shm_lock()
+// lock_shm()
 #include "shmem.h"
 // hash_password()
 #include "config/password.h"
@@ -500,8 +500,8 @@ int get_json_config(struct ftl_conn *api, cJSON *json, const bool detailed)
 				continue;
 			// Check equality of paths up to the requested level (if any)
 			// Examples:
-			//  requested was /config/dnsmasq -> skip all entries that do not start in dnsmasq.
-			//  requested was /config/dnsmasq/dhcp -> skip all entries that do not start in dhcp
+			//  requested was /config/dns -> skip all entries that do not start in dns
+			//  requested was /config/dns/dhcp -> skip all entries that do not start in dhcp
 			//  etc.
 			if(!check_paths_equal(conf_item->p, requested_path, min_level - 1))
 				continue;
--- a/src/api/dhcp.c
+++ b/src/api/dhcp.c
@@ -45,11 +45,11 @@ int api_dhcp_leases_GET(struct ftl_conn *api)

 		// Parse line
 		unsigned long expires = 0;
-		char hwaddr[18] = { 0 };
+		char hwaddr[48] = { 0 };
 		char ip[INET6_ADDRSTRLEN] = { 0 };
 		char name[65] = { 0 };
 		char clientid[765] = { 0 };
-		const int ret = sscanf(line, "%lu %17s %45s %64s %764s", &expires, hwaddr, ip, name, clientid);
+		const int ret = sscanf(line, "%lu %47s %45s %64s %764s", &expires, hwaddr, ip, name, clientid);

 		// Skip invalid lines
 		if(ret != 5)
--- a/src/api/docs/content/specs/action.yaml
+++ b/src/api/docs/content/specs/action.yaml
@@ -4,11 +4,13 @@ components:
    gravity:
      post:
        summary: Run gravity
+        parameters:
+          - $ref: 'action.yaml#/components/parameters/color'
        tags:
          - Actions
        operationId: "action_gravity"
        description: |
-          Update Pi-hole's adlists by running `pihole -g`. The output of the process is streamed with chunked encoding.
+          Update Pi-hole's adlists by running `pihole -g`. The output of the process is streamed with chunked encoding. Use the optional `color` query parameter to include ANSI color escape codes in the output.
        responses:
          '200':
            description: OK
@@ -183,6 +185,17 @@ components:
                    - $ref: 'action.yaml#/components/errors/forbidden'
                    - $ref: 'common.yaml#/components/schemas/took'

+  parameters:
+    color:
+      name: color
+      in: query
+      description: Include ANSI color escape codes in the streamed output. Defaults to false to prevent colored output for API consumers that don't need formatting.
+      schema:
+        type: boolean
+        default: false
+        example: true
+      required: false
+
  errors:
    forbidden:
      description: |
--- a/src/api/docs/content/specs/auth.yaml
+++ b/src/api/docs/content/specs/auth.yaml
@@ -117,6 +117,8 @@ components:
                    $ref: 'auth.yaml#/components/examples/errors/rate_limit'
                  no_seats:
                    $ref: 'auth.yaml#/components/examples/errors/no_seats'
+                  totp_rate_limit:
+                    $ref: 'auth.yaml#/components/examples/errors/totp_rate_limit'
      delete:
        summary: Delete session
        tags:
@@ -588,6 +590,14 @@ components:
            message: "Reused 2FA token"
            hint: "wait for new token"
          took: 0.003
+      totp_rate_limit:
+        summary: 2FA token requested too soon
+        value:
+          error:
+            key: "rate_limiting"
+            message: "Rate-limiting 2FA token requests, try again later"
+            hint: null
+          took: 0.001
      rate_limit:
        summary: Rate limit exceeded
        value:
--- a/src/api/docs/content/specs/config.yaml
+++ b/src/api/docs/content/specs/config.yaml
@@ -625,6 +625,8 @@ components:
                  type: boolean
                netlink:
                  type: boolean
+                timing:
+                  type: boolean
                all:
                  type: boolean
    topics:
@@ -874,6 +876,7 @@ components:
            reserved: false
            ntp: false
            netlink: false
+            timing: false
            all: false
          took: 0.003
    config_one:
@@ -919,7 +922,7 @@ components:
            error:
              key: "bad_request"
              message: "Invalid path depth"
-              hint: "Use, e.g., DELETE /config/dnsmasq/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams"
+              hint: "Use, e.g., DELETE /config/dns/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams"
            took: 0.003
        item_already_present:
          summary: Item to be added exists already
@@ -946,7 +949,7 @@ components:
        type: string
      required: true
      description: config element
-      example: "dnsmasq/upstreams"
+      example: "dns/upstreams"
    value:
      in: path
      name: value
--- a/src/api/docs/content/specs/info.yaml
+++ b/src/api/docs/content/specs/info.yaml
@@ -994,8 +994,20 @@ components:
                  example: "pihole"
        queries:
          type: integer
-          description: Number of queries in long-term database
+          description: Number of queries in in-memory database
          example: 536956
+        earliest_timestamp:
+          type: number
+          description: Unix timestamp of the earliest query in the in-memory database (Defaults to 0.0 if no queries are stored in memory)
+          example: 1611742120.5
+        queries_disk:
+          type: integer
+          description: Number of queries in on-disk database
+          example: 1234567
+        earliest_timestamp_disk:
+          type: number
+          description: Unix timestamp of the earliest query in the on-disk database (Defaults to 0.0 if no queries are stored on disk)
+          example: 1608450520.3
        sqlite_version:
          type: string
          description: Version of embedded SQLite3 engine
--- a/src/api/docs/content/specs/search.yaml
+++ b/src/api/docs/content/specs/search.yaml
@@ -162,7 +162,7 @@ components:
              properties:
                partial:
                  type: boolean
-                  description: Whether partial matching was requested
+                  description: Whether partial matching was requested. Note that partial matching may not find complex regex entries.
                  example: false
                N:
                  type: integer
--- a/src/api/info.c
+++ b/src/api/info.c
@@ -146,9 +146,17 @@ int api_info_database(struct ftl_conn *api)
 	JSON_ADD_ITEM_TO_OBJECT(owner, "group", group);
 	JSON_ADD_ITEM_TO_OBJECT(json, "owner", owner);

-	// Add number of queries in on-disk database
-	const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage");
+	// Add number of queries and earliest timestamp in in-memory database
+	double earliest_timestamp_mem = 0.0;
+	const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage", &earliest_timestamp_mem);
 	JSON_ADD_NUMBER_TO_OBJECT(json, "queries", queries_in_database);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "earliest_timestamp", earliest_timestamp_mem);
+
+	// Add number of queries and earliest timestamp in on-disk database
+	double earliest_timestamp_disk = 0.0;
+	const int queries_in_disk_database = get_number_of_queries_in_DB(NULL, "disk.query_storage", &earliest_timestamp_disk);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "queries_disk", queries_in_disk_database);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "earliest_timestamp_disk", earliest_timestamp_disk);

 	// Add SQLite library version
 	JSON_REF_STR_IN_OBJECT(json, "sqlite_version", get_sqlite3_version());
--- a/src/api/search.c
+++ b/src/api/search.c
@@ -29,6 +29,7 @@ static int search_table(struct ftl_conn *api, const char *item,
 	if(ids != NULL)
 	{
 		// Set item to NULL to indicate that we are searching for IDs
+		// instead of domains
 		item = NULL;
 		// Strip "[" and "]" from ids
 		ids[strlen(ids)-1] = '\0';
@@ -208,6 +209,19 @@ int api_search(struct ftl_conn *api)
 		return ret;
 	}

+	// Match partially against regex filters using LIKE '%term%' matching
+	// (works only for simple regexes)
+	unsigned int Nregex = 0u;
+	if(partial)
+	{
+		ret = search_table(api, punycode, GRAVITY_DOMAINLIST_ALL_REGEX, NULL, limit, &Nregex, partial, domains);
+		if(ret != 200)
+		{
+			free(punycode);
+			return ret;
+		}
+	}
+
 	// Search through gravity
 	cJSON *gravity = JSON_NEW_ARRAY();
 	cJSON *gravity_patterns = NULL;
@@ -236,11 +250,10 @@ int api_search(struct ftl_conn *api)
 	cJSON *allow_ids = cJSON_GetObjectItem(regex_ids, "allow");

 	// Get allow regex filters
-	unsigned int Nregex = 0u;
 	if(cJSON_GetArraySize(allow_ids) > 0)
 	{
 		char *allow_list = cJSON_PrintUnformatted(allow_ids);
-		ret = search_table(api,punycode, GRAVITY_DOMAINLIST_ALLOW_REGEX, allow_list, limit, &Nregex, false, domains);
+		ret = search_table(api, NULL, GRAVITY_DOMAINLIST_ALLOW_REGEX, allow_list, limit, &Nregex, false, domains);
 		free(allow_list);
 		if(ret != 200)
 		{
@@ -252,7 +265,7 @@ int api_search(struct ftl_conn *api)
 	if(cJSON_GetArraySize(deny_ids) > 0)
 	{
 		char *deny_list = cJSON_PrintUnformatted(deny_ids);
-		ret = search_table(api, punycode, GRAVITY_DOMAINLIST_DENY_REGEX, deny_list, limit, &Nregex, false, domains);
+		ret = search_table(api, NULL, GRAVITY_DOMAINLIST_DENY_REGEX, deny_list, limit, &Nregex, false, domains);
 		free(deny_list);
 		if(ret != 200)
 		{
--- a/src/args.c
+++ b/src/args.c
@@ -412,6 +412,7 @@ void parse_args(int argc, char *argv[])
 	// Generate X.509 certificate
 	if(argc > 1 && strcmp(argv[1], "--gen-x509") == 0)
 	{
+#ifdef HAVE_MBEDTLS
 		if(argc < 3 || argc > 5)
 		{
 			printf("Usage: %s --gen-x509 <output file> [<domain>] [rsa]\n", argv[0]);
@@ -431,6 +432,10 @@ void parse_args(int argc, char *argv[])
 		const bool rsa = argc > 4 && strcasecmp(argv[4], "rsa") == 0;

 		exit(generate_certificate(argv[2], rsa, domain, config.webserver.tls.validity.v.ui) ? EXIT_SUCCESS : EXIT_FAILURE);
+#else
+		printf("Error: FTL was compiled without TLS support. Certificate generation is not available.\n");
+		exit(EXIT_FAILURE);
+#endif
 	}

 	// Parse X.509 certificate
@@ -438,6 +443,7 @@ void parse_args(int argc, char *argv[])
 	  (strcmp(argv[1], "--read-x509") == 0 ||
 	   strcmp(argv[1], "--read-x509-key") == 0))
 	{
+#ifdef HAVE_MBEDTLS
 		if(argc > 4)
 		{
 			printf("Usage: %s %s [<input file>] [<domain>]\n", argv[0], argv[1]);
@@ -480,6 +486,10 @@ void parse_args(int argc, char *argv[])
 			printf("Certificate does not match domain %s\n", argv[3]);
 			exit(EXIT_FAILURE);
 		}
+#else
+		printf("Error: FTL was compiled without TLS support. Certificate reading is not available.\n");
+		exit(EXIT_FAILURE);
+#endif
 	}

 	// If the first argument is "gravity" (e.g., /usr/bin/pihole-FTL gravity),
--- a/src/config/config.c
+++ b/src/config/config.c
@@ -1625,6 +1625,12 @@ void initConfig(struct config *conf)
 	conf->debug.netlink.d.b = false;
 	conf->debug.netlink.c = validate_stub; // Only type-based checking

+	conf->debug.timing.k = "debug.timing";
+	conf->debug.timing.h = "Print timing information from various parts of FTL";
+	conf->debug.timing.t = CONF_BOOL;
+	conf->debug.timing.d.b = false;
+	conf->debug.timing.c = validate_stub; // Only type-based checking
+
 	conf->debug.all.k = "debug.all";
 	conf->debug.all.h = "Set all debug flags at once. This is a convenience option to enable all debug flags at once. Note that this option is not persistent, setting it to true will enable all *remaining* debug flags but unsetting it will disable *all* debug flags.";
 	conf->debug.all.t = CONF_ALL_DEBUG_BOOL;
--- a/src/config/config.h
+++ b/src/config/config.h
@@ -358,6 +358,7 @@ struct config {
 		struct conf_item reserved;
 		struct conf_item ntp;
 		struct conf_item netlink;
+		struct conf_item timing;
 		// all must be the last item in this struct
 		struct conf_item all;
 	} debug;
--- a/src/database/CMakeLists.txt
+++ b/src/database/CMakeLists.txt
@@ -22,8 +22,6 @@ set(sqlite3_sources
 add_library(sqlite3 OBJECT ${sqlite3_sources})
 # Define the function that will be called to initialize the sqlite3 shell
 target_compile_definitions(sqlite3 PRIVATE "-DSQLITE_SHELL_INIT_PROC=pihole_sqlite3_initalize")
-# Define that we want to statically link the percentile extension into the sqlite3 library
-target_compile_definitions(sqlite3 PRIVATE "-DSQLITE_STATIC_PERCENTILE=1")
 target_compile_options(sqlite3 PRIVATE -Wno-implicit-fallthrough -Wno-cast-function-type -Wno-sign-compare -Wno-implicit-function-declaration -Wno-int-conversion)

 if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
--- a/src/database/common.h
+++ b/src/database/common.h
@@ -89,4 +89,30 @@ extern const char *sqlite3ErrName(int rc);
 	}\
 }

+// Macro to time a database operation expression EXPR if debug.timing is
+// enabled.
+#define TIMED_DB_OP(EXPR) { \
+	if(!config.debug.timing.v.b) { EXPR; } \
+	else { \
+		struct timespec _timed_start, _timed_end; \
+		clock_gettime(CLOCK_MONOTONIC, &_timed_start); \
+		(EXPR); \
+		clock_gettime(CLOCK_MONOTONIC, &_timed_end); \
+		long _timed_elapsed = (_timed_end.tv_sec - _timed_start.tv_sec) * 10000 + (_timed_end.tv_nsec - _timed_start.tv_nsec) / 100000; \
+		log_debug(DEBUG_TIMING, "Database operation %s took %.1f ms", str(EXPR), 0.1*_timed_elapsed); \
+	}}
+
+// Macro to time a database operation expression EXPR that returns a value if
+// debug.timing is enabled.
+#define TIMED_DB_OP_RESULT(_result, EXPR) { \
+	if(!config.debug.timing.v.b) { _result = EXPR; } \
+	else { \
+		struct timespec _timed_start, _timed_end; \
+		clock_gettime(CLOCK_MONOTONIC, &_timed_start); \
+		 _result = (EXPR); \
+		clock_gettime(CLOCK_MONOTONIC, &_timed_end); \
+		long _timed_elapsed = (_timed_end.tv_sec - _timed_start.tv_sec) * 10000 + (_timed_end.tv_nsec - _timed_start.tv_nsec) / 100000; \
+		log_debug(DEBUG_TIMING, "Database operation %s took %.1f ms", str(EXPR), 0.1*_timed_elapsed); \
+	}}
+
 #endif //DATABASE_COMMON_H
--- a/src/database/database-thread.c
+++ b/src/database/database-thread.c
@@ -135,9 +135,7 @@ void *DB_thread(void *val)

 			// Save data to database
 			DBOPEN_OR_AGAIN();
-			lock_shm();
-			export_queries_to_disk(false);
-			unlock_shm();
+			TIMED_DB_OP(export_queries_to_disk(false));

 			// Intermediate cancellation-point
 			if(killed)
@@ -146,8 +144,8 @@ void *DB_thread(void *val)
 			// Check if GC should be done on the database
 			if(DBdeleteoldqueries)
 			{
-				// No thread locks needed
-				delete_old_queries_in_DB(db);
+				/* No thread locks needed */
+				TIMED_DB_OP(delete_old_queries_in_DB(db));
 				DBdeleteoldqueries = false;
 			}

@@ -165,7 +163,7 @@ void *DB_thread(void *val)
 		if(now - lastAnalyze >= DATABASE_ANALYZE_INTERVAL)
 		{
 			DBOPEN_OR_AGAIN();
-			analyze_database(db);
+			TIMED_DB_OP(analyze_database(db));
 			lastAnalyze = now;
 			DBCLOSE_OR_BREAK();
 		}
@@ -179,7 +177,7 @@ void *DB_thread(void *val)
 		if(now  - lastMACVendor >= DATABASE_MACVENDOR_INTERVAL)
 		{
 			DBOPEN_OR_AGAIN();
-			updateMACVendorRecords(db);
+			TIMED_DB_OP(updateMACVendorRecords(db));
 			lastMACVendor = now;
 			DBCLOSE_OR_BREAK();
 		}
@@ -192,7 +190,7 @@ void *DB_thread(void *val)
 		if(get_and_clear_event(PARSE_NEIGHBOR_CACHE))
 		{
 			DBOPEN_OR_AGAIN();
-			parse_neighbor_cache(db);
+			TIMED_DB_OP(parse_neighbor_cache(db));
 			DBCLOSE_OR_BREAK();
 		}

@@ -204,7 +202,7 @@ void *DB_thread(void *val)
 		{
 			DBOPEN_OR_AGAIN();
 			lock_shm();
-			reimport_aliasclients(db);
+			TIMED_DB_OP(reimport_aliasclients(db));
 			unlock_shm();
 			DBCLOSE_OR_BREAK();
 		}
--- a/src/database/query-table.c
+++ b/src/database/query-table.c
@@ -238,7 +238,7 @@ bool init_memory_database(void)
 	// queries from the in-memory database (including the query with ID 0)
 	// to the on-disk database.
 	rc = sqlite3_prepare_v3(_memdb, "INSERT INTO disk.query_storage SELECT * FROM query_storage " \
-	                                      "WHERE id > IFNULL((SELECT MAX(id) FROM disk.query_storage), -1) "\
+	                                      "WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.query_storage) "\
 	                                        "AND timestamp < ?",
 	                        -1, SQLITE_PREPARE_PERSISTENT, &queries_to_disk_stmt, NULL);
 	if( rc != SQLITE_OK )
@@ -247,11 +247,15 @@ bool init_memory_database(void)
 		return false;
 	}

+	// Export linking tables to disk database
+	// We limit the export to new records to avoid the overhead of many
+	// IGNORE executions for records that are already present on disk. It
+	// follows the same logic as for the main query_storage table above.
 	const char *subtable_sql[SUBTABLE_STMTS] = {
-		"INSERT OR IGNORE INTO disk.domain_by_id SELECT * FROM domain_by_id",
-		"INSERT OR IGNORE INTO disk.client_by_id SELECT * FROM client_by_id",
-		"INSERT OR IGNORE INTO disk.forward_by_id SELECT * FROM forward_by_id",
-		"INSERT OR IGNORE INTO disk.addinfo_by_id SELECT * FROM addinfo_by_id",
+		"INSERT OR IGNORE INTO disk.domain_by_id SELECT * FROM domain_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.domain_by_id)",
+		"INSERT OR IGNORE INTO disk.client_by_id SELECT * FROM client_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.client_by_id)",
+		"INSERT OR IGNORE INTO disk.forward_by_id SELECT * FROM forward_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.forward_by_id)",
+		"INSERT OR IGNORE INTO disk.addinfo_by_id SELECT * FROM addinfo_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.addinfo_by_id)",
 		"UPDATE disk.sqlite_sequence SET seq = (SELECT seq FROM sqlite_sequence WHERE disk.sqlite_sequence.name = sqlite_sequence.name)"
 	};

@@ -368,7 +372,7 @@ static bool get_memdb_size(sqlite3 *db, size_t *memsize, int *queries)
 	*memsize = page_count * page_size;

 	// Get number of queries in the memory table
-	if((*queries = get_number_of_queries_in_DB(db, "query_storage")) == DB_FAILED)
+	if((*queries = get_number_of_queries_in_DB(db, "query_storage", NULL)) == DB_FAILED)
 		return false;

 	return true;
@@ -504,23 +508,31 @@ bool detach_database(sqlite3* db, const char **message, const char *alias)
 	return okay;
 }

-// Get number of queries either in the temp or in the on-diks database
+// Get number of queries either in the temp or in the on-disk database
 // This routine is used by the API routines.
-int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename)
+int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, double *earliest_timestamp)
 {
 	int rc = 0, num = 0;
 	sqlite3_stmt *stmt = NULL;

-	// Count number of rows
-	const size_t buflen = 42 + strlen(tablename);
-	char *querystr = calloc(buflen, sizeof(char));
-	snprintf(querystr, buflen, "SELECT COUNT(*) FROM %s", tablename);
-
 	// The database pointer may be NULL, meaning we want the memdb
 	if(db == NULL)
 		db = get_memdb();

-	// PRAGMA page_size
+	// Build query string based on whether we need the earliest timestamp too
+	const size_t buflen = 38 + strlen(tablename);
+	char *querystr = calloc(buflen, sizeof(char));
+	if(earliest_timestamp != NULL)
+	{
+		// Get both count and earliest timestamp
+		snprintf(querystr, buflen, "SELECT COUNT(*), MIN(timestamp) FROM %s", tablename);
+	}
+	else
+	{
+		// Get only count
+		snprintf(querystr, buflen, "SELECT COUNT(*) FROM %s", tablename);
+	}
+
 	rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
 	if( rc != SQLITE_OK )
 	{
@@ -532,7 +544,13 @@ int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename)
 	}
 	rc = sqlite3_step(stmt);
 	if( rc == SQLITE_ROW )
+	{
+		// Get count from first column
 		num = sqlite3_column_int(stmt, 0);
+		// Get timestamp from second column if requested
+		if(earliest_timestamp != NULL && sqlite3_column_type(stmt, 1) != SQLITE_NULL)
+			*earliest_timestamp = sqlite3_column_double(stmt, 1);
+	}
 	sqlite3_finalize(stmt);
 	free(querystr);

@@ -617,8 +635,8 @@ bool import_queries_from_disk(void)
 	}

 	// Get number of queries on disk before detaching
-	disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage");
-	mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage");
+	disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage", NULL);
+	mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage", NULL);

 	log_info("Imported %u queries from the on-disk database (it has %u rows)", mem_db_num, disk_db_num);

@@ -640,7 +658,7 @@ bool export_queries_to_disk(const bool final)

 	// Only try to export to database if it is known to not be broken
 	if(FTLDBerror())
-	return false;
+		return false;

 	// Start database timer
 	timer_start(DATABASE_WRITE_TIMER);
@@ -710,7 +728,7 @@ bool export_queries_to_disk(const bool final)
 		}

 		// Update number of queries in the disk database
-		disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage");
+		disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage", NULL);
 	}

 	// Export linking tables and current AUTOINCREMENT values to the disk database
@@ -778,7 +796,7 @@ bool delete_old_queries_from_db(const bool use_memdb, const double mintime)
 		        mintime, sqlite3_errstr(rc));

 	// Update number of queries in in-memory database
-	const int new_num = get_number_of_queries_in_DB(NULL, "query_storage");
+	const int new_num = get_number_of_queries_in_DB(NULL, "query_storage", NULL);
 	log_debug(DEBUG_GC, "delete_old_queries_from_db(): Deleted %i (%u) queries, new number of queries in memory: %i",
 	          sqlite3_changes(db), (mem_db_num - new_num), new_num);
 	mem_db_num = new_num;
@@ -1738,7 +1756,7 @@ bool queries_to_database(void)
 	}

 	// Update number of queries in in-memory database
-	mem_db_num = get_number_of_queries_in_DB(NULL, "query_storage");
+	mem_db_num = get_number_of_queries_in_DB(NULL, "query_storage", NULL);

 	if(config.debug.database.v.b && updated + added > 0)
 	{
--- a/src/database/query-table.h
+++ b/src/database/query-table.h
@@ -113,7 +113,7 @@ void close_memory_database(void);
 bool import_queries_from_disk(void);
 bool attach_database(sqlite3* db, const char **message, const char *path, const char *alias);
 bool detach_database(sqlite3* db, const char **message, const char *alias);
-int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename);
+int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, double *earliest_timestamp);
 bool export_queries_to_disk(const bool final);
 bool delete_old_queries_from_db(const bool use_memdb, const double mintime);
 bool add_additional_info_column(sqlite3 *db);
--- a/src/database/shell.c
+++ b/src/database/shell.c
--- a/src/database/sqlite3-ext.c
+++ b/src/database/sqlite3-ext.c
@@ -225,9 +225,6 @@ static int sqlite3_pihole_extensions_init(sqlite3 *db, char **pzErrMsg, const st
 		        sqlite3_errstr(rc));
 	}

-	// Initialize the percentile extension
-	sqlite3_percentile_init(db, pzErrMsg, pApi);
-
 	return rc;
 }

--- a/src/database/sqlite3-ext.h
+++ b/src/database/sqlite3-ext.h
@@ -14,7 +14,4 @@
 // Initialization point for SQLite3 extensions
 void pihole_sqlite3_initalize(void);

-// Defined in shell.c
-extern int sqlite3_percentile_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);
-
 #endif // SQLITE3_EXT_H
--- a/src/database/sqlite3.c
+++ b/src/database/sqlite3.c
--- a/src/database/sqlite3.h
+++ b/src/database/sqlite3.h
@@ -146,9 +146,12 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.50.4"
-#define SQLITE_VERSION_NUMBER 3050004
-#define SQLITE_SOURCE_ID      "2025-07-30 19:33:53 4d8adfb30e03f9cf27f800a2c1ba3c48fb4ca1b08b0f5ed59a4d5ecbf45e20a3"
+#define SQLITE_VERSION        "3.51.0"
+#define SQLITE_VERSION_NUMBER 3051000
+#define SQLITE_SOURCE_ID      "2025-11-04 19:38:17 fb2c931ae597f8d00a37574ff67aeed3eced4e5547f9120744ae4bfa8e74527b"
+#define SQLITE_SCM_BRANCH     "trunk"
+#define SQLITE_SCM_TAGS       "release major-release version-3.51.0"
+#define SQLITE_SCM_DATETIME   "2025-11-04T19:38:17.314Z"

 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -168,9 +171,9 @@ extern "C" {
 ** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 );
 ** </pre></blockquote>)^
 **
-** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION]
-** macro.  ^The sqlite3_libversion() function returns a pointer to the
-** to the sqlite3_version[] string constant.  The sqlite3_libversion()
+** ^The sqlite3_version[] string constant contains the text of the
+** [SQLITE_VERSION] macro.  ^The sqlite3_libversion() function returns a
+** pointer to the sqlite3_version[] string constant.  The sqlite3_libversion()
 ** function is provided for use in DLLs since DLL users usually do not have
 ** direct access to string constants within the DLL.  ^The
 ** sqlite3_libversion_number() function returns an integer equal to
@@ -370,7 +373,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** without having to use a lot of C code.
 **
 ** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded,
-** semicolon-separate SQL statements passed into its 2nd argument,
+** semicolon-separated SQL statements passed into its 2nd argument,
 ** in the context of the [database connection] passed in as its 1st
 ** argument.  ^If the callback function of the 3rd argument to
 ** sqlite3_exec() is not NULL, then it is invoked for each result row
@@ -403,7 +406,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** result row is NULL then the corresponding string pointer for the
 ** sqlite3_exec() callback is a NULL pointer.  ^The 4th argument to the
 ** sqlite3_exec() callback is an array of pointers to strings where each
-** entry represents the name of corresponding result column as obtained
+** entry represents the name of a corresponding result column as obtained
 ** from [sqlite3_column_name()].
 **
 ** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer
@@ -497,6 +500,9 @@ SQLITE_API int sqlite3_exec(
 #define SQLITE_ERROR_MISSING_COLLSEQ   (SQLITE_ERROR | (1<<8))
 #define SQLITE_ERROR_RETRY             (SQLITE_ERROR | (2<<8))
 #define SQLITE_ERROR_SNAPSHOT          (SQLITE_ERROR | (3<<8))
+#define SQLITE_ERROR_RESERVESIZE       (SQLITE_ERROR | (4<<8))
+#define SQLITE_ERROR_KEY               (SQLITE_ERROR | (5<<8))
+#define SQLITE_ERROR_UNABLE            (SQLITE_ERROR | (6<<8))
 #define SQLITE_IOERR_READ              (SQLITE_IOERR | (1<<8))
 #define SQLITE_IOERR_SHORT_READ        (SQLITE_IOERR | (2<<8))
 #define SQLITE_IOERR_WRITE             (SQLITE_IOERR | (3<<8))
@@ -531,6 +537,8 @@ SQLITE_API int sqlite3_exec(
 #define SQLITE_IOERR_DATA              (SQLITE_IOERR | (32<<8))
 #define SQLITE_IOERR_CORRUPTFS         (SQLITE_IOERR | (33<<8))
 #define SQLITE_IOERR_IN_PAGE           (SQLITE_IOERR | (34<<8))
+#define SQLITE_IOERR_BADKEY            (SQLITE_IOERR | (35<<8))
+#define SQLITE_IOERR_CODEC             (SQLITE_IOERR | (36<<8))
 #define SQLITE_LOCKED_SHAREDCACHE      (SQLITE_LOCKED |  (1<<8))
 #define SQLITE_LOCKED_VTAB             (SQLITE_LOCKED |  (2<<8))
 #define SQLITE_BUSY_RECOVERY           (SQLITE_BUSY   |  (1<<8))
@@ -589,7 +597,7 @@ SQLITE_API int sqlite3_exec(
 ** Note in particular that passing the SQLITE_OPEN_EXCLUSIVE flag into
 ** [sqlite3_open_v2()] does *not* cause the underlying database file
 ** to be opened using O_EXCL.  Passing SQLITE_OPEN_EXCLUSIVE into
-** [sqlite3_open_v2()] has historically be a no-op and might become an
+** [sqlite3_open_v2()] has historically been a no-op and might become an
 ** error in future versions of SQLite.
 */
 #define SQLITE_OPEN_READONLY         0x00000001  /* Ok for sqlite3_open_v2() */
@@ -683,7 +691,7 @@ SQLITE_API int sqlite3_exec(
 ** SQLite uses one of these integer values as the second
 ** argument to calls it makes to the xLock() and xUnlock() methods
 ** of an [sqlite3_io_methods] object.  These values are ordered from
-** lest restrictive to most restrictive.
+** least restrictive to most restrictive.
 **
 ** The argument to xLock() is always SHARED or higher.  The argument to
 ** xUnlock is either SHARED or NONE.
@@ -924,7 +932,7 @@ struct sqlite3_io_methods {
 ** connection.  See also [SQLITE_FCNTL_FILE_POINTER].
 **
 ** <li>[[SQLITE_FCNTL_SYNC_OMITTED]]
-** No longer in use.
+** The SQLITE_FCNTL_SYNC_OMITTED file-control is no longer used.
 **
 ** <li>[[SQLITE_FCNTL_SYNC]]
 ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and
@@ -999,7 +1007,7 @@ struct sqlite3_io_methods {
 **
 ** <li>[[SQLITE_FCNTL_VFSNAME]]
 ** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of
-** all [VFSes] in the VFS stack.  The names are of all VFS shims and the
+** all [VFSes] in the VFS stack.  The names of all VFS shims and the
 ** final bottom-level VFS are written into memory obtained from
 ** [sqlite3_malloc()] and the result is stored in the char* variable
 ** that the fourth parameter of [sqlite3_file_control()] points to.
@@ -1013,7 +1021,7 @@ struct sqlite3_io_methods {
 ** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level
 ** [VFSes] currently in use.  ^(The argument X in
 ** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be
-** of type "[sqlite3_vfs] **".  This opcodes will set *X
+** of type "[sqlite3_vfs] **".  This opcode will set *X
 ** to a pointer to the top-level VFS.)^
 ** ^When there are multiple VFS shims in the stack, this opcode finds the
 ** upper-most shim only.
@@ -1203,7 +1211,7 @@ struct sqlite3_io_methods {
 ** <li>[[SQLITE_FCNTL_EXTERNAL_READER]]
 ** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect
 ** whether or not there is a database client in another process with a wal-mode
-** transaction open on the database or not. It is only available on unix.The
+** transaction open on the database or not. It is only available on unix. The
 ** (void*) argument passed with this file-control should be a pointer to a
 ** value of type (int). The integer value is set to 1 if the database is a wal
 ** mode database and there exists at least one client in another process that
@@ -1221,6 +1229,15 @@ struct sqlite3_io_methods {
 ** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control
 ** purges the contents of the in-memory page cache. If there is an open
 ** transaction, or if the db is a temp-db, this opcode is a no-op, not an error.
+**
+** <li>[[SQLITE_FCNTL_FILESTAT]]
+** The [SQLITE_FCNTL_FILESTAT] opcode returns low-level diagnostic information
+** about the [sqlite3_file] objects used access the database and journal files
+** for the given schema.  The fourth parameter to [sqlite3_file_control()]
+** should be an initialized [sqlite3_str] pointer.  JSON text describing
+** various aspects of the sqlite3_file object is appended to the sqlite3_str.
+** The SQLITE_FCNTL_FILESTAT opcode is usually a no-op, unless compile-time
+** options are used to enable it.
 ** </ul>
 */
 #define SQLITE_FCNTL_LOCKSTATE               1
@@ -1266,6 +1283,7 @@ struct sqlite3_io_methods {
 #define SQLITE_FCNTL_RESET_CACHE            42
 #define SQLITE_FCNTL_NULL_IO                43
 #define SQLITE_FCNTL_BLOCK_ON_CONNECT       44
+#define SQLITE_FCNTL_FILESTAT               45

 /* deprecated names */
 #define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
@@ -1628,7 +1646,7 @@ struct sqlite3_vfs {
 ** SQLite interfaces so that an application usually does not need to
 ** invoke sqlite3_initialize() directly.  For example, [sqlite3_open()]
 ** calls sqlite3_initialize() so the SQLite library will be automatically
-** initialized when [sqlite3_open()] is called if it has not be initialized
+** initialized when [sqlite3_open()] is called if it has not been initialized
 ** already.  ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT]
 ** compile-time option, then the automatic calls to sqlite3_initialize()
 ** are omitted and the application must call sqlite3_initialize() directly
@@ -1885,21 +1903,21 @@ struct sqlite3_mem_methods {
 ** The [sqlite3_mem_methods]
 ** structure is filled with the currently defined memory allocation routines.)^
 ** This option can be used to overload the default memory allocation
-** routines with a wrapper that simulations memory allocation failure or
+** routines with a wrapper that simulates memory allocation failure or
 ** tracks memory usage, for example. </dd>
 **
 ** [[SQLITE_CONFIG_SMALL_MALLOC]] <dt>SQLITE_CONFIG_SMALL_MALLOC</dt>
-** <dd> ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of
+** <dd> ^The SQLITE_CONFIG_SMALL_MALLOC option takes a single argument of
 ** type int, interpreted as a boolean, which if true provides a hint to
 ** SQLite that it should avoid large memory allocations if possible.
 ** SQLite will run faster if it is free to make large memory allocations,
-** but some application might prefer to run slower in exchange for
+** but some applications might prefer to run slower in exchange for
 ** guarantees about memory fragmentation that are possible if large
 ** allocations are avoided.  This hint is normally off.
 ** </dd>
 **
 ** [[SQLITE_CONFIG_MEMSTATUS]] <dt>SQLITE_CONFIG_MEMSTATUS</dt>
-** <dd> ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int,
+** <dd> ^The SQLITE_CONFIG_MEMSTATUS option takes a single argument of type int,
 ** interpreted as a boolean, which enables or disables the collection of
 ** memory allocation statistics. ^(When memory allocation statistics are
 ** disabled, the following SQLite interfaces become non-operational:
@@ -1944,7 +1962,7 @@ struct sqlite3_mem_methods {
 ** ^If pMem is NULL and N is non-zero, then each database connection
 ** does an initial bulk allocation for page cache memory
 ** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or
-** of -1024*N bytes if N is negative, . ^If additional
+** of -1024*N bytes if N is negative. ^If additional
 ** page cache memory is needed beyond what is provided by the initial
 ** allocation, then SQLite goes to [sqlite3_malloc()] separately for each
 ** additional cache line. </dd>
@@ -1973,7 +1991,7 @@ struct sqlite3_mem_methods {
 ** <dd> ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a
 ** pointer to an instance of the [sqlite3_mutex_methods] structure.
 ** The argument specifies alternative low-level mutex routines to be used
-** in place the mutex routines built into SQLite.)^  ^SQLite makes a copy of
+** in place of the mutex routines built into SQLite.)^  ^SQLite makes a copy of
 ** the content of the [sqlite3_mutex_methods] structure before the call to
 ** [sqlite3_config()] returns. ^If SQLite is compiled with
 ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then
@@ -2015,7 +2033,7 @@ struct sqlite3_mem_methods {
 **
 ** [[SQLITE_CONFIG_GETPCACHE2]] <dt>SQLITE_CONFIG_GETPCACHE2</dt>
 ** <dd> ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which
-** is a pointer to an [sqlite3_pcache_methods2] object.  SQLite copies of
+** is a pointer to an [sqlite3_pcache_methods2] object.  SQLite copies off
 ** the current page cache implementation into that object.)^ </dd>
 **
 ** [[SQLITE_CONFIG_LOG]] <dt>SQLITE_CONFIG_LOG</dt>
@@ -2032,7 +2050,7 @@ struct sqlite3_mem_methods {
 ** the logger function is a copy of the first parameter to the corresponding
 ** [sqlite3_log()] call and is intended to be a [result code] or an
 ** [extended result code].  ^The third parameter passed to the logger is
-** log message after formatting via [sqlite3_snprintf()].
+** a log message after formatting via [sqlite3_snprintf()].
 ** The SQLite logging interface is not reentrant; the logger function
 ** supplied by the application must not invoke any SQLite interface.
 ** In a multi-threaded application, the application-defined logger
@@ -2223,7 +2241,7 @@ struct sqlite3_mem_methods {
 ** These constants are the available integer configuration options that
 ** can be passed as the second parameter to the [sqlite3_db_config()] interface.
 **
-** The [sqlite3_db_config()] interface is a var-args functions.  It takes a
+** The [sqlite3_db_config()] interface is a var-args function.  It takes a
 ** variable number of parameters, though always at least two.  The number of
 ** parameters passed into sqlite3_db_config() depends on which of these
 ** constants is given as the second parameter.  This documentation page
@@ -2335,17 +2353,20 @@ struct sqlite3_mem_methods {
 **
 ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]]
 ** <dt>SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER</dt>
-** <dd> ^This option is used to enable or disable the
-** [fts3_tokenizer()] function which is part of the
-** [FTS3] full-text search engine extension.
-** There must be two additional arguments.
-** The first argument is an integer which is 0 to disable fts3_tokenizer() or
-** positive to enable fts3_tokenizer() or negative to leave the setting
-** unchanged.
-** The second parameter is a pointer to an integer into which
-** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled
-** following this call.  The second parameter may be a NULL pointer, in
-** which case the new setting is not reported back. </dd>
+** <dd> ^This option is used to enable or disable using the
+** [fts3_tokenizer()] function - part of the [FTS3] full-text search engine
+** extension - without using bound parameters as the parameters. Doing so
+** is disabled by default. There must be two additional arguments. The first
+** argument is an integer. If it is passed 0, then using fts3_tokenizer()
+** without bound parameters is disabled. If it is passed a positive value,
+** then calling fts3_tokenizer without bound parameters is enabled. If it
+** is passed a negative value, this setting is not modified - this can be
+** used to query for the current setting. The second parameter is a pointer
+** to an integer into which is written 0 or 1 to indicate the current value
+** of this setting (after it is modified, if applicable).  The second
+** parameter may be a NULL pointer, in which case the value of the setting
+** is not reported back. Refer to [FTS3] documentation for further details.
+** </dd>
 **
 ** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]]
 ** <dt>SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION</dt>
@@ -2357,8 +2378,8 @@ struct sqlite3_mem_methods {
 ** When the first argument to this interface is 1, then only the C-API is
 ** enabled and the SQL function remains disabled.  If the first argument to
 ** this interface is 0, then both the C-API and the SQL function are disabled.
-** If the first argument is -1, then no changes are made to state of either the
-** C-API or the SQL function.
+** If the first argument is -1, then no changes are made to the state of either
+** the C-API or the SQL function.
 ** The second parameter is a pointer to an integer into which
 ** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface
 ** is disabled or enabled following this call.  The second parameter may
@@ -2476,7 +2497,7 @@ struct sqlite3_mem_methods {
 ** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]]
 ** <dt>SQLITE_DBCONFIG_LEGACY_ALTER_TABLE</dt>
 ** <dd>The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates
-** the legacy behavior of the [ALTER TABLE RENAME] command such it
+** the legacy behavior of the [ALTER TABLE RENAME] command such that it
 ** behaves as it did prior to [version 3.24.0] (2018-06-04).  See the
 ** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for
 ** additional information. This feature can also be turned on and off
@@ -2525,7 +2546,7 @@ struct sqlite3_mem_methods {
 ** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</dt>
 ** <dd>The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates
 ** the legacy file format flag.  When activated, this flag causes all newly
-** created database file to have a schema format version number (the 4-byte
+** created database files to have a schema format version number (the 4-byte
 ** integer found at offset 44 into the database header) of 1.  This in turn
 ** means that the resulting database file will be readable and writable by
 ** any SQLite version back to 3.0.0 ([dateof:3.0.0]).  Without this setting,
@@ -2552,7 +2573,7 @@ struct sqlite3_mem_methods {
 ** the database handle both when the SQL statement is prepared and when it
 ** is stepped. The flag is set (collection of statistics is enabled)
 ** by default. <p>This option takes two arguments: an integer and a pointer to
-** an integer..  The first argument is 1, 0, or -1 to enable, disable, or
+** an integer.  The first argument is 1, 0, or -1 to enable, disable, or
 ** leave unchanged the statement scanstatus option.  If the second argument
 ** is not NULL, then the value of the statement scanstatus setting after
 ** processing the first argument is written into the integer that the second
@@ -2595,8 +2616,8 @@ struct sqlite3_mem_methods {
 ** <dd>The SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE option enables or disables the
 ** ability of the [ATTACH DATABASE] SQL command to open a database for writing.
 ** This capability is enabled by default.  Applications can disable or
-** reenable this capability using the current DBCONFIG option.  If the
-** the this capability is disabled, the [ATTACH] command will still work,
+** reenable this capability using the current DBCONFIG option.  If
+** this capability is disabled, the [ATTACH] command will still work,
 ** but the database will be opened read-only.  If this option is disabled,
 ** then the ability to create a new database using [ATTACH] is also disabled,
 ** regardless of the value of the [SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE]
@@ -2630,7 +2651,7 @@ struct sqlite3_mem_methods {
 **
 ** <p>Most of the SQLITE_DBCONFIG options take two arguments, so that the
 ** overall call to [sqlite3_db_config()] has a total of four parameters.
-** The first argument (the third parameter to sqlite3_db_config()) is a integer.
+** The first argument (the third parameter to sqlite3_db_config()) is an integer.
 ** The second argument is a pointer to an integer.  If the first argument is 1,
 ** then the option becomes enabled.  If the first integer argument is 0, then the
 ** option is disabled.  If the first argument is -1, then the option setting
@@ -2920,7 +2941,7 @@ SQLITE_API int sqlite3_is_interrupted(sqlite3*);
 ** ^These routines return 0 if the statement is incomplete.  ^If a
 ** memory allocation fails, then SQLITE_NOMEM is returned.
 **
-** ^These routines do not parse the SQL statements thus
+** ^These routines do not parse the SQL statements and thus
 ** will not detect syntactically incorrect SQL.
 **
 ** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior
@@ -3037,7 +3058,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms);
 ** indefinitely if possible. The results of passing any other negative value
 ** are undefined.
 **
-** Internally, each SQLite database handle store two timeout values - the
+** Internally, each SQLite database handle stores two timeout values - the
 ** busy-timeout (used for rollback mode databases, or if the VFS does not
 ** support blocking locks) and the setlk-timeout (used for blocking locks
 ** on wal-mode databases). The sqlite3_busy_timeout() method sets both
@@ -3067,7 +3088,7 @@ SQLITE_API int sqlite3_setlk_timeout(sqlite3*, int ms, int flags);
 ** This is a legacy interface that is preserved for backwards compatibility.
 ** Use of this interface is not recommended.
 **
-** Definition: A <b>result table</b> is memory data structure created by the
+** Definition: A <b>result table</b> is a memory data structure created by the
 ** [sqlite3_get_table()] interface.  A result table records the
 ** complete query results from one or more queries.
 **
@@ -3210,7 +3231,7 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list);
 ** ^Calling sqlite3_free() with a pointer previously returned
 ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so
 ** that it might be reused.  ^The sqlite3_free() routine is
-** a no-op if is called with a NULL pointer.  Passing a NULL pointer
+** a no-op if it is called with a NULL pointer.  Passing a NULL pointer
 ** to sqlite3_free() is harmless.  After being freed, memory
 ** should neither be read nor written.  Even reading previously freed
 ** memory might result in a segmentation fault or other severe error.
@@ -3228,13 +3249,13 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list);
 ** sqlite3_free(X).
 ** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation
 ** of at least N bytes in size or NULL if insufficient memory is available.
-** ^If M is the size of the prior allocation, then min(N,M) bytes
-** of the prior allocation are copied into the beginning of buffer returned
+** ^If M is the size of the prior allocation, then min(N,M) bytes of the
+** prior allocation are copied into the beginning of the buffer returned
 ** by sqlite3_realloc(X,N) and the prior allocation is freed.
 ** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the
 ** prior allocation is not freed.
 **
-** ^The sqlite3_realloc64(X,N) interfaces works the same as
+** ^The sqlite3_realloc64(X,N) interface works the same as
 ** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead
 ** of a 32-bit signed integer.
 **
@@ -3284,7 +3305,7 @@ SQLITE_API sqlite3_uint64 sqlite3_msize(void*);
 ** was last reset.  ^The values returned by [sqlite3_memory_used()] and
 ** [sqlite3_memory_highwater()] include any overhead
 ** added by SQLite in its implementation of [sqlite3_malloc()],
-** but not overhead added by the any underlying system library
+** but not overhead added by any underlying system library
 ** routines that [sqlite3_malloc()] may call.
 **
 ** ^The memory high-water mark is reset to the current value of
@@ -3736,7 +3757,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** there is no harm in trying.)
 **
 ** ^(<dt>[SQLITE_OPEN_SHAREDCACHE]</dt>
-** <dd>The database is opened [shared cache] enabled, overriding
+** <dd>The database is opened with [shared cache] enabled, overriding
 ** the default shared cache setting provided by
 ** [sqlite3_enable_shared_cache()].)^
 ** The [use of shared cache mode is discouraged] and hence shared cache
@@ -3744,7 +3765,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** this option is a no-op.
 **
 ** ^(<dt>[SQLITE_OPEN_PRIVATECACHE]</dt>
-** <dd>The database is opened [shared cache] disabled, overriding
+** <dd>The database is opened with [shared cache] disabled, overriding
 ** the default shared cache setting provided by
 ** [sqlite3_enable_shared_cache()].)^
 **
@@ -4162,7 +4183,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename);
 ** subsequent calls to other SQLite interface functions.)^
 **
 ** ^The sqlite3_errstr(E) interface returns the English-language text
-** that describes the [result code] E, as UTF-8, or NULL if E is not an
+** that describes the [result code] E, as UTF-8, or NULL if E is not a
 ** result code for which a text error message is available.
 ** ^(Memory to hold the error message string is managed internally
 ** and must not be freed by the application)^.
@@ -4170,7 +4191,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename);
 ** ^If the most recent error references a specific token in the input
 ** SQL, the sqlite3_error_offset() interface returns the byte offset
 ** of the start of that token.  ^The byte offset returned by
-** sqlite3_error_offset() assumes that the input SQL is UTF8.
+** sqlite3_error_offset() assumes that the input SQL is UTF-8.
 ** ^If the most recent error does not reference a specific token in the input
 ** SQL, then the sqlite3_error_offset() function returns -1.
 **
@@ -4195,6 +4216,34 @@ SQLITE_API const void *sqlite3_errmsg16(sqlite3*);
 SQLITE_API const char *sqlite3_errstr(int);
 SQLITE_API int sqlite3_error_offset(sqlite3 *db);

+/*
+** CAPI3REF: Set Error Codes And Message
+** METHOD: sqlite3
+**
+** Set the error code of the database handle passed as the first argument
+** to errcode, and the error message to a copy of nul-terminated string
+** zErrMsg. If zErrMsg is passed NULL, then the error message is set to
+** the default message associated with the supplied error code.  Subsequent
+** calls to [sqlite3_errcode()] and [sqlite3_errmsg()] and similar will
+** return the values set by this routine in place of what was previously
+** set by SQLite itself.
+**
+** This function returns SQLITE_OK if the error code and error message are
+** successfully set, SQLITE_NOMEM if an OOM occurs, and SQLITE_MISUSE if
+** the database handle is NULL or invalid.
+**
+** The error code and message set by this routine remains in effect until
+** they are changed, either by another call to this routine or until they are
+** changed to by SQLite itself to reflect the result of some subsquent
+** API call.
+**
+** This function is intended for use by SQLite extensions or wrappers.  The
+** idea is that an extension or wrapper can use this routine to set error
+** messages and error codes and thus behave more like a core SQLite
+** feature from the point of view of an application.
+*/
+SQLITE_API int sqlite3_set_errmsg(sqlite3 *db, int errcode, const char *zErrMsg);
+
 /*
 ** CAPI3REF: Prepared Statement Object
 ** KEYWORDS: {prepared statement} {prepared statements}
@@ -4269,8 +4318,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal);
 **
 ** These constants define various performance limits
 ** that can be lowered at run-time using [sqlite3_limit()].
-** The synopsis of the meanings of the various limits is shown below.
-** Additional information is available at [limits | Limits in SQLite].
+** A concise description of these limits follows, and additional information
+** is available at [limits | Limits in SQLite].
 **
 ** <dl>
 ** [[SQLITE_LIMIT_LENGTH]] ^(<dt>SQLITE_LIMIT_LENGTH</dt>
@@ -4335,7 +4384,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal);
 /*
 ** CAPI3REF: Prepare Flags
 **
-** These constants define various flags that can be passed into
+** These constants define various flags that can be passed into the
 ** "prepFlags" parameter of the [sqlite3_prepare_v3()] and
 ** [sqlite3_prepare16_v3()] interfaces.
 **
@@ -4422,7 +4471,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal);
 ** there is a small performance advantage to passing an nByte parameter that
 ** is the number of bytes in the input string <i>including</i>
 ** the nul-terminator.
-** Note that nByte measure the length of the input in bytes, not
+** Note that nByte measures the length of the input in bytes, not
 ** characters, even for the UTF-16 interfaces.
 **
 ** ^If pzTail is not NULL then *pzTail is made to point to the first byte
@@ -4556,7 +4605,7 @@ SQLITE_API int sqlite3_prepare16_v3(
 **
 ** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory
 ** is available to hold the result, or if the result would exceed the
-** the maximum string length determined by the [SQLITE_LIMIT_LENGTH].
+** maximum string length determined by the [SQLITE_LIMIT_LENGTH].
 **
 ** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of
 ** bound parameter expansions.  ^The [SQLITE_OMIT_TRACE] compile-time
@@ -4744,7 +4793,7 @@ typedef struct sqlite3_value sqlite3_value;
 **
 ** The context in which an SQL function executes is stored in an
 ** sqlite3_context object.  ^A pointer to an sqlite3_context object
-** is always first parameter to [application-defined SQL functions].
+** is always the first parameter to [application-defined SQL functions].
 ** The application-defined SQL function implementation will pass this
 ** pointer through into calls to [sqlite3_result_int | sqlite3_result()],
 ** [sqlite3_aggregate_context()], [sqlite3_user_data()],
@@ -4868,9 +4917,11 @@ typedef struct sqlite3_context sqlite3_context;
 ** associated with the pointer P of type T.  ^D is either a NULL pointer or
 ** a pointer to a destructor function for P. ^SQLite will invoke the
 ** destructor D with a single argument of P when it is finished using
-** P.  The T parameter should be a static string, preferably a string
-** literal. The sqlite3_bind_pointer() routine is part of the
-** [pointer passing interface] added for SQLite 3.20.0.
+** P, even if the call to sqlite3_bind_pointer() fails.  Due to a
+** historical design quirk, results are undefined if D is
+** SQLITE_TRANSIENT. The T parameter should be a static string,
+** preferably a string literal. The sqlite3_bind_pointer() routine is
+** part of the [pointer passing interface] added for SQLite 3.20.0.
 **
 ** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer
 ** for the [prepared statement] or with a prepared statement for which
@@ -5481,7 +5532,7 @@ SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol);
 **
 ** ^The sqlite3_finalize() function is called to delete a [prepared statement].
 ** ^If the most recent evaluation of the statement encountered no errors
-** or if the statement is never been evaluated, then sqlite3_finalize() returns
+** or if the statement has never been evaluated, then sqlite3_finalize() returns
 ** SQLITE_OK.  ^If the most recent evaluation of statement S failed, then
 ** sqlite3_finalize(S) returns the appropriate [error code] or
 ** [extended error code].
@@ -5713,7 +5764,7 @@ SQLITE_API int sqlite3_create_window_function(
 /*
 ** CAPI3REF: Text Encodings
 **
-** These constant define integer codes that represent the various
+** These constants define integer codes that represent the various
 ** text encodings supported by SQLite.
 */
 #define SQLITE_UTF8           1    /* IMP: R-37514-35566 */
@@ -5805,7 +5856,7 @@ SQLITE_API int sqlite3_create_window_function(
 ** result.
 ** Every function that invokes [sqlite3_result_subtype()] should have this
 ** property.  If it does not, then the call to [sqlite3_result_subtype()]
-** might become a no-op if the function is used as term in an
+** might become a no-op if the function is used as a term in an
 ** [expression index].  On the other hand, SQL functions that never invoke
 ** [sqlite3_result_subtype()] should avoid setting this property, as the
 ** purpose of this property is to disable certain optimizations that are
@@ -5932,7 +5983,7 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6
 ** sqlite3_value_nochange(X) interface returns true if and only if
 ** the column corresponding to X is unchanged by the UPDATE operation
 ** that the xUpdate method call was invoked to implement and if
-** and the prior [xColumn] method call that was invoked to extracted
+** the prior [xColumn] method call that was invoked to extract
 ** the value for that column returned without setting a result (probably
 ** because it queried [sqlite3_vtab_nochange()] and found that the column
 ** was unchanging).  ^Within an [xUpdate] method, any value for which
@@ -6205,6 +6256,7 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi
 ** or a NULL pointer if there were no prior calls to
 ** sqlite3_set_clientdata() with the same values of D and N.
 ** Names are compared using strcmp() and are thus case sensitive.
+** It returns 0 on success and SQLITE_NOMEM on allocation failure.
 **
 ** If P and X are both non-NULL, then the destructor X is invoked with
 ** argument P on the first of the following occurrences:
@@ -8881,9 +8933,18 @@ SQLITE_API int sqlite3_status64(
 ** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a
 ** non-zero [error code] on failure.
 **
+** ^The sqlite3_db_status64(D,O,C,H,R) routine works exactly the same
+** way as sqlite3_db_status(D,O,C,H,R) routine except that the C and H
+** parameters are pointer to 64-bit integers (type: sqlite3_int64) instead
+** of pointers to 32-bit integers, which allows larger status values
+** to be returned.  If a status value exceeds 2,147,483,647 then
+** sqlite3_db_status() will truncate the value whereas sqlite3_db_status64()
+** will return the full value.
+**
 ** See also: [sqlite3_status()] and [sqlite3_stmt_status()].
 */
 SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
+SQLITE_API int sqlite3_db_status64(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int);

 /*
 ** CAPI3REF: Status Parameters for database connections
@@ -8980,6 +9041,10 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
 ** If an IO or other error occurs while writing a page to disk, the effect
 ** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The
 ** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0.
+** <p>
+** ^(There is overlap between the quantities measured by this parameter
+** (SQLITE_DBSTATUS_CACHE_WRITE) and SQLITE_DBSTATUS_TEMPBUF_SPILL.
+** Resetting one will reduce the other.)^
 ** </dd>
 **
 ** [[SQLITE_DBSTATUS_CACHE_SPILL]] ^(<dt>SQLITE_DBSTATUS_CACHE_SPILL</dt>
@@ -8995,6 +9060,18 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
 ** <dd>This parameter returns zero for the current value if and only if
 ** all foreign key constraints (deferred or immediate) have been
 ** resolved.)^  ^The highwater mark is always 0.
+**
+** [[SQLITE_DBSTATUS_TEMPBUF_SPILL] ^(<dt>SQLITE_DBSTATUS_TEMPBUF_SPILL</dt>
+** <dd>^(This parameter returns the number of bytes written to temporary
+** files on disk that could have been kept in memory had sufficient memory
+** been available.  This value includes writes to intermediate tables that
+** are part of complex queries, external sorts that spill to disk, and
+** writes to TEMP tables.)^
+** ^The highwater mark is always 0.
+** <p>
+** ^(There is overlap between the quantities measured by this parameter
+** (SQLITE_DBSTATUS_TEMPBUF_SPILL) and SQLITE_DBSTATUS_CACHE_WRITE.
+** Resetting one will reduce the other.)^
 ** </dd>
 ** </dl>
 */
@@ -9011,7 +9088,8 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
 #define SQLITE_DBSTATUS_DEFERRED_FKS        10
 #define SQLITE_DBSTATUS_CACHE_USED_SHARED   11
 #define SQLITE_DBSTATUS_CACHE_SPILL         12
-#define SQLITE_DBSTATUS_MAX                 12   /* Largest defined DBSTATUS */
+#define SQLITE_DBSTATUS_TEMPBUF_SPILL       13
+#define SQLITE_DBSTATUS_MAX                 13   /* Largest defined DBSTATUS */


 /*
@@ -9776,7 +9854,7 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...);
 ** is the number of pages currently in the write-ahead log file,
 ** including those that were just committed.
 **
-** The callback function should normally return [SQLITE_OK].  ^If an error
+** ^The callback function should normally return [SQLITE_OK].  ^If an error
 ** code is returned, that error will propagate back up through the
 ** SQLite code base to cause the statement that provoked the callback
 ** to report an error, though the commit will have still occurred. If the
@@ -9784,13 +9862,26 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...);
 ** that does not correspond to any valid SQLite error code, the results
 ** are undefined.
 **
-** A single database handle may have at most a single write-ahead log callback
-** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any
-** previously registered write-ahead log callback. ^The return value is
-** a copy of the third parameter from the previous call, if any, or 0.
-** ^Note that the [sqlite3_wal_autocheckpoint()] interface and the
-** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will
-** overwrite any prior [sqlite3_wal_hook()] settings.
+** ^A single database handle may have at most a single write-ahead log
+** callback registered at one time. ^Calling [sqlite3_wal_hook()]
+** replaces the default behavior or previously registered write-ahead
+** log callback.
+**
+** ^The return value is a copy of the third parameter from the
+** previous call, if any, or 0.
+**
+** ^The [sqlite3_wal_autocheckpoint()] interface and the
+** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and
+** will overwrite any prior [sqlite3_wal_hook()] settings.
+**
+** ^If a write-ahead log callback is set using this function then
+** [sqlite3_wal_checkpoint_v2()] or [PRAGMA wal_checkpoint]
+** should be invoked periodically to keep the write-ahead log file
+** from growing without bound.
+**
+** ^Passing a NULL pointer for the callback disables automatic
+** checkpointing entirely. To re-enable the default behavior, call
+** sqlite3_wal_autocheckpoint(db,1000) or use [PRAGMA wal_checkpoint].
 */
 SQLITE_API void *sqlite3_wal_hook(
  sqlite3*,
@@ -9807,7 +9898,7 @@ SQLITE_API void *sqlite3_wal_hook(
 ** to automatically [checkpoint]
 ** after committing a transaction if there are N or
 ** more frames in the [write-ahead log] file.  ^Passing zero or
-** a negative value as the nFrame parameter disables automatic
+** a negative value as the N parameter disables automatic
 ** checkpoints entirely.
 **
 ** ^The callback registered by this function replaces any existing callback
@@ -9823,9 +9914,10 @@ SQLITE_API void *sqlite3_wal_hook(
 **
 ** ^Every new [database connection] defaults to having the auto-checkpoint
 ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT]
-** pages.  The use of this interface
-** is only necessary if the default setting is found to be suboptimal
-** for a particular application.
+** pages.
+**
+** ^The use of this interface is only necessary if the default setting
+** is found to be suboptimal for a particular application.
 */
 SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N);

@@ -9890,6 +9982,11 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb);
 **   ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the
 **   addition that it also truncates the log file to zero bytes just prior
 **   to a successful return.
+**
+** <dt>SQLITE_CHECKPOINT_NOOP<dd>
+**   ^This mode always checkpoints zero frames. The only reason to invoke
+**   a NOOP checkpoint is to access the values returned by
+**   sqlite3_wal_checkpoint_v2() via output parameters *pnLog and *pnCkpt.
 ** </dl>
 **
 ** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in
@@ -9960,6 +10057,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2(
 ** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the
 ** meaning of each of these checkpoint modes.
 */
+#define SQLITE_CHECKPOINT_NOOP    -1  /* Do no work at all */
 #define SQLITE_CHECKPOINT_PASSIVE  0  /* Do as much as possible w/o blocking */
 #define SQLITE_CHECKPOINT_FULL     1  /* Wait for writers, then checkpoint */
 #define SQLITE_CHECKPOINT_RESTART  2  /* Like FULL but wait for readers */
@@ -10787,7 +10885,7 @@ typedef struct sqlite3_snapshot {
 ** The [sqlite3_snapshot_get()] interface is only available when the
 ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used.
 */
-SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get(
+SQLITE_API int sqlite3_snapshot_get(
  sqlite3 *db,
  const char *zSchema,
  sqlite3_snapshot **ppSnapshot
@@ -10836,7 +10934,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get(
 ** The [sqlite3_snapshot_open()] interface is only available when the
 ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used.
 */
-SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open(
+SQLITE_API int sqlite3_snapshot_open(
  sqlite3 *db,
  const char *zSchema,
  sqlite3_snapshot *pSnapshot
@@ -10853,7 +10951,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open(
 ** The [sqlite3_snapshot_free()] interface is only available when the
 ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used.
 */
-SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*);
+SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot*);

 /*
 ** CAPI3REF: Compare the ages of two snapshot handles.
@@ -10880,7 +10978,7 @@ SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*);
 ** This interface is only available if SQLite is compiled with the
 ** [SQLITE_ENABLE_SNAPSHOT] option.
 */
-SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
+SQLITE_API int sqlite3_snapshot_cmp(
  sqlite3_snapshot *p1,
  sqlite3_snapshot *p2
 );
@@ -10908,7 +11006,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
 ** This interface is only available if SQLite is compiled with the
 ** [SQLITE_ENABLE_SNAPSHOT] option.
 */
-SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
+SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);

 /*
 ** CAPI3REF: Serialize a database
@@ -10982,12 +11080,13 @@ SQLITE_API unsigned char *sqlite3_serialize(
 **
 ** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the
 ** [database connection] D to disconnect from database S and then
-** reopen S as an in-memory database based on the serialization contained
-** in P.  The serialized database P is N bytes in size.  M is the size of
-** the buffer P, which might be larger than N.  If M is larger than N, and
-** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is
-** permitted to add content to the in-memory database as long as the total
-** size does not exceed M bytes.
+** reopen S as an in-memory database based on the serialization
+** contained in P.  If S is a NULL pointer, the main database is
+** used. The serialized database P is N bytes in size.  M is the size
+** of the buffer P, which might be larger than N.  If M is larger than
+** N, and the SQLITE_DESERIALIZE_READONLY bit is not set in F, then
+** SQLite is permitted to add content to the in-memory database as
+** long as the total size does not exceed M bytes.
 **
 ** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will
 ** invoke sqlite3_free() on the serialization buffer when the database
@@ -11054,6 +11153,54 @@ SQLITE_API int sqlite3_deserialize(
 #define SQLITE_DESERIALIZE_RESIZEABLE  2 /* Resize using sqlite3_realloc64() */
 #define SQLITE_DESERIALIZE_READONLY    4 /* Database is read-only */

+/*
+** CAPI3REF: Bind array values to the CARRAY table-valued function
+**
+** The sqlite3_carray_bind(S,I,P,N,F,X) interface binds an array value to
+** one of the first argument of the [carray() table-valued function].  The
+** S parameter is a pointer to the [prepared statement] that uses the carray()
+** functions.  I is the parameter index to be bound.  P is a pointer to the
+** array to be bound, and N is the number of eements in the array.  The
+** F argument is one of constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64],
+** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], or [SQLITE_CARRAY_BLOB] to
+** indicate the datatype of the array being bound.  The X argument is not a
+** NULL pointer, then SQLite will invoke the function X on the P parameter
+** after it has finished using P, even if the call to
+** sqlite3_carray_bind() fails. The special-case finalizer
+** SQLITE_TRANSIENT has no effect here.
+*/
+SQLITE_API int sqlite3_carray_bind(
+  sqlite3_stmt *pStmt,        /* Statement to be bound */
+  int i,                      /* Parameter index */
+  void *aData,                /* Pointer to array data */
+  int nData,                  /* Number of data elements */
+  int mFlags,                 /* CARRAY flags */
+  void (*xDel)(void*)         /* Destructor for aData */
+);
+
+/*
+** CAPI3REF: Datatypes for the CARRAY table-valued function
+**
+** The fifth argument to the [sqlite3_carray_bind()] interface musts be
+** one of the following constants, to specify the datatype of the array
+** that is being bound into the [carray table-valued function].
+*/
+#define SQLITE_CARRAY_INT32     0    /* Data is 32-bit signed integers */
+#define SQLITE_CARRAY_INT64     1    /* Data is 64-bit signed integers */
+#define SQLITE_CARRAY_DOUBLE    2    /* Data is doubles */
+#define SQLITE_CARRAY_TEXT      3    /* Data is char* */
+#define SQLITE_CARRAY_BLOB      4    /* Data is struct iovec */
+
+/*
+** Versions of the above #defines that omit the initial SQLITE_, for
+** legacy compatibility.
+*/
+#define CARRAY_INT32     0    /* Data is 32-bit signed integers */
+#define CARRAY_INT64     1    /* Data is 64-bit signed integers */
+#define CARRAY_DOUBLE    2    /* Data is doubles */
+#define CARRAY_TEXT      3    /* Data is char* */
+#define CARRAY_BLOB      4    /* Data is struct iovec */
+
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.
@@ -12313,14 +12460,32 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*);
 ** update the "main" database attached to handle db with the changes found in
 ** the changeset passed via the second and third arguments.
 **
+** All changes made by these functions are enclosed in a savepoint transaction.
+** If any other error (aside from a constraint failure when attempting to
+** write to the target database) occurs, then the savepoint transaction is
+** rolled back, restoring the target database to its original state, and an
+** SQLite error code returned. Additionally, starting with version 3.51.0,
+** an error code and error message that may be accessed using the
+** [sqlite3_errcode()] and [sqlite3_errmsg()] APIs are left in the database
+** handle.
+**
 ** The fourth argument (xFilter) passed to these functions is the "filter
-** callback". If it is not NULL, then for each table affected by at least one
-** change in the changeset, the filter callback is invoked with
-** the table name as the second argument, and a copy of the context pointer
-** passed as the sixth argument as the first. If the "filter callback"
-** returns zero, then no attempt is made to apply any changes to the table.
-** Otherwise, if the return value is non-zero or the xFilter argument to
-** is NULL, all changes related to the table are attempted.
+** callback". This may be passed NULL, in which case all changes in the
+** changeset are applied to the database. For sqlite3changeset_apply() and
+** sqlite3_changeset_apply_v2(), if it is not NULL, then it is invoked once
+** for each table affected by at least one change in the changeset. In this
+** case the table name is passed as the second argument, and a copy of
+** the context pointer passed as the sixth argument to apply() or apply_v2()
+** as the first. If the "filter callback" returns zero, then no attempt is
+** made to apply any changes to the table. Otherwise, if the return value is
+** non-zero, all changes related to the table are attempted.
+**
+** For sqlite3_changeset_apply_v3(), the xFilter callback is invoked once
+** per change. The second argument in this case is an sqlite3_changeset_iter
+** that may be queried using the usual APIs for the details of the current
+** change. If the "filter callback" returns zero in this case, then no attempt
+** is made to apply the current change. If it returns non-zero, the change
+** is applied.
 **
 ** For each table that is not excluded by the filter callback, this function
 ** tests that the target database contains a compatible table. A table is
@@ -12341,11 +12506,11 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*);
 ** one such warning is issued for each table in the changeset.
 **
 ** For each change for which there is a compatible table, an attempt is made
-** to modify the table contents according to the UPDATE, INSERT or DELETE
-** change. If a change cannot be applied cleanly, the conflict handler
-** function passed as the fifth argument to sqlite3changeset_apply() may be
-** invoked. A description of exactly when the conflict handler is invoked for
-** each type of change is below.
+** to modify the table contents according to each UPDATE, INSERT or DELETE
+** change that is not excluded by a filter callback. If a change cannot be
+** applied cleanly, the conflict handler function passed as the fifth argument
+** to sqlite3changeset_apply() may be invoked. A description of exactly when
+** the conflict handler is invoked for each type of change is below.
 **
 ** Unlike the xFilter argument, xConflict may not be passed NULL. The results
 ** of passing anything other than a valid function pointer as the xConflict
@@ -12441,12 +12606,6 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*);
 ** This can be used to further customize the application's conflict
 ** resolution strategy.
 **
-** All changes made by these functions are enclosed in a savepoint transaction.
-** If any other error (aside from a constraint failure when attempting to
-** write to the target database) occurs, then the savepoint transaction is
-** rolled back, restoring the target database to its original state, and an
-** SQLite error code returned.
-**
 ** If the output parameters (ppRebase) and (pnRebase) are non-NULL and
 ** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2()
 ** may set (*ppRebase) to point to a "rebase" that may be used with the
@@ -12496,6 +12655,23 @@ SQLITE_API int sqlite3changeset_apply_v2(
  void **ppRebase, int *pnRebase, /* OUT: Rebase data */
  int flags                       /* SESSION_CHANGESETAPPLY_* flags */
 );
+SQLITE_API int sqlite3changeset_apply_v3(
+  sqlite3 *db,                    /* Apply change to "main" db of this handle */
+  int nChangeset,                 /* Size of changeset in bytes */
+  void *pChangeset,               /* Changeset blob */
+  int(*xFilter)(
+    void *pCtx,                   /* Copy of sixth arg to _apply() */
+    sqlite3_changeset_iter *p     /* Handle describing change */
+  ),
+  int(*xConflict)(
+    void *pCtx,                   /* Copy of sixth arg to _apply() */
+    int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
+    sqlite3_changeset_iter *p     /* Handle describing change and conflict */
+  ),
+  void *pCtx,                     /* First argument passed to xConflict */
+  void **ppRebase, int *pnRebase, /* OUT: Rebase data */
+  int flags                       /* SESSION_CHANGESETAPPLY_* flags */
+);

 /*
 ** CAPI3REF: Flags for sqlite3changeset_apply_v2
@@ -12915,6 +13091,23 @@ SQLITE_API int sqlite3changeset_apply_v2_strm(
  void **ppRebase, int *pnRebase,
  int flags
 );
+SQLITE_API int sqlite3changeset_apply_v3_strm(
+  sqlite3 *db,                    /* Apply change to "main" db of this handle */
+  int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */
+  void *pIn,                                          /* First arg for xInput */
+  int(*xFilter)(
+    void *pCtx,                   /* Copy of sixth arg to _apply() */
+    sqlite3_changeset_iter *p
+  ),
+  int(*xConflict)(
+    void *pCtx,                   /* Copy of sixth arg to _apply() */
+    int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
+    sqlite3_changeset_iter *p     /* Handle describing change and conflict */
+  ),
+  void *pCtx,                     /* First argument passed to xConflict */
+  void **ppRebase, int *pnRebase,
+  int flags
+);
 SQLITE_API int sqlite3changeset_concat_strm(
  int (*xInputA)(void *pIn, void *pData, int *pnData),
  void *pInA,
--- a/src/datastructure.c
+++ b/src/datastructure.c
@@ -32,10 +32,11 @@
 #include "lookup-table.h"

 // converts upper to lower case, and leaves other characters unchanged
+// Optimized version using pointer arithmetic for better performance
 void strtolower(char *str)
 {
-	int i = 0;
-	while(str[i]){ str[i] = tolower(str[i]); i++; }
+	for(; *str; ++str)
+		*str = tolower(*str);
 }

 /**
@@ -54,7 +55,7 @@ void strtolower(char *str)
 */
 static uint32_t __attribute__ ((pure)) hashStr(const char *s)
 {
-	// Jenkins' One-at-a-Time hash
+	// Jenkins' One-at-a-Time hash (optimized version)
 	// (http://www.burtleburtle.net/bob/hash/doobs.html)
 	uint32_t hash = 0;
 	for(; *s; ++s)
@@ -64,6 +65,7 @@ static uint32_t __attribute__ ((pure)) hashStr(const char *s)
 		hash ^= hash >> 6;
 	}

+	// Final mixing to ensure good distribution
 	hash += hash << 3;
 	hash ^= hash >> 11;
 	hash += hash << 15;
@@ -1005,16 +1007,19 @@ const char * __attribute__ ((pure)) get_blocked_statuslist(void)
 	unsigned int first = 0;
 	// Open parenthesis
 	blocked_list[0] = '(';
+	size_t pos = 1;  // Track current position instead of calling strlen repeatedly
 	for(enum query_status status = 0; status < QUERY_STATUS_MAX; status++)
 		if(is_blocked(status))
-			snprintf(blocked_list + strlen(blocked_list),
-			         sizeof(blocked_list) - strlen(blocked_list),
-			         "%s%d", first++ < 1 ? "" : ",", status);
+		{
+			int written = snprintf(blocked_list + pos, sizeof(blocked_list) - pos,
+			                      "%s%d", first++ < 1 ? "" : ",", status);
+			if(written > 0 && (size_t)written < sizeof(blocked_list) - pos)
+				pos += written;
+		}

 	// Close parenthesis
-	const size_t len = strlen(blocked_list);
-	blocked_list[len] = ')';
-	blocked_list[len + 1] = '\0';
+	blocked_list[pos] = ')';
+	blocked_list[pos + 1] = '\0';
 	return blocked_list;
 }

@@ -1028,16 +1033,19 @@ const char * __attribute__ ((pure)) get_cached_statuslist(void)
 	unsigned int first = 0;
 	// Open parenthesis
 	cached_list[0] = '(';
+	size_t pos = 1;  // Track current position instead of calling strlen repeatedly
 	for(enum query_status status = 0; status < QUERY_STATUS_MAX; status++)
 		if(is_cached(status))
-			snprintf(cached_list + strlen(cached_list),
-			         sizeof(cached_list) - strlen(cached_list),
-			         "%s%d", first++ < 1 ? "" : ",", status);
+		{
+			int written = snprintf(cached_list + pos, sizeof(cached_list) - pos,
+			                      "%s%d", first++ < 1 ? "" : ",", status);
+			if(written > 0 && (size_t)written < sizeof(cached_list) - pos)
+				pos += written;
+		}

 	// Close parenthesis
-	const size_t len = strlen(cached_list);
-	cached_list[len] = ')';
-	cached_list[len + 1] = '\0';
+	cached_list[pos] = ')';
+	cached_list[pos + 1] = '\0';
 	return cached_list;
 }

@@ -1051,16 +1059,19 @@ const char * __attribute__ ((pure)) get_permitted_statuslist(void)
 	unsigned int first = 0;
 	// Open parenthesis
 	permitted_list[0] = '(';
+	size_t pos = 1;  // Track current position instead of calling strlen repeatedly
 	for(enum query_status status = 0; status < QUERY_STATUS_MAX; status++)
 		if(!is_blocked(status))
-			snprintf(permitted_list + strlen(permitted_list),
-			         sizeof(permitted_list) - strlen(permitted_list),
-			         "%s%d", first++ < 1 ? "" : ",", status);
+		{
+			int written = snprintf(permitted_list + pos, sizeof(permitted_list) - pos,
+			                      "%s%d", first++ < 1 ? "" : ",", status);
+			if(written > 0 && (size_t)written < sizeof(permitted_list) - pos)
+				pos += written;
+		}

 	// Close parenthesis
-	const size_t len = strlen(permitted_list);
-	permitted_list[len] = ')';
-	permitted_list[len + 1] = '\0';
+	permitted_list[pos] = ')';
+	permitted_list[pos + 1] = '\0';
 	return permitted_list;
 }

--- a/src/dnsmasq_interface.c
+++ b/src/dnsmasq_interface.c
@@ -48,7 +48,7 @@
 #include "webserver/webserver.h"
 // type struct sqlite3_stmt_vec
 #include "vector.h"
-// query_to_database()
+// init_memory_database()
 #include "database/query-table.h"
 // reread_config()
 #include "config/config.h"
@@ -1695,11 +1695,11 @@ static bool FTL_check_blocking(const unsigned int queryID, const unsigned int do
 	const char *blockedDomain = domain_lower;

 	// Check exact whitelist for match
-	query->flags.allowed = in_allowlist(domain_lower, dns_cache, client) == FOUND;
+	TIMED_DB_OP_RESULT(query->flags.allowed, in_allowlist(domain_lower, dns_cache, client) == FOUND);

 	// If not found: Check regex whitelist for match
 	if(!query->flags.allowed)
-		query->flags.allowed = in_regex(domain_lower, dns_cache, client->id, REGEX_ALLOW);
+		TIMED_DB_OP_RESULT(query->flags.allowed, in_regex(domain_lower, dns_cache, client->id, REGEX_ALLOW));

 	// Check if this is a special domain
 	if(!query->flags.allowed && special_domain(query, domain_lower))
@@ -1721,7 +1721,8 @@ static bool FTL_check_blocking(const unsigned int queryID, const unsigned int do
 	// Check blacklist (exact + regex) and gravity for queried domain
 	unsigned char new_status = QUERY_UNKNOWN;
 	bool db_okay = true;
-	bool blockDomain = check_domain_blocked(domain_lower, client, query, dns_cache, &new_status, &db_okay);
+	bool blockDomain;
+	TIMED_DB_OP_RESULT(blockDomain, check_domain_blocked(domain_lower, client, query, dns_cache, &new_status, &db_okay));

 	// Check blacklist (exact + regex) and gravity for _esni.domain if enabled
 	// (defaulting to true)
@@ -1729,7 +1730,7 @@ static bool FTL_check_blocking(const unsigned int queryID, const unsigned int do
 	   !query->flags.allowed && blockDomain == NOT_FOUND &&
 	   strlen(domain_lower) > 6 && strncasecmp(domain_lower, "_esni.", 6u) == 0)
 	{
-		blockDomain = check_domain_blocked(domain_lower + 6u, client, query, dns_cache, &new_status, &db_okay);
+		TIMED_DB_OP_RESULT(blockDomain, check_domain_blocked(domain_lower + 6u, client, query, dns_cache, &new_status, &db_okay));

 		// Update DNS cache status
 		cacheStatus = dns_cache->blocking_status;
--- a/src/enums.h
+++ b/src/enums.h
@@ -156,6 +156,7 @@ enum debug_flag {
 	DEBUG_RESERVED,
 	DEBUG_NTP,
 	DEBUG_NETLINK,
+	DEBUG_TIMING,
 	DEBUG_MAX
 } __attribute__ ((packed));

--- a/src/log.c
+++ b/src/log.c
@@ -224,6 +224,8 @@ const char *debugstr(const enum debug_flag flag)
 			return "DEBUG_NTP";
 		case DEBUG_NETLINK:
 			return "DEBUG_NETLINK";
+		case DEBUG_TIMING:
+			return "DEBUG_TIMING";
 		case DEBUG_MAX:
 			return "DEBUG_MAX";
 		case DEBUG_NONE: // fall through
--- a/src/regex.c
+++ b/src/regex.c
@@ -151,7 +151,7 @@ bool compile_regex(const char *regexin, regexData *regex, char **message)
 		// Extract regular expression pattern in front of FTL-specific syntax
 		char *saveptr = NULL;
 		char *part = strtok_r(buf, FTL_REGEX_SEP, &saveptr);
-		strncpy(rgxbuf, part, strlen(regexin));
+		strncpy(rgxbuf, part, strlen(part));

 		// Analyze FTL-specific parts
 		while((part = strtok_r(NULL, FTL_REGEX_SEP, &saveptr)) != NULL)
--- a/src/shmem.c
+++ b/src/shmem.c
@@ -134,6 +134,10 @@ typedef struct {
 		volatile pid_t pid;
 		volatile pid_t tid;
 	} owner;
+	struct {
+		struct timespec begin;
+		struct timespec end;
+	} time;
 } ShmLock;
 static ShmLock *shmLock = NULL;
 static ShmSettings *shmSettings = NULL;
@@ -391,6 +395,7 @@ void _lock_shm(const char *func, const int line, const char *file)

 	result = pthread_mutex_lock(&shmLock->lock.inner);

+	clock_gettime(CLOCK_MONOTONIC, &shmLock->time.begin);
 	log_debug(DEBUG_LOCKS, "Obtained SHM lock for %s() (%s:%i)", func, file, line);

 	if(result != 0)
@@ -433,6 +438,15 @@ void _unlock_shm(const char *func, const int line, const char * file)
 	if(result != 0)
 		log_err("Failed to unlock outer SHM lock: %s", strerror(result));

+	clock_gettime(CLOCK_MONOTONIC, &shmLock->time.end);
+	if(config.debug.timing.v.b)
+	{
+		const double lock_time = (shmLock->time.end.tv_sec - shmLock->time.begin.tv_sec) / 1000.0 +
+		                         (shmLock->time.end.tv_nsec - shmLock->time.begin.tv_nsec) / 1e6;
+		log_debug(DEBUG_TIMING, "SHM lock held for %.3f ms in %s() (%s:%i)",
+		          lock_time, func, file, line);
+	}
+
 	log_debug(DEBUG_LOCKS, "Removed SHM lock in %s() (%s:%i)", func, file, line);
 }

--- a/src/tools/gravity-parseList.c
+++ b/src/tools/gravity-parseList.c
@@ -272,6 +272,18 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis
 	unsigned int exact_domains = 0, abp_domains = 0, invalid_domains = 0;
 	while((read = getline(&line, &len, fpin)) != -1)
 	{
+
+		// Handle UTF-8 BOM (Byte Order Mark) if present at start of file
+		if (read >= 3 &&
+			(unsigned char)line[0] == 0xEF &&
+			(unsigned char)line[1] == 0xBB &&
+			(unsigned char)line[2] == 0xBF)
+		{
+			// Shift line contents left by 3 bytes to remove BOM
+			memmove(line, line + 3, read - 3);
+			read -= 3;
+		}
+
 		// Update total read bytes
 		total_read += read;
 		lineno++;
--- a/src/vector.c
+++ b/src/vector.c
@@ -78,10 +78,18 @@ void set_sqlite3_stmt_vec(sqlite3_stmt_vec *v, unsigned int index, sqlite3_stmt
 	if(index >= v->capacity)
 	{
 		// Allocate more memory when trying to set a statement vector entry with
-		// an index larger than the current array size (this makes set an
-		// equivalent alternative to append)
-		if(!resize_sqlite3_stmt_vec(v, index + VEC_ALLOC_STEP))
-			return;
+		// an index larger than the current array size. Use exponential growth
+		// for better performance with large datasets.
+		unsigned int new_capacity = v->capacity * VEC_GROWTH_FACTOR;
+		if(new_capacity <= index)
+			new_capacity = index + VEC_ALLOC_STEP;
+		// Overflow check
+		if(new_capacity > v->capacity)
+		{
+			// Resize vector
+			if(!resize_sqlite3_stmt_vec(v, new_capacity))
+				return;
+		}
 	}

 	// Set item
@@ -102,10 +110,11 @@ sqlite3_stmt * __attribute__((pure)) get_sqlite3_stmt_vec(sqlite3_stmt_vec *v, u

 	if(index >= v->capacity)
 	{
-		// Silently increase size of vector if trying to read out-of-bounds
-		// Return NULL if the allocation fails
-		if(!resize_sqlite3_stmt_vec(v, index + VEC_ALLOC_STEP))
-			return NULL;
+		// Silently return NULL when trying to get a statement vector
+		// entry with an index larger than the current array size. The
+		// code will later initiate a refreshing of the prepared
+		// statements in this case.
+		return NULL;
 	}

 	sqlite3_stmt* item = v->items[index];
--- a/src/vector.h
+++ b/src/vector.h
@@ -20,6 +20,7 @@
 #include "database/sqlite3.h"

 #define VEC_ALLOC_STEP 10u
+#define VEC_GROWTH_FACTOR 2u  // For exponential growth when expanding beyond initial capacity

 typedef struct sqlite3_stmt_vec {
 	unsigned int capacity;
--- a/src/webserver/webserver.c
+++ b/src/webserver/webserver.c
@@ -166,12 +166,6 @@ static int redirect_root_handler(struct mg_connection *conn, void *input)
 		// 308 Permanent Redirect from http://pi.hole -> http://pi.hole/admin/
 		if(strcmp(uri, "/") == 0 || strcmp(uri, config.webserver.paths.prefix.v.s) == 0)
 		{
-			if(strcmp(uri, prefix_webhome) == 0)
-			{
-				log_debug(DEBUG_API, "Not redirecting %s (matches webhome)",
-					  prefix_webhome);
-				return 0;
-			}
 			log_debug(DEBUG_API, "Redirecting / --308--> %s",
 			          prefix_webhome);
 			mg_send_http_redirect(conn, prefix_webhome, 308);
@@ -843,7 +837,14 @@ void http_init(void)
 	// prefix should be stripped away by the reverse proxy
 	mg_set_request_handler(ctx, "/api", api_handler, NULL);

-	mg_set_request_handler(ctx, "/$", redirect_root_handler, NULL);
+	if(strcmp(prefix_webhome, "/") == 0)
+	{
+		log_debug(DEBUG_API, "Not redirecting root since webhome is '%s'",
+			  prefix_webhome);
+	} else {
+		// Redirect requests to / to the webhome path.
+		mg_set_request_handler(ctx, "/$", redirect_root_handler, NULL);
+	}

 	if(strcmp(config.webserver.paths.webhome.v.s, "/") == 0 &&
 	   config.dns.blocking.mode.v.blocking_mode == MODE_IP)
--- a/test/pihole.toml
+++ b/test/pihole.toml
@@ -1,7 +1,7 @@
-# Pi-hole configuration file (v6.2.3-312-g9f1eb768-dirty) on branch (no branch, rebasing misc_dnsmasq_warn)
+# Pi-hole configuration file (v6.3.2-8-g58a618e2-dirty) on branch tweak/db_performance
 # Encoding: UTF-8
 # This file is managed by pihole-FTL
-# Last updated on 2025-10-23 17:03:51 UTC
+# Last updated on 2025-11-01 13:12:20 UTC

 [dns]
  # Upstream DNS Servers to be used by Pi-hole. If this is not set, Pi-hole will not
@@ -1667,14 +1667,20 @@
  #     true or false
  netlink = true ### CHANGED, default = false

+  # Print timing information from various parts of FTL
+  #
+  # Allowed values are:
+  #     true or false
+  timing = true ### CHANGED, default = false
+
  # Set all debug flags at once. This is a convenience option to enable all debug flags
  # at once. Note that this option is not persistent, setting it to true will enable all
  # *remaining* debug flags but unsetting it will disable *all* debug flags.
  all = true ### CHANGED, default = false

 # Configuration statistics:
-# 161 total entries out of which 108 entries are default
-# --> 53 entries are modified
+# 162 total entries out of which 108 entries are default
+# --> 54 entries are modified
 # 3 entries are forced through environment:
 #   - misc.nice
 #   - misc.check.shmem