From a9d6884b5a5a2f02f8f8255d7ef398d30354a19f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 11 Jul 2025 21:08:28 +0100 Subject: [PATCH 01/26] Implement performance optimizations: string processing, vector growth, and compiler flags Co-authored-by: DL6ER <16748619+DL6ER@users.noreply.github.com> --- .github/release.yml | 1 + src/CMakeLists.txt | 2 +- src/args.c | 10 +++++++++ src/datastructure.c | 53 +++++++++++++++++++++++++++------------------ src/regex.c | 2 +- src/vector.c | 18 ++++++++++----- src/vector.h | 1 + 7 files changed, 59 insertions(+), 28 deletions(-) diff --git a/.github/release.yml b/.github/release.yml index 2e8776e9..e12a3ff7 100644 --- a/.github/release.yml +++ b/.github/release.yml @@ -2,6 +2,7 @@ changelog: exclude: labels: - internal + - dependencies authors: - dependabot - github-actions diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cc265087..a6df73b3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -216,7 +216,7 @@ endif() set(CMAKE_C_FLAGS "-std=c17 -pipe ${WARN_FLAGS} -D_FILE_OFFSET_BITS=64 ${HARDENING_FLAGS} ${DEBUG_FLAGS} ${CMAKE_C_FLAGS} -DHAVE_POLL_H ${SQLITE_DEFINES}") set(CMAKE_C_FLAGS_DEBUG "-O0 -g3") -set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") +set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG -funroll-loops") set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELEASE} -g3") set(CMAKE_C_FLAGS_MINSIZEREL "-Os -DNDEBUG") diff --git a/src/args.c b/src/args.c index af4e626d..ad020cc7 100644 --- a/src/args.c +++ b/src/args.c @@ -408,6 +408,7 @@ void parse_args(int argc, char *argv[]) // Generate X.509 certificate if(argc > 1 && strcmp(argv[1], "--gen-x509") == 0) { +#ifdef HAVE_MBEDTLS if(argc < 3 || argc > 5) { printf("Usage: %s --gen-x509 [] [rsa]\n", argv[0]); @@ -427,6 +428,10 @@ void parse_args(int argc, char *argv[]) const bool rsa = argc > 4 && strcasecmp(argv[4], "rsa") == 0; exit(generate_certificate(argv[2], rsa, domain, config.webserver.tls.validity.v.ui) ? EXIT_SUCCESS : EXIT_FAILURE); +#else + printf("Error: FTL was compiled without TLS support. Certificate generation is not available.\n"); + exit(EXIT_FAILURE); +#endif } // Parse X.509 certificate @@ -434,6 +439,7 @@ void parse_args(int argc, char *argv[]) (strcmp(argv[1], "--read-x509") == 0 || strcmp(argv[1], "--read-x509-key") == 0)) { +#ifdef HAVE_MBEDTLS if(argc > 4) { printf("Usage: %s %s [] []\n", argv[0], argv[1]); @@ -476,6 +482,10 @@ void parse_args(int argc, char *argv[]) printf("Certificate does not match domain %s\n", argv[3]); exit(EXIT_FAILURE); } +#else + printf("Error: FTL was compiled without TLS support. Certificate reading is not available.\n"); + exit(EXIT_FAILURE); +#endif } // If the first argument is "gravity" (e.g., /usr/bin/pihole-FTL gravity), diff --git a/src/datastructure.c b/src/datastructure.c index 169c7af5..9bd1cf22 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -32,10 +32,11 @@ #include "lookup-table.h" // converts upper to lower case, and leaves other characters unchanged +// Optimized version using pointer arithmetic for better performance void strtolower(char *str) { - int i = 0; - while(str[i]){ str[i] = tolower(str[i]); i++; } + for(; *str; ++str) + *str = tolower(*str); } /** @@ -54,7 +55,7 @@ void strtolower(char *str) */ static uint32_t __attribute__ ((pure)) hashStr(const char *s) { - // Jenkins' One-at-a-Time hash + // Jenkins' One-at-a-Time hash (optimized version) // (http://www.burtleburtle.net/bob/hash/doobs.html) uint32_t hash = 0; for(; *s; ++s) @@ -64,6 +65,7 @@ static uint32_t __attribute__ ((pure)) hashStr(const char *s) hash ^= hash >> 6; } + // Final mixing to ensure good distribution hash += hash << 3; hash ^= hash >> 11; hash += hash << 15; @@ -997,16 +999,19 @@ const char * __attribute__ ((pure)) get_blocked_statuslist(void) unsigned int first = 0; // Open parenthesis blocked_list[0] = '('; + size_t pos = 1; // Track current position instead of calling strlen repeatedly for(enum query_status status = 0; status < QUERY_STATUS_MAX; status++) if(is_blocked(status)) - snprintf(blocked_list + strlen(blocked_list), - sizeof(blocked_list) - strlen(blocked_list), - "%s%d", first++ < 1 ? "" : ",", status); + { + int written = snprintf(blocked_list + pos, sizeof(blocked_list) - pos, + "%s%d", first++ < 1 ? "" : ",", status); + if(written > 0) + pos += written; + } // Close parenthesis - const size_t len = strlen(blocked_list); - blocked_list[len] = ')'; - blocked_list[len + 1] = '\0'; + blocked_list[pos] = ')'; + blocked_list[pos + 1] = '\0'; return blocked_list; } @@ -1020,16 +1025,19 @@ const char * __attribute__ ((pure)) get_cached_statuslist(void) unsigned int first = 0; // Open parenthesis cached_list[0] = '('; + size_t pos = 1; // Track current position instead of calling strlen repeatedly for(enum query_status status = 0; status < QUERY_STATUS_MAX; status++) if(is_cached(status)) - snprintf(cached_list + strlen(cached_list), - sizeof(cached_list) - strlen(cached_list), - "%s%d", first++ < 1 ? "" : ",", status); + { + int written = snprintf(cached_list + pos, sizeof(cached_list) - pos, + "%s%d", first++ < 1 ? "" : ",", status); + if(written > 0) + pos += written; + } // Close parenthesis - const size_t len = strlen(cached_list); - cached_list[len] = ')'; - cached_list[len + 1] = '\0'; + cached_list[pos] = ')'; + cached_list[pos + 1] = '\0'; return cached_list; } @@ -1043,16 +1051,19 @@ const char * __attribute__ ((pure)) get_permitted_statuslist(void) unsigned int first = 0; // Open parenthesis permitted_list[0] = '('; + size_t pos = 1; // Track current position instead of calling strlen repeatedly for(enum query_status status = 0; status < QUERY_STATUS_MAX; status++) if(!is_blocked(status)) - snprintf(permitted_list + strlen(permitted_list), - sizeof(permitted_list) - strlen(permitted_list), - "%s%d", first++ < 1 ? "" : ",", status); + { + int written = snprintf(permitted_list + pos, sizeof(permitted_list) - pos, + "%s%d", first++ < 1 ? "" : ",", status); + if(written > 0) + pos += written; + } // Close parenthesis - const size_t len = strlen(permitted_list); - permitted_list[len] = ')'; - permitted_list[len + 1] = '\0'; + permitted_list[pos] = ')'; + permitted_list[pos + 1] = '\0'; return permitted_list; } diff --git a/src/regex.c b/src/regex.c index ecb78262..86784d9f 100644 --- a/src/regex.c +++ b/src/regex.c @@ -151,7 +151,7 @@ bool compile_regex(const char *regexin, regexData *regex, char **message) // Extract regular expression pattern in front of FTL-specific syntax char *saveptr = NULL; char *part = strtok_r(buf, FTL_REGEX_SEP, &saveptr); - strncpy(rgxbuf, part, strlen(regexin)); + strncpy(rgxbuf, part, strlen(part)); // Analyze FTL-specific parts while((part = strtok_r(NULL, FTL_REGEX_SEP, &saveptr)) != NULL) diff --git a/src/vector.c b/src/vector.c index 937655d7..e7b1cdc5 100644 --- a/src/vector.c +++ b/src/vector.c @@ -78,9 +78,13 @@ void set_sqlite3_stmt_vec(sqlite3_stmt_vec *v, unsigned int index, sqlite3_stmt if(index >= v->capacity) { // Allocate more memory when trying to set a statement vector entry with - // an index larger than the current array size (this makes set an - // equivalent alternative to append) - if(!resize_sqlite3_stmt_vec(v, index + VEC_ALLOC_STEP)) + // an index larger than the current array size. Use exponential growth + // for better performance with large datasets. + unsigned int new_capacity = v->capacity * VEC_GROWTH_FACTOR; + if(new_capacity <= index) + new_capacity = index + VEC_ALLOC_STEP; + + if(!resize_sqlite3_stmt_vec(v, new_capacity)) return; } @@ -103,8 +107,12 @@ sqlite3_stmt * __attribute__((pure)) get_sqlite3_stmt_vec(sqlite3_stmt_vec *v, u if(index >= v->capacity) { // Silently increase size of vector if trying to read out-of-bounds - // Return NULL if the allocation fails - if(!resize_sqlite3_stmt_vec(v, index + VEC_ALLOC_STEP)) + // Use exponential growth for better performance with large datasets. + unsigned int new_capacity = v->capacity * VEC_GROWTH_FACTOR; + if(new_capacity <= index) + new_capacity = index + VEC_ALLOC_STEP; + + if(!resize_sqlite3_stmt_vec(v, new_capacity)) return NULL; } diff --git a/src/vector.h b/src/vector.h index a3b77fd0..ef116618 100644 --- a/src/vector.h +++ b/src/vector.h @@ -20,6 +20,7 @@ #include "database/sqlite3.h" #define VEC_ALLOC_STEP 10u +#define VEC_GROWTH_FACTOR 2u // For exponential growth when expanding beyond initial capacity typedef struct sqlite3_stmt_vec { unsigned int capacity; From cd1e96efa2951cca0938d0c5ae79eebfb0b3e2c2 Mon Sep 17 00:00:00 2001 From: Dominik Date: Wed, 17 Sep 2025 09:10:53 +0200 Subject: [PATCH 02/26] Address review comments Signed-off-by: Dominik --- src/datastructure.c | 6 +++--- src/vector.c | 23 ++++++++++++----------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/datastructure.c b/src/datastructure.c index 9bd1cf22..e05f7756 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -1005,7 +1005,7 @@ const char * __attribute__ ((pure)) get_blocked_statuslist(void) { int written = snprintf(blocked_list + pos, sizeof(blocked_list) - pos, "%s%d", first++ < 1 ? "" : ",", status); - if(written > 0) + if(written > 0 && (size_t)written < sizeof(blocked_list) - pos) pos += written; } @@ -1031,7 +1031,7 @@ const char * __attribute__ ((pure)) get_cached_statuslist(void) { int written = snprintf(cached_list + pos, sizeof(cached_list) - pos, "%s%d", first++ < 1 ? "" : ",", status); - if(written > 0) + if(written > 0 && (size_t)written < sizeof(cached_list) - pos) pos += written; } @@ -1057,7 +1057,7 @@ const char * __attribute__ ((pure)) get_permitted_statuslist(void) { int written = snprintf(permitted_list + pos, sizeof(permitted_list) - pos, "%s%d", first++ < 1 ? "" : ",", status); - if(written > 0) + if(written > 0 && (size_t)written < sizeof(permitted_list) - pos) pos += written; } diff --git a/src/vector.c b/src/vector.c index e7b1cdc5..9ec92c0b 100644 --- a/src/vector.c +++ b/src/vector.c @@ -83,9 +83,13 @@ void set_sqlite3_stmt_vec(sqlite3_stmt_vec *v, unsigned int index, sqlite3_stmt unsigned int new_capacity = v->capacity * VEC_GROWTH_FACTOR; if(new_capacity <= index) new_capacity = index + VEC_ALLOC_STEP; - - if(!resize_sqlite3_stmt_vec(v, new_capacity)) - return; + // Overflow check + if(new_capacity > v->capacity) + { + // Resize vector + if(!resize_sqlite3_stmt_vec(v, new_capacity)) + return; + } } // Set item @@ -106,14 +110,11 @@ sqlite3_stmt * __attribute__((pure)) get_sqlite3_stmt_vec(sqlite3_stmt_vec *v, u if(index >= v->capacity) { - // Silently increase size of vector if trying to read out-of-bounds - // Use exponential growth for better performance with large datasets. - unsigned int new_capacity = v->capacity * VEC_GROWTH_FACTOR; - if(new_capacity <= index) - new_capacity = index + VEC_ALLOC_STEP; - - if(!resize_sqlite3_stmt_vec(v, new_capacity)) - return NULL; + // Silently return NULL when trying to get a statement vector + // entry with an index larger than the current array size. The + // code will later initiate a refreshing of the prepared + // statements in this case. + return NULL; } sqlite3_stmt* item = v->items[index]; From eace39c5f25949d83cf0282a9012fe25369c1125 Mon Sep 17 00:00:00 2001 From: averyvigolo Date: Sun, 24 Aug 2025 18:35:37 +0100 Subject: [PATCH 03/26] Only use redirect_root_handler if webhome is set (fixes #2518) This fixes an infinite redirect on the home page when authentication is enabled. The redirects are caused by XHR calls to the API, which receive a 401 response, and the error handlers simply reload the page. If webhome is not set, the default request handler should be used to properly handle authentication. So, conditionally enable redirect_root_handler, if webhome is not empty or `/`. This fixes the problem, as there's an immediate redirect to /login, before any XHR calls. Remove the initial fix in https://github.com/pi-hole/FTL/pull/2521, as it is no longer necessary. That fix involved checking in redirect_root_handler, if the redirect destination is the same as the request URI. Signed-off-by: averyvigolo --- src/webserver/webserver.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 6323afd4..df7f3e30 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -166,12 +166,6 @@ static int redirect_root_handler(struct mg_connection *conn, void *input) // 308 Permanent Redirect from http://pi.hole -> http://pi.hole/admin/ if(strcmp(uri, "/") == 0 || strcmp(uri, config.webserver.paths.prefix.v.s) == 0) { - if(strcmp(uri, prefix_webhome) == 0) - { - log_debug(DEBUG_API, "Not redirecting %s (matches webhome)", - prefix_webhome); - return 0; - } log_debug(DEBUG_API, "Redirecting / --308--> %s", prefix_webhome); mg_send_http_redirect(conn, prefix_webhome, 308); @@ -843,7 +837,14 @@ void http_init(void) // prefix should be stripped away by the reverse proxy mg_set_request_handler(ctx, "/api", api_handler, NULL); - mg_set_request_handler(ctx, "/$", redirect_root_handler, NULL); + if(strcmp(prefix_webhome, "/") == 0) + { + log_debug(DEBUG_API, "Not redirecting root since webhome is '%s'", + prefix_webhome); + } else { + // Redirect requests to / to the webhome path. + mg_set_request_handler(ctx, "/$", redirect_root_handler, NULL); + } if(strcmp(config.webserver.paths.webhome.v.s, "/") == 0 && config.dns.blocking.mode.v.blocking_mode == MODE_IP) From 58a618e233554a152b23ff0ac931a9f7d2f74804 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 1 Nov 2025 13:58:38 +0100 Subject: [PATCH 04/26] Add debug.db_timing to allow for timing of database functions in the DB thread Signed-off-by: Dominik --- src/api/docs/content/specs/config.yaml | 3 +++ src/config/config.c | 6 ++++++ src/config/config.h | 1 + src/database/common.h | 12 ++++++++++++ src/database/database-thread.c | 14 +++++++------- src/enums.h | 1 + src/log.c | 2 ++ 7 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 9e4c9be9..961f488d 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -625,6 +625,8 @@ components: type: boolean netlink: type: boolean + db_timing: + type: boolean all: type: boolean topics: @@ -874,6 +876,7 @@ components: reserved: false ntp: false netlink: false + db_timing: false all: false took: 0.003 config_one: diff --git a/src/config/config.c b/src/config/config.c index bb012c15..021a5278 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1625,6 +1625,12 @@ void initConfig(struct config *conf) conf->debug.netlink.d.b = false; conf->debug.netlink.c = validate_stub; // Only type-based checking + conf->debug.db_timing.k = "debug.db_timing"; + conf->debug.db_timing.h = "Print timing information about database operations"; + conf->debug.db_timing.t = CONF_BOOL; + conf->debug.db_timing.d.b = false; + conf->debug.db_timing.c = validate_stub; // Only type-based checking + conf->debug.all.k = "debug.all"; conf->debug.all.h = "Set all debug flags at once. This is a convenience option to enable all debug flags at once. Note that this option is not persistent, setting it to true will enable all *remaining* debug flags but unsetting it will disable *all* debug flags."; conf->debug.all.t = CONF_ALL_DEBUG_BOOL; diff --git a/src/config/config.h b/src/config/config.h index 0ad68605..2ee7dc64 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -358,6 +358,7 @@ struct config { struct conf_item reserved; struct conf_item ntp; struct conf_item netlink; + struct conf_item db_timing; // all must be the last item in this struct struct conf_item all; } debug; diff --git a/src/database/common.h b/src/database/common.h index 6a26a374..6ebad0ef 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -89,4 +89,16 @@ extern const char *sqlite3ErrName(int rc); }\ } +// Macro to time a database operation expression EXPR if debug.db_timing is +// enabled. +#define TIMED_DB_OP(EXPR) do { \ + if(!config.debug.db_timing.v.b) { EXPR; break; } \ + struct timespec _timed_start, _timed_end; \ + clock_gettime(CLOCK_MONOTONIC, &_timed_start); \ + (EXPR); \ + clock_gettime(CLOCK_MONOTONIC, &_timed_end); \ + long _timed_elapsed = (_timed_end.tv_sec - _timed_start.tv_sec) * 1000 + (_timed_end.tv_nsec - _timed_start.tv_nsec) / 1000000; \ + log_debug(DEBUG_DB_TIMING, "Database operation %s took %ld ms", str(EXPR), _timed_elapsed); \ + } while(0) + #endif //DATABASE_COMMON_H diff --git a/src/database/database-thread.c b/src/database/database-thread.c index 50537e14..0ffed501 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -136,7 +136,7 @@ void *DB_thread(void *val) // Save data to database DBOPEN_OR_AGAIN(); lock_shm(); - export_queries_to_disk(false); + TIMED_DB_OP(export_queries_to_disk(false)); unlock_shm(); // Intermediate cancellation-point @@ -146,8 +146,8 @@ void *DB_thread(void *val) // Check if GC should be done on the database if(DBdeleteoldqueries) { - // No thread locks needed - delete_old_queries_in_DB(db); + /* No thread locks needed */ + TIMED_DB_OP(delete_old_queries_in_DB(db)); DBdeleteoldqueries = false; } @@ -165,7 +165,7 @@ void *DB_thread(void *val) if(now - lastAnalyze >= DATABASE_ANALYZE_INTERVAL) { DBOPEN_OR_AGAIN(); - analyze_database(db); + TIMED_DB_OP(analyze_database(db)); lastAnalyze = now; DBCLOSE_OR_BREAK(); } @@ -179,7 +179,7 @@ void *DB_thread(void *val) if(now - lastMACVendor >= DATABASE_MACVENDOR_INTERVAL) { DBOPEN_OR_AGAIN(); - updateMACVendorRecords(db); + TIMED_DB_OP(updateMACVendorRecords(db)); lastMACVendor = now; DBCLOSE_OR_BREAK(); } @@ -192,7 +192,7 @@ void *DB_thread(void *val) if(get_and_clear_event(PARSE_NEIGHBOR_CACHE)) { DBOPEN_OR_AGAIN(); - parse_neighbor_cache(db); + TIMED_DB_OP(parse_neighbor_cache(db)); DBCLOSE_OR_BREAK(); } @@ -204,7 +204,7 @@ void *DB_thread(void *val) { DBOPEN_OR_AGAIN(); lock_shm(); - reimport_aliasclients(db); + TIMED_DB_OP(reimport_aliasclients(db)); unlock_shm(); DBCLOSE_OR_BREAK(); } diff --git a/src/enums.h b/src/enums.h index ca524783..1accf0a3 100644 --- a/src/enums.h +++ b/src/enums.h @@ -156,6 +156,7 @@ enum debug_flag { DEBUG_RESERVED, DEBUG_NTP, DEBUG_NETLINK, + DEBUG_DB_TIMING, DEBUG_MAX } __attribute__ ((packed)); diff --git a/src/log.c b/src/log.c index c5612d0e..ac23c81c 100644 --- a/src/log.c +++ b/src/log.c @@ -224,6 +224,8 @@ const char *debugstr(const enum debug_flag flag) return "DEBUG_NTP"; case DEBUG_NETLINK: return "DEBUG_NETLINK"; + case DEBUG_DB_TIMING: + return "DEBUG_DB_TIMING"; case DEBUG_MAX: return "DEBUG_MAX"; case DEBUG_NONE: // fall through From c254ff735ae23f91b575356f75f4ddd94811d75b Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 1 Nov 2025 14:35:39 +0100 Subject: [PATCH 05/26] Also add timing for locked regions and rename it to debug.timing Signed-off-by: Dominik --- src/api/config.c | 2 +- src/api/docs/content/specs/config.yaml | 4 ++-- src/config/config.c | 10 +++++----- src/config/config.h | 2 +- src/database/common.h | 8 ++++---- src/dnsmasq_interface.c | 2 +- src/enums.h | 2 +- src/log.c | 4 ++-- src/shmem.c | 14 ++++++++++++++ test/pihole.toml | 14 ++++++++++---- 10 files changed, 41 insertions(+), 21 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index 3451c9e3..b7f3198c 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -22,7 +22,7 @@ #include "config/toml_writer.h" // write_dnsmasq_config() #include "config/dnsmasq_config.h" -// shm_lock() +// lock_shm() #include "shmem.h" // hash_password() #include "config/password.h" diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 961f488d..da0bdbab 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -625,7 +625,7 @@ components: type: boolean netlink: type: boolean - db_timing: + timing: type: boolean all: type: boolean @@ -876,7 +876,7 @@ components: reserved: false ntp: false netlink: false - db_timing: false + timing: false all: false took: 0.003 config_one: diff --git a/src/config/config.c b/src/config/config.c index 021a5278..c3e3af2c 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1625,11 +1625,11 @@ void initConfig(struct config *conf) conf->debug.netlink.d.b = false; conf->debug.netlink.c = validate_stub; // Only type-based checking - conf->debug.db_timing.k = "debug.db_timing"; - conf->debug.db_timing.h = "Print timing information about database operations"; - conf->debug.db_timing.t = CONF_BOOL; - conf->debug.db_timing.d.b = false; - conf->debug.db_timing.c = validate_stub; // Only type-based checking + conf->debug.timing.k = "debug.timing"; + conf->debug.timing.h = "Print timing information from various parts of FTL"; + conf->debug.timing.t = CONF_BOOL; + conf->debug.timing.d.b = false; + conf->debug.timing.c = validate_stub; // Only type-based checking conf->debug.all.k = "debug.all"; conf->debug.all.h = "Set all debug flags at once. This is a convenience option to enable all debug flags at once. Note that this option is not persistent, setting it to true will enable all *remaining* debug flags but unsetting it will disable *all* debug flags."; diff --git a/src/config/config.h b/src/config/config.h index 2ee7dc64..877c0a0d 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -358,7 +358,7 @@ struct config { struct conf_item reserved; struct conf_item ntp; struct conf_item netlink; - struct conf_item db_timing; + struct conf_item timing; // all must be the last item in this struct struct conf_item all; } debug; diff --git a/src/database/common.h b/src/database/common.h index 6ebad0ef..647efc49 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -89,16 +89,16 @@ extern const char *sqlite3ErrName(int rc); }\ } -// Macro to time a database operation expression EXPR if debug.db_timing is +// Macro to time a database operation expression EXPR if debug.timing is // enabled. #define TIMED_DB_OP(EXPR) do { \ - if(!config.debug.db_timing.v.b) { EXPR; break; } \ + if(!config.debug.timing.v.b) { EXPR; break; } \ struct timespec _timed_start, _timed_end; \ clock_gettime(CLOCK_MONOTONIC, &_timed_start); \ (EXPR); \ clock_gettime(CLOCK_MONOTONIC, &_timed_end); \ - long _timed_elapsed = (_timed_end.tv_sec - _timed_start.tv_sec) * 1000 + (_timed_end.tv_nsec - _timed_start.tv_nsec) / 1000000; \ - log_debug(DEBUG_DB_TIMING, "Database operation %s took %ld ms", str(EXPR), _timed_elapsed); \ + long _timed_elapsed = (_timed_end.tv_sec - _timed_start.tv_sec) * 10000 + (_timed_end.tv_nsec - _timed_start.tv_nsec) / 100000; \ + log_debug(DEBUG_TIMING, "Database operation %s took %.1f ms", str(EXPR), 0.1*_timed_elapsed); \ } while(0) #endif //DATABASE_COMMON_H diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index eb84f04f..133f8c8d 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -48,7 +48,7 @@ #include "webserver/webserver.h" // type struct sqlite3_stmt_vec #include "vector.h" -// query_to_database() +// init_memory_database() #include "database/query-table.h" // reread_config() #include "config/config.h" diff --git a/src/enums.h b/src/enums.h index 1accf0a3..99981b78 100644 --- a/src/enums.h +++ b/src/enums.h @@ -156,7 +156,7 @@ enum debug_flag { DEBUG_RESERVED, DEBUG_NTP, DEBUG_NETLINK, - DEBUG_DB_TIMING, + DEBUG_TIMING, DEBUG_MAX } __attribute__ ((packed)); diff --git a/src/log.c b/src/log.c index ac23c81c..5a1d92f3 100644 --- a/src/log.c +++ b/src/log.c @@ -224,8 +224,8 @@ const char *debugstr(const enum debug_flag flag) return "DEBUG_NTP"; case DEBUG_NETLINK: return "DEBUG_NETLINK"; - case DEBUG_DB_TIMING: - return "DEBUG_DB_TIMING"; + case DEBUG_TIMING: + return "DEBUG_TIMING"; case DEBUG_MAX: return "DEBUG_MAX"; case DEBUG_NONE: // fall through diff --git a/src/shmem.c b/src/shmem.c index e3b83945..5e270f2a 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -134,6 +134,10 @@ typedef struct { volatile pid_t pid; volatile pid_t tid; } owner; + struct { + struct timespec begin; + struct timespec end; + } time; } ShmLock; static ShmLock *shmLock = NULL; static ShmSettings *shmSettings = NULL; @@ -391,6 +395,7 @@ void _lock_shm(const char *func, const int line, const char *file) result = pthread_mutex_lock(&shmLock->lock.inner); + clock_gettime(CLOCK_MONOTONIC, &shmLock->time.begin); log_debug(DEBUG_LOCKS, "Obtained SHM lock for %s() (%s:%i)", func, file, line); if(result != 0) @@ -433,6 +438,15 @@ void _unlock_shm(const char *func, const int line, const char * file) if(result != 0) log_err("Failed to unlock outer SHM lock: %s", strerror(result)); + clock_gettime(CLOCK_MONOTONIC, &shmLock->time.end); + if(config.debug.timing.v.b) + { + const double lock_time = (shmLock->time.end.tv_sec - shmLock->time.begin.tv_sec) / 1000.0 + + (shmLock->time.end.tv_nsec - shmLock->time.begin.tv_nsec) / 1e6; + log_debug(DEBUG_TIMING, "SHM lock held for %.3f ms in %s() (%s:%i)", + lock_time, func, file, line); + } + log_debug(DEBUG_LOCKS, "Removed SHM lock in %s() (%s:%i)", func, file, line); } diff --git a/test/pihole.toml b/test/pihole.toml index 4c52db6a..58fb443c 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -1,7 +1,7 @@ -# Pi-hole configuration file (v6.2.3-312-g9f1eb768-dirty) on branch (no branch, rebasing misc_dnsmasq_warn) +# Pi-hole configuration file (v6.3.2-8-g58a618e2-dirty) on branch tweak/db_performance # Encoding: UTF-8 # This file is managed by pihole-FTL -# Last updated on 2025-10-23 17:03:51 UTC +# Last updated on 2025-11-01 13:12:20 UTC [dns] # Upstream DNS Servers to be used by Pi-hole. If this is not set, Pi-hole will not @@ -1667,14 +1667,20 @@ # true or false netlink = true ### CHANGED, default = false + # Print timing information from various parts of FTL + # + # Allowed values are: + # true or false + timing = true ### CHANGED, default = false + # Set all debug flags at once. This is a convenience option to enable all debug flags # at once. Note that this option is not persistent, setting it to true will enable all # *remaining* debug flags but unsetting it will disable *all* debug flags. all = true ### CHANGED, default = false # Configuration statistics: -# 161 total entries out of which 108 entries are default -# --> 53 entries are modified +# 162 total entries out of which 108 entries are default +# --> 54 entries are modified # 3 entries are forced through environment: # - misc.nice # - misc.check.shmem From f5a2a1f216d1730a8b0f3d2483ad71afd1ef4acb Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Wed, 5 Nov 2025 13:55:04 -0300 Subject: [PATCH 06/26] Fix API specs and example for dns.upstreams in config.yaml Replace "dnsmasq/upstreams" with the correct "dns/upstreams" Signed-off-by: RD WebDesign --- src/api/docs/content/specs/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 9e4c9be9..7a2602fe 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -919,7 +919,7 @@ components: error: key: "bad_request" message: "Invalid path depth" - hint: "Use, e.g., DELETE /config/dnsmasq/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams" + hint: "Use, e.g., DELETE /config/dns/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams" took: 0.003 item_already_present: summary: Item to be added exists already @@ -946,7 +946,7 @@ components: type: string required: true description: config element - example: "dnsmasq/upstreams" + example: "dns/upstreams" value: in: path name: value From 94a05cb9e111c733399a1d26d4ee4535f2b22739 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Wed, 5 Nov 2025 14:21:36 -0300 Subject: [PATCH 07/26] Also fix the comments Update comments to use the correct options (`config/dns` instead of `config/dnsmasq`) on the examples Signed-off-by: RD WebDesign --- src/api/config.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index 3451c9e3..e2d12b72 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -500,8 +500,8 @@ int get_json_config(struct ftl_conn *api, cJSON *json, const bool detailed) continue; // Check equality of paths up to the requested level (if any) // Examples: - // requested was /config/dnsmasq -> skip all entries that do not start in dnsmasq. - // requested was /config/dnsmasq/dhcp -> skip all entries that do not start in dhcp + // requested was /config/dns -> skip all entries that do not start in dnsmasq. + // requested was /config/dns/dhcp -> skip all entries that do not start in dhcp // etc. if(!check_paths_equal(conf_item->p, requested_path, min_level - 1)) continue; From c2b0e64aeca7125a943890e92aacdf22679cbb2b Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Thu, 6 Nov 2025 16:21:02 -0300 Subject: [PATCH 08/26] Apply suggestion Co-authored-by: Dominik Signed-off-by: RD WebDesign --- src/api/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/config.c b/src/api/config.c index e2d12b72..e0868afa 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -500,7 +500,7 @@ int get_json_config(struct ftl_conn *api, cJSON *json, const bool detailed) continue; // Check equality of paths up to the requested level (if any) // Examples: - // requested was /config/dns -> skip all entries that do not start in dnsmasq. + // requested was /config/dns -> skip all entries that do not start in dns // requested was /config/dns/dhcp -> skip all entries that do not start in dhcp // etc. if(!check_paths_equal(conf_item->p, requested_path, min_level - 1)) From 5d67f43493c3c0df326bb7e0c351c0e4df0155e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 8 Nov 2025 10:05:04 +0000 Subject: [PATCH 09/26] Bump the github_action-dependencies group across 1 directory with 2 updates Bumps the github_action-dependencies group with 2 updates in the / directory: [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) and [codespell-project/actions-codespell](https://github.com/codespell-project/actions-codespell). Updates `docker/setup-qemu-action` from 3.6.0 to 3.7.0 - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/29109295f81e9208d7d86ff1c6c12d2833863392...c7c53464625b32c7a7e944ae62b3e17d2b600130) Updates `codespell-project/actions-codespell` from 2.1 to 2.2 - [Release notes](https://github.com/codespell-project/actions-codespell/releases) - [Commits](https://github.com/codespell-project/actions-codespell/compare/406322ec52dd7b488e48c1c4b82e2a8b3a1bf630...8f01853be192eb0f849a5c7d721450e7a467c579) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-version: 3.7.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github_action-dependencies - dependency-name: codespell-project/actions-codespell dependency-version: '2.2' dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github_action-dependencies ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- .github/workflows/codespell.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ddb812b0..18b650e0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -104,7 +104,7 @@ jobs: # QEMU should come before Buildx - name: Set up QEMU - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 #v3.6.0 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 #v3.7.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 #v3.11.1 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 3e9c4a51..3c4861c4 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 - name: Spell-Checking - uses: codespell-project/actions-codespell@406322ec52dd7b488e48c1c4b82e2a8b3a1bf630 #v2.1 + uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 #v2.2 with: ignore_words_file: .github/.codespellignore skip: ./src/database/sqlite3.c,./src/database/sqlite3.h,./src/database/shell.c,./src/lua,./src/dnsmasq,./src/tre-regex,./.git,./test/libs,./src/webserver/civetweb,./src/zip/miniz,./src/api/docs/content/external,./src/database/sqlite3_rsync.c,./package-lock.json,./src/config/tomlc17 From 9423573404730bbdcdd44248440fcbeeb90666e2 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 8 Nov 2025 12:16:01 +0100 Subject: [PATCH 10/26] We do not need to loch the SHM objects during export to disk database inside a TRANSACTION Signed-off-by: Dominik --- src/database/database-thread.c | 2 -- src/database/query-table.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/database/database-thread.c b/src/database/database-thread.c index 50537e14..a3396628 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -135,9 +135,7 @@ void *DB_thread(void *val) // Save data to database DBOPEN_OR_AGAIN(); - lock_shm(); export_queries_to_disk(false); - unlock_shm(); // Intermediate cancellation-point if(killed) diff --git a/src/database/query-table.c b/src/database/query-table.c index 322e2c38..d0974591 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -640,7 +640,7 @@ bool export_queries_to_disk(const bool final) // Only try to export to database if it is known to not be broken if(FTLDBerror()) - return false; + return false; // Start database timer timer_start(DATABASE_WRITE_TIMER); From 6a141f620d1ec668f03178c9e3500a169c0283ee Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 8 Nov 2025 12:16:01 +0100 Subject: [PATCH 11/26] We do not need to lock the SHM objects during export to disk database inside a TRANSACTION Signed-off-by: Dominik --- src/database/database-thread.c | 2 -- src/database/query-table.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/database/database-thread.c b/src/database/database-thread.c index 0ffed501..dbbee41f 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -135,9 +135,7 @@ void *DB_thread(void *val) // Save data to database DBOPEN_OR_AGAIN(); - lock_shm(); TIMED_DB_OP(export_queries_to_disk(false)); - unlock_shm(); // Intermediate cancellation-point if(killed) diff --git a/src/database/query-table.c b/src/database/query-table.c index 322e2c38..d0974591 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -640,7 +640,7 @@ bool export_queries_to_disk(const bool final) // Only try to export to database if it is known to not be broken if(FTLDBerror()) - return false; + return false; // Start database timer timer_start(DATABASE_WRITE_TIMER); From 719eb57f44308028caa44623f4d1a1c564d0848c Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 8 Nov 2025 13:57:48 +0100 Subject: [PATCH 12/26] Limit the scan table size when exporting queries to disk. This will mean speedup espectially when only few entries are exported (typical case) Signed-off-by: Dominik --- src/database/query-table.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index d0974591..0e9f97d9 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -238,7 +238,7 @@ bool init_memory_database(void) // queries from the in-memory database (including the query with ID 0) // to the on-disk database. rc = sqlite3_prepare_v3(_memdb, "INSERT INTO disk.query_storage SELECT * FROM query_storage " \ - "WHERE id > IFNULL((SELECT MAX(id) FROM disk.query_storage), -1) "\ + "WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.query_storage) "\ "AND timestamp < ?", -1, SQLITE_PREPARE_PERSISTENT, &queries_to_disk_stmt, NULL); if( rc != SQLITE_OK ) @@ -247,11 +247,15 @@ bool init_memory_database(void) return false; } + // Export linking tables to disk database + // We limit the export to new records to avoid the overhead of many + // IGNORE executions for records that are already present on disk. It + // follows the same logic as for the main query_storage table above. const char *subtable_sql[SUBTABLE_STMTS] = { - "INSERT OR IGNORE INTO disk.domain_by_id SELECT * FROM domain_by_id", - "INSERT OR IGNORE INTO disk.client_by_id SELECT * FROM client_by_id", - "INSERT OR IGNORE INTO disk.forward_by_id SELECT * FROM forward_by_id", - "INSERT OR IGNORE INTO disk.addinfo_by_id SELECT * FROM addinfo_by_id", + "INSERT OR IGNORE INTO disk.domain_by_id SELECT * FROM domain_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.domain_by_id)", + "INSERT OR IGNORE INTO disk.client_by_id SELECT * FROM client_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.client_by_id)", + "INSERT OR IGNORE INTO disk.forward_by_id SELECT * FROM forward_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.forward_by_id)", + "INSERT OR IGNORE INTO disk.addinfo_by_id SELECT * FROM addinfo_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.addinfo_by_id)", "UPDATE disk.sqlite_sequence SET seq = (SELECT seq FROM sqlite_sequence WHERE disk.sqlite_sequence.name = sqlite_sequence.name)" }; @@ -504,7 +508,7 @@ bool detach_database(sqlite3* db, const char **message, const char *alias) return okay; } -// Get number of queries either in the temp or in the on-diks database +// Get number of queries either in the temp or in the on-disk database // This routine is used by the API routines. int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename) { From 06f5a42a7fb6877e9a5834683055d60dc70e88bf Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sun, 9 Nov 2025 14:30:59 +1000 Subject: [PATCH 13/26] gravity update - silently discard unicode BOM if present Some adlists in UTF-8 may contain unnecessary unicode BOM at the start of the file. This change silently discards these, if present, instead of flagging them as non-domain entries. Signed-off-by: Rob Gill --- src/tools/gravity-parseList.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index 5e543451..5375372e 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -272,6 +272,18 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis unsigned int exact_domains = 0, abp_domains = 0, invalid_domains = 0; while((read = getline(&line, &len, fpin)) != -1) { + + // Handle UTF-8 BOM (Byte Order Mark) if present at start of file + if (read >= 3 && + (unsigned char)line[0] == 0xEF && + (unsigned char)line[1] == 0xBB && + (unsigned char)line[2] == 0xBF) + { + // Shift line contents left by 3 bytes to remove BOM + memmove(line, line + 3, read - 3); + read -= 3; + } + // Update total read bytes total_read += read; lineno++; From a98fcbf21dcdaa32dfef654c201480722ec08469 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sun, 9 Nov 2025 08:52:29 +0100 Subject: [PATCH 14/26] Update embedded SQLite3 to 3.51.0 Signed-off-by: Dominik --- src/database/shell.c | 2534 +++++++++--------- src/database/sqlite3.c | 5683 ++++++++++++++++++++++++++++++---------- src/database/sqlite3.h | 413 ++- 3 files changed, 5833 insertions(+), 2797 deletions(-) diff --git a/src/database/shell.c b/src/database/shell.c index 4c9d2fb9..18177331 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -124,8 +124,9 @@ typedef sqlite3_uint64 u64; typedef unsigned char u8; #include #include -// print_FTL_version() -#include "../log.h" +#ifndef _WIN32 +# include +#endif #if !defined(_WIN32) && !defined(WIN32) # include @@ -650,8 +651,19 @@ static int cli_strncmp(const char *a, const char *b, size_t n){ return strncmp(a,b,n); } -/* Return the current wall-clock time */ +/* Return the current wall-clock time in microseconds since the +** Unix epoch (1970-01-01T00:00:00Z) +*/ static sqlite3_int64 timeOfDay(void){ +#if defined(_WIN64) + sqlite3_uint64 t; + FILETIME tm; + GetSystemTimePreciseAsFileTime(&tm); + t = ((u64)tm.dwHighDateTime<<32) | (u64)tm.dwLowDateTime; + t += 116444736000000000LL; + t /= 10; + return t; +#elif defined(_WIN32) static sqlite3_vfs *clockVfs = 0; sqlite3_int64 t; if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); @@ -663,7 +675,12 @@ static sqlite3_int64 timeOfDay(void){ clockVfs->xCurrentTime(clockVfs, &r); t = (sqlite3_int64)(r*86400000.0); } - return t; + return t*1000; +#else + struct timeval sNow; + (void)gettimeofday(&sNow,0); + return ((i64)sNow.tv_sec)*1000000 + sNow.tv_usec; +#endif } #if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) @@ -708,8 +725,8 @@ static void endTimer(FILE *out){ sqlite3_int64 iEnd = timeOfDay(); struct rusage sEnd; getrusage(RUSAGE_SELF, &sEnd); - sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", - (iEnd - iBegin)*0.001, + sqlite3_fprintf(out, "Run Time: real %.6f user %.6f sys %.6f\n", + (iEnd - iBegin)*0.000001, timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); } @@ -787,10 +804,19 @@ static void endTimer(FILE *out){ FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; sqlite3_int64 ftWallEnd = timeOfDay(); getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); - sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", - (ftWallEnd - ftWallBegin)*0.001, +#ifdef _WIN64 + /* microsecond precision on 64-bit windows */ + sqlite3_fprintf(out, "Run Time: real %.6f user %f sys %f\n", + (ftWallEnd - ftWallBegin)*0.000001, timeDiff(&ftUserBegin, &ftUserEnd), timeDiff(&ftKernelBegin, &ftKernelEnd)); +#else + /* millisecond precisino on 32-bit windows */ + sqlite3_fprintf(out, "Run Time: real %.3f user %.3f sys %.3f\n", + (ftWallEnd - ftWallBegin)*0.000001, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#endif } } @@ -1130,7 +1156,7 @@ static int decodeUtf8(const unsigned char *z, int *pU){ && (z[3] & 0xc0)==0x80 ){ *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 - | (z[4] & 0x3f); + | (z[3] & 0x3f); return 4; } *pU = 0; @@ -1193,14 +1219,24 @@ static int isVt100(const unsigned char *z){ ** Take into account zero-width and double-width Unicode characters. ** In other words, a zero-width character does not count toward the ** the w limit. A double-width character counts as two. +** +** w should normally be a small number. A couple hundred at most. This +** routine caps w at 100 million to avoid integer overflow issues. */ static void utf8_width_print(FILE *out, int w, const char *zUtf){ const unsigned char *a = (const unsigned char*)zUtf; + static const int mxW = 10000000; unsigned char c; int i = 0; int n = 0; int k; - int aw = w<0 ? -w : w; + int aw; + if( w<-mxW ){ + w = -mxW; + }else if( w>mxW ){ + w= mxW; + } + aw = w<0 ? -w : w; if( zUtf==0 ) zUtf = ""; while( (c = a[i])!=0 ){ if( (c&0xc0)==0xc0 ){ @@ -1270,12 +1306,21 @@ static int strlen30(const char *z){ /* ** Return the length of a string in characters. Multibyte UTF8 characters -** count as a single character. +** count as a single character for single-width characters, or as two +** characters for double-width characters. */ static int strlenChar(const char *z){ int n = 0; while( *z ){ - if( (0xc0&*(z++))!=0x80 ) n++; + if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = decodeUtf8((const u8*)z, &u); + z += len; + n += cli_wcwidth(u); + } } return n; } @@ -1318,7 +1363,7 @@ static FILE * openChrSource(const char *zFile){ ** This routine reads a line of text from FILE in, stores ** the text in memory obtained from malloc() and returns a pointer ** to the text. NULL is returned at end of file, or if malloc() -** fails. +** fails, or if the length of the line is longer than about a gigabyte. ** ** If zLine is not NULL then it is a malloced buffer returned from ** a previous call to this routine that may be reused. @@ -1329,6 +1374,10 @@ static char *local_getline(char *zLine, FILE *in){ while( 1 ){ if( n+100>nLine ){ + if( nLine>=1073741773 ){ + free(zLine); + return 0; + } nLine = nLine*2 + 100; zLine = realloc(zLine, nLine); shell_check_oom(zLine); @@ -1412,10 +1461,14 @@ static int hexDigitValue(char c){ /* ** Interpret zArg as an integer value, possibly with suffixes. +** +** If the value specified by zArg is outside the range of values that +** can be represented using a 64-bit twos-complement integer, then return +** the nearest representable value. */ static sqlite3_int64 integerValue(const char *zArg){ - sqlite3_int64 v = 0; - static const struct { char *zSuffix; int iMult; } aMult[] = { + sqlite3_uint64 v = 0; + static const struct { char *zSuffix; unsigned int iMult; } aMult[] = { { "KiB", 1024 }, { "MiB", 1024*1024 }, { "GiB", 1024*1024*1024 }, @@ -1438,22 +1491,30 @@ static sqlite3_int64 integerValue(const char *zArg){ int x; zArg += 2; while( (x = hexDigitValue(zArg[0]))>=0 ){ + if( v > 0x0fffffffffffffffULL ) goto integer_overflow; v = (v<<4) + x; zArg++; } }else{ while( IsDigit(zArg[0]) ){ - v = v*10 + zArg[0] - '0'; + if( v>=922337203685477580LL ){ + if( v>922337203685477580LL || zArg[0]>='8' ) goto integer_overflow; + } + v = v*10 + (zArg[0] - '0'); zArg++; } } for(i=0; i0x7fffffffffffffffULL ) goto integer_overflow; + return isNeg? -(sqlite3_int64)v : (sqlite3_int64)v; +integer_overflow: + return isNeg ? (i64)0x8000000000000000LL : 0x7fffffffffffffffLL; } /* @@ -1461,9 +1522,9 @@ static sqlite3_int64 integerValue(const char *zArg){ */ typedef struct ShellText ShellText; struct ShellText { - char *z; - int n; - int nAlloc; + char *zTxt; /* The text */ + i64 n; /* Number of bytes of zTxt[] actually used */ + i64 nAlloc; /* Number of bytes allocated for zTxt[] */ }; /* @@ -1473,7 +1534,7 @@ static void initText(ShellText *p){ memset(p, 0, sizeof(*p)); } static void freeText(ShellText *p){ - free(p->z); + sqlite3_free(p->zTxt); initText(p); } @@ -1498,26 +1559,26 @@ static void appendText(ShellText *p, const char *zAppend, char quote){ } } - if( p->z==0 || p->n+len>=p->nAlloc ){ + if( p->zTxt==0 || p->n+len>=p->nAlloc ){ p->nAlloc = p->nAlloc*2 + len + 20; - p->z = realloc(p->z, p->nAlloc); - shell_check_oom(p->z); + p->zTxt = sqlite3_realloc64(p->zTxt, p->nAlloc); + shell_check_oom(p->zTxt); } if( quote ){ - char *zCsr = p->z+p->n; + char *zCsr = p->zTxt+p->n; *zCsr++ = quote; for(i=0; in = (int)(zCsr - p->z); + p->n = (i64)(zCsr - p->zTxt); *zCsr = '\0'; }else{ - memcpy(p->z+p->n, zAppend, nAppend); + memcpy(p->zTxt+p->n, zAppend, nAppend); p->n += nAppend; - p->z[p->n] = '\0'; + p->zTxt[p->n] = '\0'; } } @@ -1542,6 +1603,9 @@ static char quoteChar(const char *zName){ /* ** Construct a fake object name and column list to describe the structure ** of the view, virtual table, or table valued function zSchema.zName. +** +** The returned string comes from sqlite3_mprintf() and should be freed +** by the caller using sqlite3_free(). */ static char *shellFakeSchema( sqlite3 *db, /* The database connection containing the vtab */ @@ -1582,9 +1646,9 @@ static char *shellFakeSchema( sqlite3_finalize(pStmt); if( nRow==0 ){ freeText(&s); - s.z = 0; + s.zTxt = 0; } - return s.z; + return s.zTxt; } /* @@ -1687,7 +1751,7 @@ static void shellAddSchemaName( }else{ z = sqlite3_mprintf("%z\n/* %s */", z, zFake); } - free(zFake); + sqlite3_free(zFake); } if( z ){ sqlite3_result_text(pCtx, z, -1, sqlite3_free); @@ -1708,10 +1772,9 @@ static void shellAddSchemaName( #define SQLITE_EXTENSION_INIT1 #define SQLITE_EXTENSION_INIT2(X) (void)(X) -#if defined(_WIN32) && defined(_MSC_VER) -/************************* Begin test_windirent.h ******************/ +/************************* Begin ../ext/misc/windirent.h ******************/ /* -** 2015 November 30 +** 2025-06-05 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -1721,322 +1784,160 @@ static void shellAddSchemaName( ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains declarations for most of the opendir() family of -** POSIX functions on Win32 using the MSVCRT. +** +** An implementation of opendir(), readdir(), and closedir() for Windows, +** based on the FindFirstFile(), FindNextFile(), and FindClose() APIs +** of Win32. +** +** #include this file inside any C-code module that needs to use +** opendir()/readdir()/closedir(). This file is a no-op on non-Windows +** machines. On Windows, static functions are defined that implement +** those standard interfaces. */ - #if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H) #define SQLITE_WINDIRENT_H -/* -** We need several data types from the Windows SDK header. -*/ - #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif - -#include "windows.h" - -/* -** We need several support functions from the SQLite core. -*/ - -/* #include "sqlite3.h" */ - -/* -** We need several things from the ANSI and MSVCRT headers. -*/ - +#include +#include #include #include #include -#include #include #include #include - -/* -** We may need several defines that should have been in "sys/stat.h". -*/ - +#include +#ifndef FILENAME_MAX +# define FILENAME_MAX (260) +#endif #ifndef S_ISREG -#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) #endif - #ifndef S_ISDIR -#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) #endif - #ifndef S_ISLNK -#define S_ISLNK(mode) (0) +#define S_ISLNK(m) (0) #endif +typedef unsigned short mode_t; -/* -** We may need to provide the "mode_t" type. +/* The dirent object for Windows is abbreviated. The only field really +** usable by applications is d_name[]. */ - -#ifndef MODE_T_DEFINED - #define MODE_T_DEFINED - typedef unsigned short mode_t; -#endif - -/* -** We may need to provide the "ino_t" type. -*/ - -#ifndef INO_T_DEFINED - #define INO_T_DEFINED - typedef unsigned short ino_t; -#endif - -/* -** We need to define "NAME_MAX" if it was not present in "limits.h". -*/ - -#ifndef NAME_MAX -# ifdef FILENAME_MAX -# define NAME_MAX (FILENAME_MAX) -# else -# define NAME_MAX (260) -# endif -# define DIRENT_NAME_MAX (NAME_MAX) -#endif - -/* -** We need to define "NULL_INTPTR_T" and "BAD_INTPTR_T". -*/ - -#ifndef NULL_INTPTR_T -# define NULL_INTPTR_T ((intptr_t)(0)) -#endif - -#ifndef BAD_INTPTR_T -# define BAD_INTPTR_T ((intptr_t)(-1)) -#endif - -/* -** We need to provide the necessary structures and related types. -*/ - -#ifndef DIRENT_DEFINED -#define DIRENT_DEFINED -typedef struct DIRENT DIRENT; -typedef DIRENT *LPDIRENT; -struct DIRENT { - ino_t d_ino; /* Sequence number, do not use. */ - unsigned d_attributes; /* Win32 file attributes. */ - char d_name[NAME_MAX + 1]; /* Name within the directory. */ +struct dirent { + int d_ino; /* Inode number (synthesized) */ + unsigned d_attributes; /* File attributes */ + char d_name[FILENAME_MAX]; /* Null-terminated filename */ }; -#endif -#ifndef DIR_DEFINED -#define DIR_DEFINED +/* The internals of DIR are opaque according to standards. So it +** does not matter what we put here. */ typedef struct DIR DIR; -typedef DIR *LPDIR; struct DIR { - intptr_t d_handle; /* Value returned by "_findfirst". */ - DIRENT d_first; /* DIRENT constructed based on "_findfirst". */ - DIRENT d_next; /* DIRENT constructed based on "_findnext". */ + intptr_t d_handle; /* Handle for findfirst()/findnext() */ + struct dirent cur; /* Current entry */ }; -#endif + +/* Ignore hidden and system files */ +#define WindowsFileToIgnore(a) \ + ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) /* -** Provide a macro, for use by the implementation, to determine if a -** particular directory entry should be skipped over when searching for -** the next directory entry that should be returned by the readdir(). +** Close a previously opened directory */ - -#ifndef is_filtered -# define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) -#endif - -/* -** Provide the function prototype for the POSIX compatible getenv() -** function. This function is not thread-safe. -*/ - -extern const char *windirent_getenv(const char *name); - -/* -** Finally, we can provide the function prototypes for the opendir(), -** readdir(), and closedir() POSIX functions. -*/ - -extern LPDIR opendir(const char *dirname); -extern LPDIRENT readdir(LPDIR dirp); -extern INT closedir(LPDIR dirp); - -#endif /* defined(WIN32) && defined(_MSC_VER) */ - -/************************* End test_windirent.h ********************/ -/************************* Begin test_windirent.c ******************/ -/* -** 2015 November 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains code to implement most of the opendir() family of -** POSIX functions on Win32 using the MSVCRT. -*/ - -#if defined(_WIN32) && defined(_MSC_VER) -/* #include "test_windirent.h" */ - -/* -** Implementation of the POSIX getenv() function using the Win32 API. -** This function is not thread-safe. -*/ -const char *windirent_getenv( - const char *name -){ - static char value[32768]; /* Maximum length, per MSDN */ - DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */ - DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */ - - memset(value, 0, sizeof(value)); - dwRet = GetEnvironmentVariableA(name, value, dwSize); - if( dwRet==0 || dwRet>dwSize ){ - /* - ** The function call to GetEnvironmentVariableA() failed -OR- - ** the buffer is not large enough. Either way, return NULL. - */ - return 0; - }else{ - /* - ** The function call to GetEnvironmentVariableA() succeeded - ** -AND- the buffer contains the entire value. - */ - return value; +static int closedir(DIR *pDir){ + int rc = 0; + if( pDir==0 ){ + return EINVAL; } + if( pDir->d_handle!=0 && pDir->d_handle!=(-1) ){ + rc = _findclose(pDir->d_handle); + } + sqlite3_free(pDir); + return rc; } /* -** Implementation of the POSIX opendir() function using the MSVCRT. +** Open a new directory. The directory name should be UTF-8 encoded. +** appropriate translations happen automatically. */ -LPDIR opendir( - const char *dirname /* Directory name, UTF8 encoding */ -){ - struct _wfinddata_t data; - LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR)); - SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]); +static DIR *opendir(const char *zDirName){ + DIR *pDir; wchar_t *b1; sqlite3_int64 sz; + struct _wfinddata_t data; - if( dirp==NULL ) return NULL; - memset(dirp, 0, sizeof(DIR)); - - /* TODO: Remove this if Unix-style root paths are not used. */ - if( sqlite3_stricmp(dirname, "/")==0 ){ - dirname = windirent_getenv("SystemDrive"); - } - + pDir = sqlite3_malloc64( sizeof(DIR) ); + if( pDir==0 ) return 0; + memset(pDir, 0, sizeof(DIR)); memset(&data, 0, sizeof(data)); - sz = strlen(dirname); + sz = strlen(zDirName); b1 = sqlite3_malloc64( (sz+3)*sizeof(b1[0]) ); if( b1==0 ){ - closedir(dirp); + closedir(pDir); return NULL; } - sz = MultiByteToWideChar(CP_UTF8, 0, dirname, sz, b1, sz); + sz = MultiByteToWideChar(CP_UTF8, 0, zDirName, sz, b1, sz); b1[sz++] = '\\'; b1[sz++] = '*'; b1[sz] = 0; - if( sz+1>(sqlite3_int64)namesize ){ - closedir(dirp); + if( sz+1>sizeof(data.name)/sizeof(data.name[0]) ){ + closedir(pDir); sqlite3_free(b1); return NULL; } memcpy(data.name, b1, (sz+1)*sizeof(b1[0])); sqlite3_free(b1); - dirp->d_handle = _wfindfirst(data.name, &data); - - if( dirp->d_handle==BAD_INTPTR_T ){ - closedir(dirp); + pDir->d_handle = _wfindfirst(data.name, &data); + if( pDir->d_handle<0 ){ + closedir(pDir); return NULL; } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ){ -next: - + while( WindowsFileToIgnore(data) ){ memset(&data, 0, sizeof(data)); - if( _wfindnext(dirp->d_handle, &data)==-1 ){ - closedir(dirp); + if( _wfindnext(pDir->d_handle, &data)==-1 ){ + closedir(pDir); return NULL; } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; } - - dirp->d_first.d_attributes = data.attrib; + pDir->cur.d_ino = 0; + pDir->cur.d_attributes = data.attrib; WideCharToMultiByte(CP_UTF8, 0, data.name, -1, - dirp->d_first.d_name, DIRENT_NAME_MAX, 0, 0); - return dirp; + pDir->cur.d_name, FILENAME_MAX, 0, 0); + return pDir; } /* -** Implementation of the POSIX readdir() function using the MSVCRT. +** Read the next entry from a directory. +** +** The returned struct-dirent object is managed by DIR. It is only +** valid until the next readdir() or closedir() call. Only the +** d_name[] field is meaningful. The d_name[] value has been +** translated into UTF8. */ -LPDIRENT readdir( - LPDIR dirp -){ +static struct dirent *readdir(DIR *pDir){ struct _wfinddata_t data; - - if( dirp==NULL ) return NULL; - - if( dirp->d_first.d_ino==0 ){ - dirp->d_first.d_ino++; - dirp->d_next.d_ino++; - - return &dirp->d_first; + if( pDir==0 ) return 0; + if( (pDir->cur.d_ino++)==0 ){ + return &pDir->cur; } - -next: - - memset(&data, 0, sizeof(data)); - if( _wfindnext(dirp->d_handle, &data)==-1 ) return NULL; - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; - - dirp->d_next.d_ino++; - dirp->d_next.d_attributes = data.attrib; + do{ + memset(&data, 0, sizeof(data)); + if( _wfindnext(pDir->d_handle, &data)==-1 ){ + return NULL; + } + }while( WindowsFileToIgnore(data) ); + pDir->cur.d_attributes = data.attrib; WideCharToMultiByte(CP_UTF8, 0, data.name, -1, - dirp->d_next.d_name, DIRENT_NAME_MAX, 0, 0); - return &dirp->d_next; + pDir->cur.d_name, FILENAME_MAX, 0, 0); + return &pDir->cur; } -/* -** Implementation of the POSIX closedir() function using the MSVCRT. -*/ -INT closedir( - LPDIR dirp -){ - INT result = 0; +#endif /* defined(_WIN32) && defined(_MSC_VER) */ - if( dirp==NULL ) return EINVAL; - - if( dirp->d_handle!=NULL_INTPTR_T && dirp->d_handle!=BAD_INTPTR_T ){ - result = _findclose(dirp->d_handle); - } - - sqlite3_free(dirp); - return result; -} - -#endif /* defined(WIN32) && defined(_MSC_VER) */ - -/************************* End test_windirent.c ********************/ -#define dirent DIRENT -#endif +/************************* End ../ext/misc/windirent.h ********************/ /************************* Begin ../ext/misc/memtrace.c ******************/ /* ** 2019-01-21 @@ -3825,7 +3726,8 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); if( p->a==0 ) goto new_from_text_failed; memset(p->a+p->nDigit, 0, iExp); p->nDigit += iExp; @@ -3844,7 +3746,8 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); if( p->a==0 ) goto new_from_text_failed; memmove(p->a+iExp, p->a, p->nDigit); memset(p->a, 0, iExp); @@ -3852,6 +3755,10 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ p->nFrac += iExp; } } + if( p->sign ){ + for(i=0; inDigit && p->a[i]==0; i++){} + if( i>=p->nDigit ) p->sign = 0; + } return p; new_from_text_failed: @@ -3944,7 +3851,7 @@ static void decimal_result(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_null(pCtx); return; } - z = sqlite3_malloc( p->nDigit+4 ); + z = sqlite3_malloc64( (sqlite3_int64)p->nDigit+4 ); if( z==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -4009,7 +3916,7 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ for(nZero=0; nZeroa[nZero]==0; nZero++){} nFrac = p->nFrac + (nDigit - p->nDigit); nDigit -= nZero; - z = sqlite3_malloc( nDigit+20 ); + z = sqlite3_malloc64( (sqlite3_int64)nDigit+20 ); if( z==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -4054,13 +3961,21 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ ** pB!=0 ** pB->isNull==0 */ -static int decimal_cmp(const Decimal *pA, const Decimal *pB){ +static int decimal_cmp(Decimal *pA, Decimal *pB){ int nASig, nBSig, rc, n; + while( pA->nFrac>0 && pA->a[pA->nDigit-1]==0 ){ + pA->nDigit--; + pA->nFrac--; + } + while( pB->nFrac>0 && pB->a[pB->nDigit-1]==0 ){ + pB->nDigit--; + pB->nFrac--; + } if( pA->sign!=pB->sign ){ return pA->sign ? -1 : +1; } if( pA->sign ){ - const Decimal *pTemp = pA; + Decimal *pTemp = pA; pA = pB; pB = pTemp; } @@ -4222,7 +4137,8 @@ static void decimalMul(Decimal *pA, Decimal *pB){ ){ goto mul_end; } - acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); + acc = sqlite3_malloc64( (sqlite3_int64)pA->nDigit + + (sqlite3_int64)pB->nDigit + 2 ); if( acc==0 ){ pA->oom = 1; goto mul_end; @@ -4309,7 +4225,7 @@ static Decimal *decimalFromDouble(double r){ isNeg = 0; } memcpy(&a,&r,sizeof(a)); - if( a==0 ){ + if( a==0 || a==(sqlite3_int64)0x8000000000000000LL){ e = 0; m = 0; }else{ @@ -4584,512 +4500,6 @@ int sqlite3_decimal_init( } /************************* End ../ext/misc/decimal.c ********************/ -/************************* Begin ../ext/misc/percentile.c ******************/ -/* -** 2013-05-28 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains code to implement the percentile(Y,P) SQL function -** and similar as described below: -** -** (1) The percentile(Y,P) function is an aggregate function taking -** exactly two arguments. -** -** (2) If the P argument to percentile(Y,P) is not the same for every -** row in the aggregate then an error is thrown. The word "same" -** in the previous sentence means that the value differ by less -** than 0.001. -** -** (3) If the P argument to percentile(Y,P) evaluates to anything other -** than a number in the range of 0.0 to 100.0 inclusive then an -** error is thrown. -** -** (4) If any Y argument to percentile(Y,P) evaluates to a value that -** is not NULL and is not numeric then an error is thrown. -** -** (5) If any Y argument to percentile(Y,P) evaluates to plus or minus -** infinity then an error is thrown. (SQLite always interprets NaN -** values as NULL.) -** -** (6) Both Y and P in percentile(Y,P) can be arbitrary expressions, -** including CASE WHEN expressions. -** -** (7) The percentile(Y,P) aggregate is able to handle inputs of at least -** one million (1,000,000) rows. -** -** (8) If there are no non-NULL values for Y, then percentile(Y,P) -** returns NULL. -** -** (9) If there is exactly one non-NULL value for Y, the percentile(Y,P) -** returns the one Y value. -** -** (10) If there N non-NULL values of Y where N is two or more and -** the Y values are ordered from least to greatest and a graph is -** drawn from 0 to N-1 such that the height of the graph at J is -** the J-th Y value and such that straight lines are drawn between -** adjacent Y values, then the percentile(Y,P) function returns -** the height of the graph at P*(N-1)/100. -** -** (11) The percentile(Y,P) function always returns either a floating -** point number or NULL. -** -** (12) The percentile(Y,P) is implemented as a single C99 source-code -** file that compiles into a shared-library or DLL that can be loaded -** into SQLite using the sqlite3_load_extension() interface. -** -** (13) A separate median(Y) function is the equivalent percentile(Y,50). -** -** (14) A separate percentile_cont(Y,P) function is equivalent to -** percentile(Y,P/100.0). In other words, the fraction value in -** the second argument is in the range of 0 to 1 instead of 0 to 100. -** -** (15) A separate percentile_disc(Y,P) function is like -** percentile_cont(Y,P) except that instead of returning the weighted -** average of the nearest two input values, it returns the next lower -** value. So the percentile_disc(Y,P) will always return a value -** that was one of the inputs. -** -** (16) All of median(), percentile(Y,P), percentile_cont(Y,P) and -** percentile_disc(Y,P) can be used as window functions. -** -** Differences from standard SQL: -** -** * The percentile_cont(X,P) function is equivalent to the following in -** standard SQL: -** -** (percentile_cont(P) WITHIN GROUP (ORDER BY X)) -** -** The SQLite syntax is much more compact. The standard SQL syntax -** is also supported if SQLite is compiled with the -** -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES option. -** -** * No median(X) function exists in the SQL standard. App developers -** are expected to write "percentile_cont(0.5)WITHIN GROUP(ORDER BY X)". -** -** * No percentile(Y,P) function exists in the SQL standard. Instead of -** percential(Y,P), developers must write this: -** "percentile_cont(P/100.0) WITHIN GROUP (ORDER BY Y)". Note that -** the fraction parameter to percentile() goes from 0 to 100 whereas -** the fraction parameter in SQL standard percentile_cont() goes from -** 0 to 1. -** -** Implementation notes as of 2024-08-31: -** -** * The regular aggregate-function versions of these routines work -** by accumulating all values in an array of doubles, then sorting -** that array using quicksort before computing the answer. Thus -** the runtime is O(NlogN) where N is the number of rows of input. -** -** * For the window-function versions of these routines, the array of -** inputs is sorted as soon as the first value is computed. Thereafter, -** the array is kept in sorted order using an insert-sort. This -** results in O(N*K) performance where K is the size of the window. -** One can imagine alternative implementations that give O(N*logN*logK) -** performance, but they require more complex logic and data structures. -** The developers have elected to keep the asymptotically slower -** algorithm for now, for simplicity, under the theory that window -** functions are seldom used and when they are, the window size K is -** often small. The developers might revisit that decision later, -** should the need arise. -*/ -#if defined(SQLITE3_H) - /* no-op */ -#elif defined(SQLITE_STATIC_PERCENTILE) -/* # include "sqlite3.h" */ -#else -/* # include "sqlite3ext.h" */ - SQLITE_EXTENSION_INIT1 -#endif -#include -#include -#include - -/* The following object is the group context for a single percentile() -** aggregate. Remember all input Y values until the very end. -** Those values are accumulated in the Percentile.a[] array. -*/ -typedef struct Percentile Percentile; -struct Percentile { - unsigned nAlloc; /* Number of slots allocated for a[] */ - unsigned nUsed; /* Number of slots actually used in a[] */ - char bSorted; /* True if a[] is already in sorted order */ - char bKeepSorted; /* True if advantageous to keep a[] sorted */ - char bPctValid; /* True if rPct is valid */ - double rPct; /* Fraction. 0.0 to 1.0 */ - double *a; /* Array of Y values */ -}; - -/* Details of each function in the percentile family */ -typedef struct PercentileFunc PercentileFunc; -struct PercentileFunc { - const char *zName; /* Function name */ - char nArg; /* Number of arguments */ - char mxFrac; /* Maximum value of the "fraction" input */ - char bDiscrete; /* True for percentile_disc() */ -}; -static const PercentileFunc aPercentFunc[] = { - { "median", 1, 1, 0 }, - { "percentile", 2, 100, 0 }, - { "percentile_cont", 2, 1, 0 }, - { "percentile_disc", 2, 1, 1 }, -}; - -/* -** Return TRUE if the input floating-point number is an infinity. -*/ -static int percentIsInfinity(double r){ - sqlite3_uint64 u; - assert( sizeof(u)==sizeof(r) ); - memcpy(&u, &r, sizeof(u)); - return ((u>>52)&0x7ff)==0x7ff; -} - -/* -** Return TRUE if two doubles differ by 0.001 or less. -*/ -static int percentSameValue(double a, double b){ - a -= b; - return a>=-0.001 && a<=0.001; -} - -/* -** Search p (which must have p->bSorted) looking for an entry with -** value y. Return the index of that entry. -** -** If bExact is true, return -1 if the entry is not found. -** -** If bExact is false, return the index at which a new entry with -** value y should be insert in order to keep the values in sorted -** order. The smallest return value in this case will be 0, and -** the largest return value will be p->nUsed. -*/ -static int percentBinarySearch(Percentile *p, double y, int bExact){ - int iFirst = 0; /* First element of search range */ - int iLast = p->nUsed - 1; /* Last element of search range */ - while( iLast>=iFirst ){ - int iMid = (iFirst+iLast)/2; - double x = p->a[iMid]; - if( xy ){ - iLast = iMid - 1; - }else{ - return iMid; - } - } - if( bExact ) return -1; - return iFirst; -} - -/* -** Generate an error for a percentile function. -** -** The error format string must have exactly one occurrence of "%%s()" -** (with two '%' characters). That substring will be replaced by the name -** of the function. -*/ -static void percentError(sqlite3_context *pCtx, const char *zFormat, ...){ - PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); - char *zMsg1; - char *zMsg2; - va_list ap; - - va_start(ap, zFormat); - zMsg1 = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - zMsg2 = zMsg1 ? sqlite3_mprintf(zMsg1, pFunc->zName) : 0; - sqlite3_result_error(pCtx, zMsg2, -1); - sqlite3_free(zMsg1); - sqlite3_free(zMsg2); -} - -/* -** The "step" function for percentile(Y,P) is called once for each -** input row. -*/ -static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ - Percentile *p; - double rPct; - int eType; - double y; - assert( argc==2 || argc==1 ); - - if( argc==1 ){ - /* Requirement 13: median(Y) is the same as percentile(Y,50). */ - rPct = 0.5; - }else{ - /* Requirement 3: P must be a number between 0 and 100 */ - PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); - eType = sqlite3_value_numeric_type(argv[1]); - rPct = sqlite3_value_double(argv[1])/(double)pFunc->mxFrac; - if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) - || rPct<0.0 || rPct>1.0 - ){ - percentError(pCtx, "the fraction argument to %%s()" - " is not between 0.0 and %.1f", - (double)pFunc->mxFrac); - return; - } - } - - /* Allocate the session context. */ - p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); - if( p==0 ) return; - - /* Remember the P value. Throw an error if the P value is different - ** from any prior row, per Requirement (2). */ - if( !p->bPctValid ){ - p->rPct = rPct; - p->bPctValid = 1; - }else if( !percentSameValue(p->rPct,rPct) ){ - percentError(pCtx, "the fraction argument to %%s()" - " is not the same for all input rows"); - return; - } - - /* Ignore rows for which Y is NULL */ - eType = sqlite3_value_type(argv[0]); - if( eType==SQLITE_NULL ) return; - - /* If not NULL, then Y must be numeric. Otherwise throw an error. - ** Requirement 4 */ - if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ - percentError(pCtx, "input to %%s() is not numeric"); - return; - } - - /* Throw an error if the Y value is infinity or NaN */ - y = sqlite3_value_double(argv[0]); - if( percentIsInfinity(y) ){ - percentError(pCtx, "Inf input to %%s()"); - return; - } - - /* Allocate and store the Y */ - if( p->nUsed>=p->nAlloc ){ - unsigned n = p->nAlloc*2 + 250; - double *a = sqlite3_realloc64(p->a, sizeof(double)*n); - if( a==0 ){ - sqlite3_free(p->a); - memset(p, 0, sizeof(*p)); - sqlite3_result_error_nomem(pCtx); - return; - } - p->nAlloc = n; - p->a = a; - } - if( p->nUsed==0 ){ - p->a[p->nUsed++] = y; - p->bSorted = 1; - }else if( !p->bSorted || y>=p->a[p->nUsed-1] ){ - p->a[p->nUsed++] = y; - }else if( p->bKeepSorted ){ - int i; - i = percentBinarySearch(p, y, 0); - if( i<(int)p->nUsed ){ - memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0])); - } - p->a[i] = y; - p->nUsed++; - }else{ - p->a[p->nUsed++] = y; - p->bSorted = 0; - } -} - -/* -** Interchange two doubles. -*/ -#define SWAP_DOUBLE(X,Y) {double ttt=(X);(X)=(Y);(Y)=ttt;} - -/* -** Sort an array of doubles. -** -** Algorithm: quicksort -** -** This is implemented separately rather than using the qsort() routine -** from the standard library because: -** -** (1) To avoid a dependency on qsort() -** (2) To avoid the function call to the comparison routine for each -** comparison. -*/ -static void percentSort(double *a, unsigned int n){ - int iLt; /* Entries before a[iLt] are less than rPivot */ - int iGt; /* Entries at or after a[iGt] are greater than rPivot */ - int i; /* Loop counter */ - double rPivot; /* The pivot value */ - - assert( n>=2 ); - if( a[0]>a[n-1] ){ - SWAP_DOUBLE(a[0],a[n-1]) - } - if( n==2 ) return; - iGt = n-1; - i = n/2; - if( a[0]>a[i] ){ - SWAP_DOUBLE(a[0],a[i]) - }else if( a[i]>a[iGt] ){ - SWAP_DOUBLE(a[i],a[iGt]) - } - if( n==3 ) return; - rPivot = a[i]; - iLt = i = 1; - do{ - if( a[i]iLt ) SWAP_DOUBLE(a[i],a[iLt]) - iLt++; - i++; - }else if( a[i]>rPivot ){ - do{ - iGt--; - }while( iGt>i && a[iGt]>rPivot ); - SWAP_DOUBLE(a[i],a[iGt]) - }else{ - i++; - } - }while( i=2 ) percentSort(a, iLt); - if( n-iGt>=2 ) percentSort(a+iGt, n-iGt); - -/* Uncomment for testing */ -#if 0 - for(i=0; ibSorted==0 ){ - assert( p->nUsed>1 ); - percentSort(p->a, p->nUsed); - p->bSorted = 1; - } - p->bKeepSorted = 1; - - /* Find and remove the row */ - i = percentBinarySearch(p, y, 1); - if( i>=0 ){ - p->nUsed--; - if( i<(int)p->nUsed ){ - memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0])); - } - } -} - -/* -** Compute the final output of percentile(). Clean up all allocated -** memory if and only if bIsFinal is true. -*/ -static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ - Percentile *p; - PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); - unsigned i1, i2; - double v1, v2; - double ix, vx; - p = (Percentile*)sqlite3_aggregate_context(pCtx, 0); - if( p==0 ) return; - if( p->a==0 ) return; - if( p->nUsed ){ - if( p->bSorted==0 ){ - assert( p->nUsed>1 ); - percentSort(p->a, p->nUsed); - p->bSorted = 1; - } - ix = p->rPct*(p->nUsed-1); - i1 = (unsigned)ix; - if( pFunc->bDiscrete ){ - vx = p->a[i1]; - }else{ - i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; - v1 = p->a[i1]; - v2 = p->a[i2]; - vx = v1 + (v2-v1)*(ix-i1); - } - sqlite3_result_double(pCtx, vx); - } - if( bIsFinal ){ - sqlite3_free(p->a); - memset(p, 0, sizeof(*p)); - }else{ - p->bKeepSorted = 1; - } -} -static void percentFinal(sqlite3_context *pCtx){ - percentCompute(pCtx, 1); -} -static void percentValue(sqlite3_context *pCtx){ - percentCompute(pCtx, 0); -} - -#if defined(_WIN32) && !defined(SQLITE3_H) && !defined(SQLITE_STATIC_PERCENTILE) - -#endif -int sqlite3_percentile_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - unsigned int i; -#ifdef SQLITE3EXT_H - SQLITE_EXTENSION_INIT2(pApi); -#else - (void)pApi; /* Unused parameter */ -#endif - (void)pzErrMsg; /* Unused parameter */ - for(i=0; i>52; m = a & ((((sqlite3_int64)1)<<52)-1); @@ -6210,19 +5625,20 @@ int sqlite3_ieee_init( ** SELECT * FROM generate_series(0,100,5); ** ** The query above returns integers from 0 through 100 counting by steps -** of 5. +** of 5. In other words, 0, 5, 10, 15, ..., 90, 95, 100. There are a total +** of 21 rows. ** ** SELECT * FROM generate_series(0,100); ** -** Integers from 0 through 100 with a step size of 1. +** Integers from 0 through 100 with a step size of 1. 101 rows. ** ** SELECT * FROM generate_series(20) LIMIT 10; ** -** Integers 20 through 29. +** Integers 20 through 29. 10 rows. ** ** SELECT * FROM generate_series(0,-100,-5); ** -** Integers 0 -5 -10 ... -100. +** Integers 0 -5 -10 ... -100. 21 rows. ** ** SELECT * FROM generate_series(0,-1); ** @@ -6298,140 +5714,89 @@ SQLITE_EXTENSION_INIT1 #include #ifndef SQLITE_OMIT_VIRTUALTABLE -/* -** Return that member of a generate_series(...) sequence whose 0-based -** index is ix. The 0th member is given by smBase. The sequence members -** progress per ix increment by smStep. -*/ -static sqlite3_int64 genSeqMember( - sqlite3_int64 smBase, - sqlite3_int64 smStep, - sqlite3_uint64 ix -){ - static const sqlite3_uint64 mxI64 = - ((sqlite3_uint64)0x7fffffff)<<32 | 0xffffffff; - if( ix>=mxI64 ){ - /* Get ix into signed i64 range. */ - ix -= mxI64; - /* With 2's complement ALU, this next can be 1 step, but is split into - * 2 for UBSAN's satisfaction (and hypothetical 1's complement ALUs.) */ - smBase += (mxI64/2) * smStep; - smBase += (mxI64 - mxI64/2) * smStep; - } - /* Under UBSAN (or on 1's complement machines), must do this last term - * in steps to avoid the dreaded (and harmless) signed multiply overflow. */ - if( ix>=2 ){ - sqlite3_int64 ix2 = (sqlite3_int64)ix/2; - smBase += ix2*smStep; - ix -= ix2; - } - return smBase + ((sqlite3_int64)ix)*smStep; -} - -/* typedef unsigned char u8; */ - -typedef struct SequenceSpec { - sqlite3_int64 iOBase; /* Original starting value ("start") */ - sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ - sqlite3_int64 iBase; /* Starting value to actually use */ - sqlite3_int64 iTerm; /* Terminal value to actually use */ - sqlite3_int64 iStep; /* Increment ("step") */ - sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */ - sqlite3_uint64 uSeqIndexNow; /* Current index during generation */ - sqlite3_int64 iValueNow; /* Current value during generation */ - u8 isNotEOF; /* Sequence generation not exhausted */ - u8 isReversing; /* Sequence is being reverse generated */ -} SequenceSpec; - -/* -** Prepare a SequenceSpec for use in generating an integer series -** given initialized iBase, iTerm and iStep values. Sequence is -** initialized per given isReversing. Other members are computed. -*/ -static void setupSequence( SequenceSpec *pss ){ - int bSameSigns; - pss->uSeqIndexMax = 0; - pss->isNotEOF = 0; - bSameSigns = (pss->iBase < 0)==(pss->iTerm < 0); - if( pss->iTerm < pss->iBase ){ - sqlite3_uint64 nuspan = 0; - if( bSameSigns ){ - nuspan = (sqlite3_uint64)(pss->iBase - pss->iTerm); - }else{ - /* Under UBSAN (or on 1's complement machines), must do this in steps. - * In this clause, iBase>=0 and iTerm<0 . */ - nuspan = 1; - nuspan += pss->iBase; - nuspan += -(pss->iTerm+1); - } - if( pss->iStep<0 ){ - pss->isNotEOF = 1; - if( nuspan==ULONG_MAX ){ - pss->uSeqIndexMax = ( pss->iStep>LLONG_MIN )? nuspan/-pss->iStep : 1; - }else if( pss->iStep>LLONG_MIN ){ - pss->uSeqIndexMax = nuspan/-pss->iStep; - } - } - }else if( pss->iTerm > pss->iBase ){ - sqlite3_uint64 puspan = 0; - if( bSameSigns ){ - puspan = (sqlite3_uint64)(pss->iTerm - pss->iBase); - }else{ - /* Under UBSAN (or on 1's complement machines), must do this in steps. - * In this clause, iTerm>=0 and iBase<0 . */ - puspan = 1; - puspan += pss->iTerm; - puspan += -(pss->iBase+1); - } - if( pss->iStep>0 ){ - pss->isNotEOF = 1; - pss->uSeqIndexMax = puspan/pss->iStep; - } - }else if( pss->iTerm == pss->iBase ){ - pss->isNotEOF = 1; - pss->uSeqIndexMax = 0; - } - pss->uSeqIndexNow = (pss->isReversing)? pss->uSeqIndexMax : 0; - pss->iValueNow = (pss->isReversing) - ? genSeqMember(pss->iBase, pss->iStep, pss->uSeqIndexMax) - : pss->iBase; -} - -/* -** Progress sequence generator to yield next value, if any. -** Leave its state to either yield next value or be at EOF. -** Return whether there is a next value, or 0 at EOF. -*/ -static int progressSequence( SequenceSpec *pss ){ - if( !pss->isNotEOF ) return 0; - if( pss->isReversing ){ - if( pss->uSeqIndexNow > 0 ){ - pss->uSeqIndexNow--; - pss->iValueNow -= pss->iStep; - }else{ - pss->isNotEOF = 0; - } - }else{ - if( pss->uSeqIndexNow < pss->uSeqIndexMax ){ - pss->uSeqIndexNow++; - pss->iValueNow += pss->iStep; - }else{ - pss->isNotEOF = 0; - } - } - return pss->isNotEOF; -} /* series_cursor is a subclass of sqlite3_vtab_cursor which will ** serve as the underlying representation of a cursor that scans -** over rows of the result +** over rows of the result. +** +** iOBase, iOTerm, and iOStep are the original values of the +** start=, stop=, and step= constraints on the query. These are +** the values reported by the start, stop, and step columns of the +** virtual table. +** +** iBase, iTerm, iStep, and bDescp are the actual values used to generate +** the sequence. These might be different from the iOxxxx values. +** For example in +** +** SELECT value FROM generate_series(1,11,2) +** WHERE value BETWEEN 4 AND 8; +** +** The iOBase is 1, but the iBase is 5. iOTerm is 11 but iTerm is 7. +** Another example: +** +** SELECT value FROM generate_series(1,15,3) ORDER BY value DESC; +** +** The cursor initialization for the above query is: +** +** iOBase = 1 iBase = 13 +** iOTerm = 15 iTerm = 1 +** iOStep = 3 iStep = 3 bDesc = 1 +** +** The actual step size is unsigned so that can have a value of +** +9223372036854775808 which is needed for querys like this: +** +** SELECT value +** FROM generate_series(9223372036854775807, +** -9223372036854775808, +** -9223372036854775808) +** ORDER BY value ASC; +** +** The setup for the previous query will be: +** +** iOBase = 9223372036854775807 iBase = -1 +** iOTerm = -9223372036854775808 iTerm = 9223372036854775807 +** iOStep = -9223372036854775808 iStep = 9223372036854775808 bDesc = 0 */ +/* typedef unsigned char u8; */ typedef struct series_cursor series_cursor; struct series_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - SequenceSpec ss; /* (this) Derived class data */ + sqlite3_int64 iOBase; /* Original starting value ("start") */ + sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ + sqlite3_int64 iOStep; /* Original step value */ + sqlite3_int64 iBase; /* Starting value to actually use */ + sqlite3_int64 iTerm; /* Terminal value to actually use */ + sqlite3_uint64 iStep; /* The step size */ + sqlite3_int64 iValue; /* Current value */ + u8 bDesc; /* iStep is really negative */ + u8 bDone; /* True if stepped past last element */ }; +/* +** Computed the difference between two 64-bit signed integers using a +** convoluted computation designed to work around the silly restriction +** against signed integer overflow in C. +*/ +static sqlite3_uint64 span64(sqlite3_int64 a, sqlite3_int64 b){ + assert( a>=b ); + return (*(sqlite3_uint64*)&a) - (*(sqlite3_uint64*)&b); +} + +/* +** Add or substract an unsigned 64-bit integer from a signed 64-bit integer +** and return the new signed 64-bit integer. +*/ +static sqlite3_int64 add64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x += b; + return *(sqlite3_int64*)&x; +} +static sqlite3_int64 sub64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x -= b; + return *(sqlite3_int64*)&x; +} + /* ** The seriesConnect() method is invoked to create a new ** series_vtab that describes the generate_series virtual table. @@ -6512,7 +5877,15 @@ static int seriesClose(sqlite3_vtab_cursor *cur){ */ static int seriesNext(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - progressSequence( & pCur->ss ); + if( pCur->iValue==pCur->iTerm ){ + pCur->bDone = 1; + }else if( pCur->bDesc ){ + pCur->iValue = sub64(pCur->iValue, pCur->iStep); + assert( pCur->iValue>=pCur->iTerm ); + }else{ + pCur->iValue = add64(pCur->iValue, pCur->iStep); + assert( pCur->iValue<=pCur->iTerm ); + } return SQLITE_OK; } @@ -6528,19 +5901,19 @@ static int seriesColumn( series_cursor *pCur = (series_cursor*)cur; sqlite3_int64 x = 0; switch( i ){ - case SERIES_COLUMN_START: x = pCur->ss.iOBase; break; - case SERIES_COLUMN_STOP: x = pCur->ss.iOTerm; break; - case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; - default: x = pCur->ss.iValueNow; break; + case SERIES_COLUMN_START: x = pCur->iOBase; break; + case SERIES_COLUMN_STOP: x = pCur->iOTerm; break; + case SERIES_COLUMN_STEP: x = pCur->iOStep; break; + default: x = pCur->iValue; break; } sqlite3_result_int64(ctx, x); return SQLITE_OK; } #ifndef LARGEST_UINT64 -#define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -#define LARGEST_UINT64 (0xffffffff|(((sqlite3_uint64)0xffffffff)<<32)) -#define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#define LARGEST_INT64 ((sqlite3_int64)0x7fffffffffffffffLL) +#define LARGEST_UINT64 ((sqlite3_uint64)0xffffffffffffffffULL) +#define SMALLEST_INT64 ((sqlite3_int64)0x8000000000000000LL) #endif /* @@ -6548,7 +5921,7 @@ static int seriesColumn( */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; - *pRowid = pCur->ss.iValueNow; + *pRowid = pCur->iValue; return SQLITE_OK; } @@ -6558,7 +5931,7 @@ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int seriesEof(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - return !pCur->ss.isNotEOF; + return pCur->bDone; } /* True to cause run-time checking of the start=, stop=, and/or step= @@ -6569,6 +5942,59 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ # define SQLITE_SERIES_CONSTRAINT_VERIFY 0 #endif +/* +** Return the number of steps between pCur->iBase and pCur->iTerm if +** the step width is pCur->iStep. +*/ +static sqlite3_uint64 seriesSteps(series_cursor *pCur){ + if( pCur->bDesc ){ + assert( pCur->iBase >= pCur->iTerm ); + return span64(pCur->iBase, pCur->iTerm)/pCur->iStep; + }else{ + assert( pCur->iBase <= pCur->iTerm ); + return span64(pCur->iTerm, pCur->iBase)/pCur->iStep; + } +} + +#if defined(SQLITE_ENABLE_MATH_FUNCTIONS) || defined(_WIN32) +/* +** Case 1 (the most common case): +** The standard math library is available so use ceil() and floor() from there. +*/ +static double seriesCeil(double r){ return ceil(r); } +static double seriesFloor(double r){ return floor(r); } +#elif defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +/* +** Case 2 (2nd most common): Use GCC/Clang builtins +*/ +static double seriesCeil(double r){ return __builtin_ceil(r); } +static double seriesFloor(double r){ return __builtin_floor(r); } +#else +/* +** Case 3 (rarely happens): Use home-grown ceil() and floor() routines. +*/ +static double seriesCeil(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r>(double)x ) x++; + return (double)x; +} +static double seriesFloor(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r<(double)x ) x--; + return (double)x; +} +#endif + /* ** This method is called to "rewind" the series_cursor object back ** to the first row of output. This method is always called at least @@ -6602,33 +6028,42 @@ static int seriesFilter( int argc, sqlite3_value **argv ){ series_cursor *pCur = (series_cursor *)pVtabCursor; - int i = 0; - int returnNoRows = 0; - sqlite3_int64 iMin = SMALLEST_INT64; - sqlite3_int64 iMax = LARGEST_INT64; - sqlite3_int64 iLimit = 0; - sqlite3_int64 iOffset = 0; + int iArg = 0; /* Arguments used so far */ + int i; /* Loop counter */ + sqlite3_int64 iMin = SMALLEST_INT64; /* Smallest allowed output value */ + sqlite3_int64 iMax = LARGEST_INT64; /* Largest allowed output value */ + sqlite3_int64 iLimit = 0; /* if >0, the value of the LIMIT */ + sqlite3_int64 iOffset = 0; /* if >0, the value of the OFFSET */ (void)idxStrUnused; + + /* If any constraints have a NULL value, then return no rows. + ** See ticket https://sqlite.org/src/info/fac496b61722daf2 + */ + for(i=0; i