diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ddb812b0..6ba1b2ee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,7 +32,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0 - name: "Calculate required variables" id: variables @@ -100,11 +100,11 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0 # QEMU should come before Buildx - name: Set up QEMU - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 #v3.6.0 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 #v3.7.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 #v3.11.1 @@ -265,7 +265,7 @@ jobs: - name: Attach binaries to release if: github.event_name == 'release' - uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 #v2.4.1 + uses: softprops/action-gh-release@5be0e66d93ac7ed76da52eca8bb058f665c3a5fe #v2.4.2 with: tag_name: ${{ github.event.release.tag_name }} files: | diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 5f4fb439..3233dea7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -58,7 +58,7 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0 - name: Install dependencies run: | @@ -85,7 +85,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee #v4.31.2 + uses: github/codeql-action/init@e12f0178983d466f2f6028f5cc7a6d786fd97f4b #v4.31.4 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -108,7 +108,7 @@ jobs: ./build.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee #v4.31.2 + uses: github/codeql-action/analyze@e12f0178983d466f2f6028f5cc7a6d786fd97f4b #v4.31.4 with: category: "/language:${{matrix.language}}" upload: failure-only # upload only in case of failure, otherwise upload later after filtering @@ -134,7 +134,7 @@ jobs: output: codeql-results/cpp.sarif - name: Upload SARIF - uses: github/codeql-action/upload-sarif@0499de31b99561a6d14a36a5f662c2a54f91beee #v4.31.2 + uses: github/codeql-action/upload-sarif@e12f0178983d466f2f6028f5cc7a6d786fd97f4b #v4.31.4 with: sarif_file: codeql-results/cpp.sarif diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 3e9c4a51..f2fd2341 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -13,10 +13,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0 - name: Spell-Checking - uses: codespell-project/actions-codespell@406322ec52dd7b488e48c1c4b82e2a8b3a1bf630 #v2.1 + uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 #v2.2 with: ignore_words_file: .github/.codespellignore skip: ./src/database/sqlite3.c,./src/database/sqlite3.h,./src/database/shell.c,./src/lua,./src/dnsmasq,./src/tre-regex,./.git,./test/libs,./src/webserver/civetweb,./src/zip/miniz,./src/api/docs/content/external,./src/database/sqlite3_rsync.c,./package-lock.json,./src/config/tomlc17 diff --git a/.github/workflows/openapi-validator.yml b/.github/workflows/openapi-validator.yml index 1b4246d9..b2181251 100644 --- a/.github/workflows/openapi-validator.yml +++ b/.github/workflows/openapi-validator.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Clone repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0 - name: Set up Node.js uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 #v6.0.0 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index dce7d190..4c61ef81 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 67f993ae..bacf3ef4 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch index f9ea8145..3a2710e8 100644 --- a/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch +++ b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch @@ -20,10 +20,10 @@ index 6280ebf6..a5e82f70 100644 #else int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char **argv; -@@ -33467,6 +33469,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ +@@ -33656,6 +33658,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ + if( stdin_is_interactive ){ char *zHome; char *zHistory; - int nHistory; + print_FTL_version(); sqlite3_fprintf(stdout, "SQLite version %s %.19s\n" /*extra-version-info*/ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cc265087..33d1262b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -40,7 +40,9 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk. # SQLITE_DEFAULT_WORKER_THREADS=0: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. Do not increase this value as it, ironically, can cause performance degradation and definitely increases total memory usage. # SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL. -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=1 -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_H -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=0 -DSQLITE_MAX_PREPARE_RETRY=200") +# SQLITE_ENABLE_CARRAY: Enable the carray virtual table module +# SQLITE_ENABLE_PERCENTILE: Enable the percentile aggregate function +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=1 -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_H -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=0 -DSQLITE_MAX_PREPARE_RETRY=200 -DSQLITE_ENABLE_CARRAY=1 -DSQLITE_ENABLE_PERCENTILE=1") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed @@ -216,7 +218,7 @@ endif() set(CMAKE_C_FLAGS "-std=c17 -pipe ${WARN_FLAGS} -D_FILE_OFFSET_BITS=64 ${HARDENING_FLAGS} ${DEBUG_FLAGS} ${CMAKE_C_FLAGS} -DHAVE_POLL_H ${SQLITE_DEFINES}") set(CMAKE_C_FLAGS_DEBUG "-O0 -g3") -set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") +set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG -funroll-loops") set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELEASE} -g3") set(CMAKE_C_FLAGS_MINSIZEREL "-Os -DNDEBUG") diff --git a/src/api/2fa.c b/src/api/2fa.c index b23165f6..e0710e77 100644 --- a/src/api/2fa.c +++ b/src/api/2fa.c @@ -195,17 +195,21 @@ static bool encode_uint8_t_array_to_base32(const uint8_t *in, const size_t in_le return true; } +static time_t last_attempt = 0; static uint32_t last_code = 0; enum totp_status verifyTOTP(const uint32_t incode) { + // Only one attempt per second is allowed + const time_t now = time(NULL); + if(now == last_attempt) + return TOTP_RATE_LIMIT; + last_attempt = now; + // Decode base32 secret uint8_t decoded_secret[RFC6238_SECRET_LEN]; if(!decode_base32_to_uint8_array(config.webserver.api.totp_secret.v.s, decoded_secret, sizeof(decoded_secret))) return false; - // Get current time - const time_t now = time(NULL); - // Verify code for the previous, the current and the next time step for(int i = -1; i <= 1; i++) { diff --git a/src/api/action.c b/src/api/action.c index 70436992..51ea9f29 100644 --- a/src/api/action.c +++ b/src/api/action.c @@ -121,7 +121,15 @@ static int run_and_stream_command(struct ftl_conn *api, const char *path, const int api_action_gravity(struct ftl_conn *api) { - return run_and_stream_command(api, "/usr/local/bin/pihole", (const char *const []){ "pihole", "-g", NULL }, "FORCE_COLOR"); + // Only set FORCE_COLOR if the client explicitly requests it via "color=true" query parameter + // This prevents ANSI escape codes from being included in the output for API consumers that don't need them + bool color = false; + const char *query = api->request != NULL ? api->request->query_string : ""; + if(query != NULL) + get_bool_var(query, "color", &color); + + const char *extra_env = color ? "FORCE_COLOR" : NULL; + return run_and_stream_command(api, "/usr/local/bin/pihole", (const char *const []){ "pihole", "-g", NULL }, extra_env); } int api_action_restartDNS(struct ftl_conn *api) diff --git a/src/api/api.h b/src/api/api.h index 8fb12658..24e689d9 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -114,6 +114,7 @@ enum totp_status { TOTP_INVALID, TOTP_CORRECT, TOTP_REUSED, + TOTP_RATE_LIMIT } __attribute__ ((packed)); enum totp_status verifyTOTP(const uint32_t code); int generateTOTP(struct ftl_conn *api); diff --git a/src/api/auth.c b/src/api/auth.c index 468fd12e..d3be6f58 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -559,6 +559,14 @@ int api_auth(struct ftl_conn *api) "Reused 2FA token", "wait for new token"); } + else if(totp == TOTP_RATE_LIMIT) + { + // 2FA validation requested too often + return send_json_error(api, 429, + "rate_limiting", + "Rate-limiting 2FA token requests, try again later", + NULL); + } else if(totp != TOTP_CORRECT) { // 2FA token is invalid diff --git a/src/api/config.c b/src/api/config.c index 3451c9e3..53068b65 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -22,7 +22,7 @@ #include "config/toml_writer.h" // write_dnsmasq_config() #include "config/dnsmasq_config.h" -// shm_lock() +// lock_shm() #include "shmem.h" // hash_password() #include "config/password.h" @@ -500,8 +500,8 @@ int get_json_config(struct ftl_conn *api, cJSON *json, const bool detailed) continue; // Check equality of paths up to the requested level (if any) // Examples: - // requested was /config/dnsmasq -> skip all entries that do not start in dnsmasq. - // requested was /config/dnsmasq/dhcp -> skip all entries that do not start in dhcp + // requested was /config/dns -> skip all entries that do not start in dns + // requested was /config/dns/dhcp -> skip all entries that do not start in dhcp // etc. if(!check_paths_equal(conf_item->p, requested_path, min_level - 1)) continue; diff --git a/src/api/dhcp.c b/src/api/dhcp.c index ff337061..318dd5b2 100644 --- a/src/api/dhcp.c +++ b/src/api/dhcp.c @@ -45,11 +45,11 @@ int api_dhcp_leases_GET(struct ftl_conn *api) // Parse line unsigned long expires = 0; - char hwaddr[18] = { 0 }; + char hwaddr[48] = { 0 }; char ip[INET6_ADDRSTRLEN] = { 0 }; char name[65] = { 0 }; char clientid[765] = { 0 }; - const int ret = sscanf(line, "%lu %17s %45s %64s %764s", &expires, hwaddr, ip, name, clientid); + const int ret = sscanf(line, "%lu %47s %45s %64s %764s", &expires, hwaddr, ip, name, clientid); // Skip invalid lines if(ret != 5) diff --git a/src/api/docs/content/specs/action.yaml b/src/api/docs/content/specs/action.yaml index 893a42cf..b5d8edeb 100644 --- a/src/api/docs/content/specs/action.yaml +++ b/src/api/docs/content/specs/action.yaml @@ -4,11 +4,13 @@ components: gravity: post: summary: Run gravity + parameters: + - $ref: 'action.yaml#/components/parameters/color' tags: - Actions operationId: "action_gravity" description: | - Update Pi-hole's adlists by running `pihole -g`. The output of the process is streamed with chunked encoding. + Update Pi-hole's adlists by running `pihole -g`. The output of the process is streamed with chunked encoding. Use the optional `color` query parameter to include ANSI color escape codes in the output. responses: '200': description: OK @@ -183,6 +185,17 @@ components: - $ref: 'action.yaml#/components/errors/forbidden' - $ref: 'common.yaml#/components/schemas/took' + parameters: + color: + name: color + in: query + description: Include ANSI color escape codes in the streamed output. Defaults to false to prevent colored output for API consumers that don't need formatting. + schema: + type: boolean + default: false + example: true + required: false + errors: forbidden: description: | diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index b885127d..3a45b621 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -117,6 +117,8 @@ components: $ref: 'auth.yaml#/components/examples/errors/rate_limit' no_seats: $ref: 'auth.yaml#/components/examples/errors/no_seats' + totp_rate_limit: + $ref: 'auth.yaml#/components/examples/errors/totp_rate_limit' delete: summary: Delete session tags: @@ -588,6 +590,14 @@ components: message: "Reused 2FA token" hint: "wait for new token" took: 0.003 + totp_rate_limit: + summary: 2FA token requested too soon + value: + error: + key: "rate_limiting" + message: "Rate-limiting 2FA token requests, try again later" + hint: null + took: 0.001 rate_limit: summary: Rate limit exceeded value: diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 9e4c9be9..f2fc8a7c 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -625,6 +625,8 @@ components: type: boolean netlink: type: boolean + timing: + type: boolean all: type: boolean topics: @@ -874,6 +876,7 @@ components: reserved: false ntp: false netlink: false + timing: false all: false took: 0.003 config_one: @@ -919,7 +922,7 @@ components: error: key: "bad_request" message: "Invalid path depth" - hint: "Use, e.g., DELETE /config/dnsmasq/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams" + hint: "Use, e.g., DELETE /config/dns/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams" took: 0.003 item_already_present: summary: Item to be added exists already @@ -946,7 +949,7 @@ components: type: string required: true description: config element - example: "dnsmasq/upstreams" + example: "dns/upstreams" value: in: path name: value diff --git a/src/api/docs/content/specs/info.yaml b/src/api/docs/content/specs/info.yaml index 2092656b..406bb2f1 100644 --- a/src/api/docs/content/specs/info.yaml +++ b/src/api/docs/content/specs/info.yaml @@ -994,8 +994,20 @@ components: example: "pihole" queries: type: integer - description: Number of queries in long-term database + description: Number of queries in in-memory database example: 536956 + earliest_timestamp: + type: number + description: Unix timestamp of the earliest query in the in-memory database (Defaults to 0.0 if no queries are stored in memory) + example: 1611742120.5 + queries_disk: + type: integer + description: Number of queries in on-disk database + example: 1234567 + earliest_timestamp_disk: + type: number + description: Unix timestamp of the earliest query in the on-disk database (Defaults to 0.0 if no queries are stored on disk) + example: 1608450520.3 sqlite_version: type: string description: Version of embedded SQLite3 engine diff --git a/src/api/docs/content/specs/search.yaml b/src/api/docs/content/specs/search.yaml index 94a068be..06c2536f 100644 --- a/src/api/docs/content/specs/search.yaml +++ b/src/api/docs/content/specs/search.yaml @@ -162,7 +162,7 @@ components: properties: partial: type: boolean - description: Whether partial matching was requested + description: Whether partial matching was requested. Note that partial matching may not find complex regex entries. example: false N: type: integer diff --git a/src/api/info.c b/src/api/info.c index 0a94f1ac..d2717900 100644 --- a/src/api/info.c +++ b/src/api/info.c @@ -146,9 +146,17 @@ int api_info_database(struct ftl_conn *api) JSON_ADD_ITEM_TO_OBJECT(owner, "group", group); JSON_ADD_ITEM_TO_OBJECT(json, "owner", owner); - // Add number of queries in on-disk database - const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage"); + // Add number of queries and earliest timestamp in in-memory database + double earliest_timestamp_mem = 0.0; + const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage", &earliest_timestamp_mem); JSON_ADD_NUMBER_TO_OBJECT(json, "queries", queries_in_database); + JSON_ADD_NUMBER_TO_OBJECT(json, "earliest_timestamp", earliest_timestamp_mem); + + // Add number of queries and earliest timestamp in on-disk database + double earliest_timestamp_disk = 0.0; + const int queries_in_disk_database = get_number_of_queries_in_DB(NULL, "disk.query_storage", &earliest_timestamp_disk); + JSON_ADD_NUMBER_TO_OBJECT(json, "queries_disk", queries_in_disk_database); + JSON_ADD_NUMBER_TO_OBJECT(json, "earliest_timestamp_disk", earliest_timestamp_disk); // Add SQLite library version JSON_REF_STR_IN_OBJECT(json, "sqlite_version", get_sqlite3_version()); diff --git a/src/api/search.c b/src/api/search.c index 3b400048..d4af1df0 100644 --- a/src/api/search.c +++ b/src/api/search.c @@ -29,6 +29,7 @@ static int search_table(struct ftl_conn *api, const char *item, if(ids != NULL) { // Set item to NULL to indicate that we are searching for IDs + // instead of domains item = NULL; // Strip "[" and "]" from ids ids[strlen(ids)-1] = '\0'; @@ -208,6 +209,19 @@ int api_search(struct ftl_conn *api) return ret; } + // Match partially against regex filters using LIKE '%term%' matching + // (works only for simple regexes) + unsigned int Nregex = 0u; + if(partial) + { + ret = search_table(api, punycode, GRAVITY_DOMAINLIST_ALL_REGEX, NULL, limit, &Nregex, partial, domains); + if(ret != 200) + { + free(punycode); + return ret; + } + } + // Search through gravity cJSON *gravity = JSON_NEW_ARRAY(); cJSON *gravity_patterns = NULL; @@ -236,11 +250,10 @@ int api_search(struct ftl_conn *api) cJSON *allow_ids = cJSON_GetObjectItem(regex_ids, "allow"); // Get allow regex filters - unsigned int Nregex = 0u; if(cJSON_GetArraySize(allow_ids) > 0) { char *allow_list = cJSON_PrintUnformatted(allow_ids); - ret = search_table(api,punycode, GRAVITY_DOMAINLIST_ALLOW_REGEX, allow_list, limit, &Nregex, false, domains); + ret = search_table(api, NULL, GRAVITY_DOMAINLIST_ALLOW_REGEX, allow_list, limit, &Nregex, false, domains); free(allow_list); if(ret != 200) { @@ -252,7 +265,7 @@ int api_search(struct ftl_conn *api) if(cJSON_GetArraySize(deny_ids) > 0) { char *deny_list = cJSON_PrintUnformatted(deny_ids); - ret = search_table(api, punycode, GRAVITY_DOMAINLIST_DENY_REGEX, deny_list, limit, &Nregex, false, domains); + ret = search_table(api, NULL, GRAVITY_DOMAINLIST_DENY_REGEX, deny_list, limit, &Nregex, false, domains); free(deny_list); if(ret != 200) { diff --git a/src/args.c b/src/args.c index faf48778..cf6296e1 100644 --- a/src/args.c +++ b/src/args.c @@ -412,6 +412,7 @@ void parse_args(int argc, char *argv[]) // Generate X.509 certificate if(argc > 1 && strcmp(argv[1], "--gen-x509") == 0) { +#ifdef HAVE_MBEDTLS if(argc < 3 || argc > 5) { printf("Usage: %s --gen-x509 [] [rsa]\n", argv[0]); @@ -431,6 +432,10 @@ void parse_args(int argc, char *argv[]) const bool rsa = argc > 4 && strcasecmp(argv[4], "rsa") == 0; exit(generate_certificate(argv[2], rsa, domain, config.webserver.tls.validity.v.ui) ? EXIT_SUCCESS : EXIT_FAILURE); +#else + printf("Error: FTL was compiled without TLS support. Certificate generation is not available.\n"); + exit(EXIT_FAILURE); +#endif } // Parse X.509 certificate @@ -438,6 +443,7 @@ void parse_args(int argc, char *argv[]) (strcmp(argv[1], "--read-x509") == 0 || strcmp(argv[1], "--read-x509-key") == 0)) { +#ifdef HAVE_MBEDTLS if(argc > 4) { printf("Usage: %s %s [] []\n", argv[0], argv[1]); @@ -480,6 +486,10 @@ void parse_args(int argc, char *argv[]) printf("Certificate does not match domain %s\n", argv[3]); exit(EXIT_FAILURE); } +#else + printf("Error: FTL was compiled without TLS support. Certificate reading is not available.\n"); + exit(EXIT_FAILURE); +#endif } // If the first argument is "gravity" (e.g., /usr/bin/pihole-FTL gravity), diff --git a/src/config/config.c b/src/config/config.c index bb012c15..c3e3af2c 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1625,6 +1625,12 @@ void initConfig(struct config *conf) conf->debug.netlink.d.b = false; conf->debug.netlink.c = validate_stub; // Only type-based checking + conf->debug.timing.k = "debug.timing"; + conf->debug.timing.h = "Print timing information from various parts of FTL"; + conf->debug.timing.t = CONF_BOOL; + conf->debug.timing.d.b = false; + conf->debug.timing.c = validate_stub; // Only type-based checking + conf->debug.all.k = "debug.all"; conf->debug.all.h = "Set all debug flags at once. This is a convenience option to enable all debug flags at once. Note that this option is not persistent, setting it to true will enable all *remaining* debug flags but unsetting it will disable *all* debug flags."; conf->debug.all.t = CONF_ALL_DEBUG_BOOL; diff --git a/src/config/config.h b/src/config/config.h index 0ad68605..877c0a0d 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -358,6 +358,7 @@ struct config { struct conf_item reserved; struct conf_item ntp; struct conf_item netlink; + struct conf_item timing; // all must be the last item in this struct struct conf_item all; } debug; diff --git a/src/database/CMakeLists.txt b/src/database/CMakeLists.txt index e438165c..22dd578f 100644 --- a/src/database/CMakeLists.txt +++ b/src/database/CMakeLists.txt @@ -22,8 +22,6 @@ set(sqlite3_sources add_library(sqlite3 OBJECT ${sqlite3_sources}) # Define the function that will be called to initialize the sqlite3 shell target_compile_definitions(sqlite3 PRIVATE "-DSQLITE_SHELL_INIT_PROC=pihole_sqlite3_initalize") -# Define that we want to statically link the percentile extension into the sqlite3 library -target_compile_definitions(sqlite3 PRIVATE "-DSQLITE_STATIC_PERCENTILE=1") target_compile_options(sqlite3 PRIVATE -Wno-implicit-fallthrough -Wno-cast-function-type -Wno-sign-compare -Wno-implicit-function-declaration -Wno-int-conversion) if (CMAKE_C_COMPILER_ID STREQUAL "Clang") diff --git a/src/database/common.h b/src/database/common.h index 6a26a374..f83c6182 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -89,4 +89,30 @@ extern const char *sqlite3ErrName(int rc); }\ } +// Macro to time a database operation expression EXPR if debug.timing is +// enabled. +#define TIMED_DB_OP(EXPR) { \ + if(!config.debug.timing.v.b) { EXPR; } \ + else { \ + struct timespec _timed_start, _timed_end; \ + clock_gettime(CLOCK_MONOTONIC, &_timed_start); \ + (EXPR); \ + clock_gettime(CLOCK_MONOTONIC, &_timed_end); \ + long _timed_elapsed = (_timed_end.tv_sec - _timed_start.tv_sec) * 10000 + (_timed_end.tv_nsec - _timed_start.tv_nsec) / 100000; \ + log_debug(DEBUG_TIMING, "Database operation %s took %.1f ms", str(EXPR), 0.1*_timed_elapsed); \ + }} + +// Macro to time a database operation expression EXPR that returns a value if +// debug.timing is enabled. +#define TIMED_DB_OP_RESULT(_result, EXPR) { \ + if(!config.debug.timing.v.b) { _result = EXPR; } \ + else { \ + struct timespec _timed_start, _timed_end; \ + clock_gettime(CLOCK_MONOTONIC, &_timed_start); \ + _result = (EXPR); \ + clock_gettime(CLOCK_MONOTONIC, &_timed_end); \ + long _timed_elapsed = (_timed_end.tv_sec - _timed_start.tv_sec) * 10000 + (_timed_end.tv_nsec - _timed_start.tv_nsec) / 100000; \ + log_debug(DEBUG_TIMING, "Database operation %s took %.1f ms", str(EXPR), 0.1*_timed_elapsed); \ + }} + #endif //DATABASE_COMMON_H diff --git a/src/database/database-thread.c b/src/database/database-thread.c index 50537e14..dbbee41f 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -135,9 +135,7 @@ void *DB_thread(void *val) // Save data to database DBOPEN_OR_AGAIN(); - lock_shm(); - export_queries_to_disk(false); - unlock_shm(); + TIMED_DB_OP(export_queries_to_disk(false)); // Intermediate cancellation-point if(killed) @@ -146,8 +144,8 @@ void *DB_thread(void *val) // Check if GC should be done on the database if(DBdeleteoldqueries) { - // No thread locks needed - delete_old_queries_in_DB(db); + /* No thread locks needed */ + TIMED_DB_OP(delete_old_queries_in_DB(db)); DBdeleteoldqueries = false; } @@ -165,7 +163,7 @@ void *DB_thread(void *val) if(now - lastAnalyze >= DATABASE_ANALYZE_INTERVAL) { DBOPEN_OR_AGAIN(); - analyze_database(db); + TIMED_DB_OP(analyze_database(db)); lastAnalyze = now; DBCLOSE_OR_BREAK(); } @@ -179,7 +177,7 @@ void *DB_thread(void *val) if(now - lastMACVendor >= DATABASE_MACVENDOR_INTERVAL) { DBOPEN_OR_AGAIN(); - updateMACVendorRecords(db); + TIMED_DB_OP(updateMACVendorRecords(db)); lastMACVendor = now; DBCLOSE_OR_BREAK(); } @@ -192,7 +190,7 @@ void *DB_thread(void *val) if(get_and_clear_event(PARSE_NEIGHBOR_CACHE)) { DBOPEN_OR_AGAIN(); - parse_neighbor_cache(db); + TIMED_DB_OP(parse_neighbor_cache(db)); DBCLOSE_OR_BREAK(); } @@ -204,7 +202,7 @@ void *DB_thread(void *val) { DBOPEN_OR_AGAIN(); lock_shm(); - reimport_aliasclients(db); + TIMED_DB_OP(reimport_aliasclients(db)); unlock_shm(); DBCLOSE_OR_BREAK(); } diff --git a/src/database/query-table.c b/src/database/query-table.c index 322e2c38..450bb52e 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -238,7 +238,7 @@ bool init_memory_database(void) // queries from the in-memory database (including the query with ID 0) // to the on-disk database. rc = sqlite3_prepare_v3(_memdb, "INSERT INTO disk.query_storage SELECT * FROM query_storage " \ - "WHERE id > IFNULL((SELECT MAX(id) FROM disk.query_storage), -1) "\ + "WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.query_storage) "\ "AND timestamp < ?", -1, SQLITE_PREPARE_PERSISTENT, &queries_to_disk_stmt, NULL); if( rc != SQLITE_OK ) @@ -247,11 +247,15 @@ bool init_memory_database(void) return false; } + // Export linking tables to disk database + // We limit the export to new records to avoid the overhead of many + // IGNORE executions for records that are already present on disk. It + // follows the same logic as for the main query_storage table above. const char *subtable_sql[SUBTABLE_STMTS] = { - "INSERT OR IGNORE INTO disk.domain_by_id SELECT * FROM domain_by_id", - "INSERT OR IGNORE INTO disk.client_by_id SELECT * FROM client_by_id", - "INSERT OR IGNORE INTO disk.forward_by_id SELECT * FROM forward_by_id", - "INSERT OR IGNORE INTO disk.addinfo_by_id SELECT * FROM addinfo_by_id", + "INSERT OR IGNORE INTO disk.domain_by_id SELECT * FROM domain_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.domain_by_id)", + "INSERT OR IGNORE INTO disk.client_by_id SELECT * FROM client_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.client_by_id)", + "INSERT OR IGNORE INTO disk.forward_by_id SELECT * FROM forward_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.forward_by_id)", + "INSERT OR IGNORE INTO disk.addinfo_by_id SELECT * FROM addinfo_by_id WHERE id > (SELECT IFNULL(MAX(id), -1) FROM disk.addinfo_by_id)", "UPDATE disk.sqlite_sequence SET seq = (SELECT seq FROM sqlite_sequence WHERE disk.sqlite_sequence.name = sqlite_sequence.name)" }; @@ -368,7 +372,7 @@ static bool get_memdb_size(sqlite3 *db, size_t *memsize, int *queries) *memsize = page_count * page_size; // Get number of queries in the memory table - if((*queries = get_number_of_queries_in_DB(db, "query_storage")) == DB_FAILED) + if((*queries = get_number_of_queries_in_DB(db, "query_storage", NULL)) == DB_FAILED) return false; return true; @@ -504,23 +508,31 @@ bool detach_database(sqlite3* db, const char **message, const char *alias) return okay; } -// Get number of queries either in the temp or in the on-diks database +// Get number of queries either in the temp or in the on-disk database // This routine is used by the API routines. -int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename) +int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, double *earliest_timestamp) { int rc = 0, num = 0; sqlite3_stmt *stmt = NULL; - // Count number of rows - const size_t buflen = 42 + strlen(tablename); - char *querystr = calloc(buflen, sizeof(char)); - snprintf(querystr, buflen, "SELECT COUNT(*) FROM %s", tablename); - // The database pointer may be NULL, meaning we want the memdb if(db == NULL) db = get_memdb(); - // PRAGMA page_size + // Build query string based on whether we need the earliest timestamp too + const size_t buflen = 38 + strlen(tablename); + char *querystr = calloc(buflen, sizeof(char)); + if(earliest_timestamp != NULL) + { + // Get both count and earliest timestamp + snprintf(querystr, buflen, "SELECT COUNT(*), MIN(timestamp) FROM %s", tablename); + } + else + { + // Get only count + snprintf(querystr, buflen, "SELECT COUNT(*) FROM %s", tablename); + } + rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); if( rc != SQLITE_OK ) { @@ -532,7 +544,13 @@ int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename) } rc = sqlite3_step(stmt); if( rc == SQLITE_ROW ) + { + // Get count from first column num = sqlite3_column_int(stmt, 0); + // Get timestamp from second column if requested + if(earliest_timestamp != NULL && sqlite3_column_type(stmt, 1) != SQLITE_NULL) + *earliest_timestamp = sqlite3_column_double(stmt, 1); + } sqlite3_finalize(stmt); free(querystr); @@ -617,8 +635,8 @@ bool import_queries_from_disk(void) } // Get number of queries on disk before detaching - disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage"); - mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage"); + disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage", NULL); + mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage", NULL); log_info("Imported %u queries from the on-disk database (it has %u rows)", mem_db_num, disk_db_num); @@ -640,7 +658,7 @@ bool export_queries_to_disk(const bool final) // Only try to export to database if it is known to not be broken if(FTLDBerror()) - return false; + return false; // Start database timer timer_start(DATABASE_WRITE_TIMER); @@ -710,7 +728,7 @@ bool export_queries_to_disk(const bool final) } // Update number of queries in the disk database - disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage"); + disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage", NULL); } // Export linking tables and current AUTOINCREMENT values to the disk database @@ -778,7 +796,7 @@ bool delete_old_queries_from_db(const bool use_memdb, const double mintime) mintime, sqlite3_errstr(rc)); // Update number of queries in in-memory database - const int new_num = get_number_of_queries_in_DB(NULL, "query_storage"); + const int new_num = get_number_of_queries_in_DB(NULL, "query_storage", NULL); log_debug(DEBUG_GC, "delete_old_queries_from_db(): Deleted %i (%u) queries, new number of queries in memory: %i", sqlite3_changes(db), (mem_db_num - new_num), new_num); mem_db_num = new_num; @@ -1738,7 +1756,7 @@ bool queries_to_database(void) } // Update number of queries in in-memory database - mem_db_num = get_number_of_queries_in_DB(NULL, "query_storage"); + mem_db_num = get_number_of_queries_in_DB(NULL, "query_storage", NULL); if(config.debug.database.v.b && updated + added > 0) { diff --git a/src/database/query-table.h b/src/database/query-table.h index 835f8677..0f4cc588 100644 --- a/src/database/query-table.h +++ b/src/database/query-table.h @@ -113,7 +113,7 @@ void close_memory_database(void); bool import_queries_from_disk(void); bool attach_database(sqlite3* db, const char **message, const char *path, const char *alias); bool detach_database(sqlite3* db, const char **message, const char *alias); -int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename); +int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, double *earliest_timestamp); bool export_queries_to_disk(const bool final); bool delete_old_queries_from_db(const bool use_memdb, const double mintime); bool add_additional_info_column(sqlite3 *db); diff --git a/src/database/shell.c b/src/database/shell.c index 4c9d2fb9..fe4a1cf7 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -124,8 +124,9 @@ typedef sqlite3_uint64 u64; typedef unsigned char u8; #include #include -// print_FTL_version() -#include "../log.h" +#ifndef _WIN32 +# include +#endif #if !defined(_WIN32) && !defined(WIN32) # include @@ -650,8 +651,19 @@ static int cli_strncmp(const char *a, const char *b, size_t n){ return strncmp(a,b,n); } -/* Return the current wall-clock time */ +/* Return the current wall-clock time in microseconds since the +** Unix epoch (1970-01-01T00:00:00Z) +*/ static sqlite3_int64 timeOfDay(void){ +#if defined(_WIN64) + sqlite3_uint64 t; + FILETIME tm; + GetSystemTimePreciseAsFileTime(&tm); + t = ((u64)tm.dwHighDateTime<<32) | (u64)tm.dwLowDateTime; + t += 116444736000000000LL; + t /= 10; + return t; +#elif defined(_WIN32) static sqlite3_vfs *clockVfs = 0; sqlite3_int64 t; if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); @@ -663,7 +675,12 @@ static sqlite3_int64 timeOfDay(void){ clockVfs->xCurrentTime(clockVfs, &r); t = (sqlite3_int64)(r*86400000.0); } - return t; + return t*1000; +#else + struct timeval sNow; + (void)gettimeofday(&sNow,0); + return ((i64)sNow.tv_sec)*1000000 + sNow.tv_usec; +#endif } #if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) @@ -708,8 +725,8 @@ static void endTimer(FILE *out){ sqlite3_int64 iEnd = timeOfDay(); struct rusage sEnd; getrusage(RUSAGE_SELF, &sEnd); - sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", - (iEnd - iBegin)*0.001, + sqlite3_fprintf(out, "Run Time: real %.6f user %.6f sys %.6f\n", + (iEnd - iBegin)*0.000001, timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); } @@ -787,10 +804,19 @@ static void endTimer(FILE *out){ FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; sqlite3_int64 ftWallEnd = timeOfDay(); getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); - sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", - (ftWallEnd - ftWallBegin)*0.001, +#ifdef _WIN64 + /* microsecond precision on 64-bit windows */ + sqlite3_fprintf(out, "Run Time: real %.6f user %f sys %f\n", + (ftWallEnd - ftWallBegin)*0.000001, timeDiff(&ftUserBegin, &ftUserEnd), timeDiff(&ftKernelBegin, &ftKernelEnd)); +#else + /* millisecond precisino on 32-bit windows */ + sqlite3_fprintf(out, "Run Time: real %.3f user %.3f sys %.3f\n", + (ftWallEnd - ftWallBegin)*0.000001, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#endif } } @@ -1130,7 +1156,7 @@ static int decodeUtf8(const unsigned char *z, int *pU){ && (z[3] & 0xc0)==0x80 ){ *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 - | (z[4] & 0x3f); + | (z[3] & 0x3f); return 4; } *pU = 0; @@ -1193,14 +1219,24 @@ static int isVt100(const unsigned char *z){ ** Take into account zero-width and double-width Unicode characters. ** In other words, a zero-width character does not count toward the ** the w limit. A double-width character counts as two. +** +** w should normally be a small number. A couple hundred at most. This +** routine caps w at 100 million to avoid integer overflow issues. */ static void utf8_width_print(FILE *out, int w, const char *zUtf){ const unsigned char *a = (const unsigned char*)zUtf; + static const int mxW = 10000000; unsigned char c; int i = 0; int n = 0; int k; - int aw = w<0 ? -w : w; + int aw; + if( w<-mxW ){ + w = -mxW; + }else if( w>mxW ){ + w= mxW; + } + aw = w<0 ? -w : w; if( zUtf==0 ) zUtf = ""; while( (c = a[i])!=0 ){ if( (c&0xc0)==0xc0 ){ @@ -1270,12 +1306,21 @@ static int strlen30(const char *z){ /* ** Return the length of a string in characters. Multibyte UTF8 characters -** count as a single character. +** count as a single character for single-width characters, or as two +** characters for double-width characters. */ static int strlenChar(const char *z){ int n = 0; while( *z ){ - if( (0xc0&*(z++))!=0x80 ) n++; + if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = decodeUtf8((const u8*)z, &u); + z += len; + n += cli_wcwidth(u); + } } return n; } @@ -1318,7 +1363,7 @@ static FILE * openChrSource(const char *zFile){ ** This routine reads a line of text from FILE in, stores ** the text in memory obtained from malloc() and returns a pointer ** to the text. NULL is returned at end of file, or if malloc() -** fails. +** fails, or if the length of the line is longer than about a gigabyte. ** ** If zLine is not NULL then it is a malloced buffer returned from ** a previous call to this routine that may be reused. @@ -1329,6 +1374,10 @@ static char *local_getline(char *zLine, FILE *in){ while( 1 ){ if( n+100>nLine ){ + if( nLine>=1073741773 ){ + free(zLine); + return 0; + } nLine = nLine*2 + 100; zLine = realloc(zLine, nLine); shell_check_oom(zLine); @@ -1412,10 +1461,14 @@ static int hexDigitValue(char c){ /* ** Interpret zArg as an integer value, possibly with suffixes. +** +** If the value specified by zArg is outside the range of values that +** can be represented using a 64-bit twos-complement integer, then return +** the nearest representable value. */ static sqlite3_int64 integerValue(const char *zArg){ - sqlite3_int64 v = 0; - static const struct { char *zSuffix; int iMult; } aMult[] = { + sqlite3_uint64 v = 0; + static const struct { char *zSuffix; unsigned int iMult; } aMult[] = { { "KiB", 1024 }, { "MiB", 1024*1024 }, { "GiB", 1024*1024*1024 }, @@ -1438,22 +1491,30 @@ static sqlite3_int64 integerValue(const char *zArg){ int x; zArg += 2; while( (x = hexDigitValue(zArg[0]))>=0 ){ + if( v > 0x0fffffffffffffffULL ) goto integer_overflow; v = (v<<4) + x; zArg++; } }else{ while( IsDigit(zArg[0]) ){ - v = v*10 + zArg[0] - '0'; + if( v>=922337203685477580LL ){ + if( v>922337203685477580LL || zArg[0]>='8' ) goto integer_overflow; + } + v = v*10 + (zArg[0] - '0'); zArg++; } } for(i=0; i0x7fffffffffffffffULL ) goto integer_overflow; + return isNeg? -(sqlite3_int64)v : (sqlite3_int64)v; +integer_overflow: + return isNeg ? (i64)0x8000000000000000LL : 0x7fffffffffffffffLL; } /* @@ -1461,9 +1522,9 @@ static sqlite3_int64 integerValue(const char *zArg){ */ typedef struct ShellText ShellText; struct ShellText { - char *z; - int n; - int nAlloc; + char *zTxt; /* The text */ + i64 n; /* Number of bytes of zTxt[] actually used */ + i64 nAlloc; /* Number of bytes allocated for zTxt[] */ }; /* @@ -1473,7 +1534,7 @@ static void initText(ShellText *p){ memset(p, 0, sizeof(*p)); } static void freeText(ShellText *p){ - free(p->z); + sqlite3_free(p->zTxt); initText(p); } @@ -1498,26 +1559,26 @@ static void appendText(ShellText *p, const char *zAppend, char quote){ } } - if( p->z==0 || p->n+len>=p->nAlloc ){ + if( p->zTxt==0 || p->n+len>=p->nAlloc ){ p->nAlloc = p->nAlloc*2 + len + 20; - p->z = realloc(p->z, p->nAlloc); - shell_check_oom(p->z); + p->zTxt = sqlite3_realloc64(p->zTxt, p->nAlloc); + shell_check_oom(p->zTxt); } if( quote ){ - char *zCsr = p->z+p->n; + char *zCsr = p->zTxt+p->n; *zCsr++ = quote; for(i=0; in = (int)(zCsr - p->z); + p->n = (i64)(zCsr - p->zTxt); *zCsr = '\0'; }else{ - memcpy(p->z+p->n, zAppend, nAppend); + memcpy(p->zTxt+p->n, zAppend, nAppend); p->n += nAppend; - p->z[p->n] = '\0'; + p->zTxt[p->n] = '\0'; } } @@ -1542,6 +1603,9 @@ static char quoteChar(const char *zName){ /* ** Construct a fake object name and column list to describe the structure ** of the view, virtual table, or table valued function zSchema.zName. +** +** The returned string comes from sqlite3_mprintf() and should be freed +** by the caller using sqlite3_free(). */ static char *shellFakeSchema( sqlite3 *db, /* The database connection containing the vtab */ @@ -1582,9 +1646,9 @@ static char *shellFakeSchema( sqlite3_finalize(pStmt); if( nRow==0 ){ freeText(&s); - s.z = 0; + s.zTxt = 0; } - return s.z; + return s.zTxt; } /* @@ -1687,7 +1751,7 @@ static void shellAddSchemaName( }else{ z = sqlite3_mprintf("%z\n/* %s */", z, zFake); } - free(zFake); + sqlite3_free(zFake); } if( z ){ sqlite3_result_text(pCtx, z, -1, sqlite3_free); @@ -1708,10 +1772,9 @@ static void shellAddSchemaName( #define SQLITE_EXTENSION_INIT1 #define SQLITE_EXTENSION_INIT2(X) (void)(X) -#if defined(_WIN32) && defined(_MSC_VER) -/************************* Begin test_windirent.h ******************/ +/************************* Begin ../ext/misc/windirent.h ******************/ /* -** 2015 November 30 +** 2025-06-05 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -1721,322 +1784,160 @@ static void shellAddSchemaName( ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains declarations for most of the opendir() family of -** POSIX functions on Win32 using the MSVCRT. +** +** An implementation of opendir(), readdir(), and closedir() for Windows, +** based on the FindFirstFile(), FindNextFile(), and FindClose() APIs +** of Win32. +** +** #include this file inside any C-code module that needs to use +** opendir()/readdir()/closedir(). This file is a no-op on non-Windows +** machines. On Windows, static functions are defined that implement +** those standard interfaces. */ - #if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H) #define SQLITE_WINDIRENT_H -/* -** We need several data types from the Windows SDK header. -*/ - #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif - -#include "windows.h" - -/* -** We need several support functions from the SQLite core. -*/ - -/* #include "sqlite3.h" */ - -/* -** We need several things from the ANSI and MSVCRT headers. -*/ - +#include +#include #include #include #include -#include #include #include #include - -/* -** We may need several defines that should have been in "sys/stat.h". -*/ - +#include +#ifndef FILENAME_MAX +# define FILENAME_MAX (260) +#endif #ifndef S_ISREG -#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) #endif - #ifndef S_ISDIR -#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) #endif - #ifndef S_ISLNK -#define S_ISLNK(mode) (0) +#define S_ISLNK(m) (0) #endif +typedef unsigned short mode_t; -/* -** We may need to provide the "mode_t" type. +/* The dirent object for Windows is abbreviated. The only field really +** usable by applications is d_name[]. */ - -#ifndef MODE_T_DEFINED - #define MODE_T_DEFINED - typedef unsigned short mode_t; -#endif - -/* -** We may need to provide the "ino_t" type. -*/ - -#ifndef INO_T_DEFINED - #define INO_T_DEFINED - typedef unsigned short ino_t; -#endif - -/* -** We need to define "NAME_MAX" if it was not present in "limits.h". -*/ - -#ifndef NAME_MAX -# ifdef FILENAME_MAX -# define NAME_MAX (FILENAME_MAX) -# else -# define NAME_MAX (260) -# endif -# define DIRENT_NAME_MAX (NAME_MAX) -#endif - -/* -** We need to define "NULL_INTPTR_T" and "BAD_INTPTR_T". -*/ - -#ifndef NULL_INTPTR_T -# define NULL_INTPTR_T ((intptr_t)(0)) -#endif - -#ifndef BAD_INTPTR_T -# define BAD_INTPTR_T ((intptr_t)(-1)) -#endif - -/* -** We need to provide the necessary structures and related types. -*/ - -#ifndef DIRENT_DEFINED -#define DIRENT_DEFINED -typedef struct DIRENT DIRENT; -typedef DIRENT *LPDIRENT; -struct DIRENT { - ino_t d_ino; /* Sequence number, do not use. */ - unsigned d_attributes; /* Win32 file attributes. */ - char d_name[NAME_MAX + 1]; /* Name within the directory. */ +struct dirent { + int d_ino; /* Inode number (synthesized) */ + unsigned d_attributes; /* File attributes */ + char d_name[FILENAME_MAX]; /* Null-terminated filename */ }; -#endif -#ifndef DIR_DEFINED -#define DIR_DEFINED +/* The internals of DIR are opaque according to standards. So it +** does not matter what we put here. */ typedef struct DIR DIR; -typedef DIR *LPDIR; struct DIR { - intptr_t d_handle; /* Value returned by "_findfirst". */ - DIRENT d_first; /* DIRENT constructed based on "_findfirst". */ - DIRENT d_next; /* DIRENT constructed based on "_findnext". */ + intptr_t d_handle; /* Handle for findfirst()/findnext() */ + struct dirent cur; /* Current entry */ }; -#endif + +/* Ignore hidden and system files */ +#define WindowsFileToIgnore(a) \ + ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) /* -** Provide a macro, for use by the implementation, to determine if a -** particular directory entry should be skipped over when searching for -** the next directory entry that should be returned by the readdir(). +** Close a previously opened directory */ - -#ifndef is_filtered -# define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) -#endif - -/* -** Provide the function prototype for the POSIX compatible getenv() -** function. This function is not thread-safe. -*/ - -extern const char *windirent_getenv(const char *name); - -/* -** Finally, we can provide the function prototypes for the opendir(), -** readdir(), and closedir() POSIX functions. -*/ - -extern LPDIR opendir(const char *dirname); -extern LPDIRENT readdir(LPDIR dirp); -extern INT closedir(LPDIR dirp); - -#endif /* defined(WIN32) && defined(_MSC_VER) */ - -/************************* End test_windirent.h ********************/ -/************************* Begin test_windirent.c ******************/ -/* -** 2015 November 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains code to implement most of the opendir() family of -** POSIX functions on Win32 using the MSVCRT. -*/ - -#if defined(_WIN32) && defined(_MSC_VER) -/* #include "test_windirent.h" */ - -/* -** Implementation of the POSIX getenv() function using the Win32 API. -** This function is not thread-safe. -*/ -const char *windirent_getenv( - const char *name -){ - static char value[32768]; /* Maximum length, per MSDN */ - DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */ - DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */ - - memset(value, 0, sizeof(value)); - dwRet = GetEnvironmentVariableA(name, value, dwSize); - if( dwRet==0 || dwRet>dwSize ){ - /* - ** The function call to GetEnvironmentVariableA() failed -OR- - ** the buffer is not large enough. Either way, return NULL. - */ - return 0; - }else{ - /* - ** The function call to GetEnvironmentVariableA() succeeded - ** -AND- the buffer contains the entire value. - */ - return value; +static int closedir(DIR *pDir){ + int rc = 0; + if( pDir==0 ){ + return EINVAL; } + if( pDir->d_handle!=0 && pDir->d_handle!=(-1) ){ + rc = _findclose(pDir->d_handle); + } + sqlite3_free(pDir); + return rc; } /* -** Implementation of the POSIX opendir() function using the MSVCRT. +** Open a new directory. The directory name should be UTF-8 encoded. +** appropriate translations happen automatically. */ -LPDIR opendir( - const char *dirname /* Directory name, UTF8 encoding */ -){ - struct _wfinddata_t data; - LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR)); - SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]); +static DIR *opendir(const char *zDirName){ + DIR *pDir; wchar_t *b1; sqlite3_int64 sz; + struct _wfinddata_t data; - if( dirp==NULL ) return NULL; - memset(dirp, 0, sizeof(DIR)); - - /* TODO: Remove this if Unix-style root paths are not used. */ - if( sqlite3_stricmp(dirname, "/")==0 ){ - dirname = windirent_getenv("SystemDrive"); - } - + pDir = sqlite3_malloc64( sizeof(DIR) ); + if( pDir==0 ) return 0; + memset(pDir, 0, sizeof(DIR)); memset(&data, 0, sizeof(data)); - sz = strlen(dirname); + sz = strlen(zDirName); b1 = sqlite3_malloc64( (sz+3)*sizeof(b1[0]) ); if( b1==0 ){ - closedir(dirp); + closedir(pDir); return NULL; } - sz = MultiByteToWideChar(CP_UTF8, 0, dirname, sz, b1, sz); + sz = MultiByteToWideChar(CP_UTF8, 0, zDirName, sz, b1, sz); b1[sz++] = '\\'; b1[sz++] = '*'; b1[sz] = 0; - if( sz+1>(sqlite3_int64)namesize ){ - closedir(dirp); + if( sz+1>sizeof(data.name)/sizeof(data.name[0]) ){ + closedir(pDir); sqlite3_free(b1); return NULL; } memcpy(data.name, b1, (sz+1)*sizeof(b1[0])); sqlite3_free(b1); - dirp->d_handle = _wfindfirst(data.name, &data); - - if( dirp->d_handle==BAD_INTPTR_T ){ - closedir(dirp); + pDir->d_handle = _wfindfirst(data.name, &data); + if( pDir->d_handle<0 ){ + closedir(pDir); return NULL; } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ){ -next: - + while( WindowsFileToIgnore(data) ){ memset(&data, 0, sizeof(data)); - if( _wfindnext(dirp->d_handle, &data)==-1 ){ - closedir(dirp); + if( _wfindnext(pDir->d_handle, &data)==-1 ){ + closedir(pDir); return NULL; } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; } - - dirp->d_first.d_attributes = data.attrib; + pDir->cur.d_ino = 0; + pDir->cur.d_attributes = data.attrib; WideCharToMultiByte(CP_UTF8, 0, data.name, -1, - dirp->d_first.d_name, DIRENT_NAME_MAX, 0, 0); - return dirp; + pDir->cur.d_name, FILENAME_MAX, 0, 0); + return pDir; } /* -** Implementation of the POSIX readdir() function using the MSVCRT. +** Read the next entry from a directory. +** +** The returned struct-dirent object is managed by DIR. It is only +** valid until the next readdir() or closedir() call. Only the +** d_name[] field is meaningful. The d_name[] value has been +** translated into UTF8. */ -LPDIRENT readdir( - LPDIR dirp -){ +static struct dirent *readdir(DIR *pDir){ struct _wfinddata_t data; - - if( dirp==NULL ) return NULL; - - if( dirp->d_first.d_ino==0 ){ - dirp->d_first.d_ino++; - dirp->d_next.d_ino++; - - return &dirp->d_first; + if( pDir==0 ) return 0; + if( (pDir->cur.d_ino++)==0 ){ + return &pDir->cur; } - -next: - - memset(&data, 0, sizeof(data)); - if( _wfindnext(dirp->d_handle, &data)==-1 ) return NULL; - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; - - dirp->d_next.d_ino++; - dirp->d_next.d_attributes = data.attrib; + do{ + memset(&data, 0, sizeof(data)); + if( _wfindnext(pDir->d_handle, &data)==-1 ){ + return NULL; + } + }while( WindowsFileToIgnore(data) ); + pDir->cur.d_attributes = data.attrib; WideCharToMultiByte(CP_UTF8, 0, data.name, -1, - dirp->d_next.d_name, DIRENT_NAME_MAX, 0, 0); - return &dirp->d_next; + pDir->cur.d_name, FILENAME_MAX, 0, 0); + return &pDir->cur; } -/* -** Implementation of the POSIX closedir() function using the MSVCRT. -*/ -INT closedir( - LPDIR dirp -){ - INT result = 0; +#endif /* defined(_WIN32) && defined(_MSC_VER) */ - if( dirp==NULL ) return EINVAL; - - if( dirp->d_handle!=NULL_INTPTR_T && dirp->d_handle!=BAD_INTPTR_T ){ - result = _findclose(dirp->d_handle); - } - - sqlite3_free(dirp); - return result; -} - -#endif /* defined(WIN32) && defined(_MSC_VER) */ - -/************************* End test_windirent.c ********************/ -#define dirent DIRENT -#endif +/************************* End ../ext/misc/windirent.h ********************/ /************************* Begin ../ext/misc/memtrace.c ******************/ /* ** 2019-01-21 @@ -2437,6 +2338,8 @@ SQLITE_EXTENSION_INIT1 #include #include #include +// print_FTL_version() +#include "../log.h" #ifndef SQLITE_AMALGAMATION /* typedef sqlite3_uint64 u64; */ @@ -3825,7 +3728,8 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); if( p->a==0 ) goto new_from_text_failed; memset(p->a+p->nDigit, 0, iExp); p->nDigit += iExp; @@ -3844,7 +3748,8 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); if( p->a==0 ) goto new_from_text_failed; memmove(p->a+iExp, p->a, p->nDigit); memset(p->a, 0, iExp); @@ -3852,6 +3757,10 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ p->nFrac += iExp; } } + if( p->sign ){ + for(i=0; inDigit && p->a[i]==0; i++){} + if( i>=p->nDigit ) p->sign = 0; + } return p; new_from_text_failed: @@ -3944,7 +3853,7 @@ static void decimal_result(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_null(pCtx); return; } - z = sqlite3_malloc( p->nDigit+4 ); + z = sqlite3_malloc64( (sqlite3_int64)p->nDigit+4 ); if( z==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -4009,7 +3918,7 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ for(nZero=0; nZeroa[nZero]==0; nZero++){} nFrac = p->nFrac + (nDigit - p->nDigit); nDigit -= nZero; - z = sqlite3_malloc( nDigit+20 ); + z = sqlite3_malloc64( (sqlite3_int64)nDigit+20 ); if( z==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -4054,13 +3963,21 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ ** pB!=0 ** pB->isNull==0 */ -static int decimal_cmp(const Decimal *pA, const Decimal *pB){ +static int decimal_cmp(Decimal *pA, Decimal *pB){ int nASig, nBSig, rc, n; + while( pA->nFrac>0 && pA->a[pA->nDigit-1]==0 ){ + pA->nDigit--; + pA->nFrac--; + } + while( pB->nFrac>0 && pB->a[pB->nDigit-1]==0 ){ + pB->nDigit--; + pB->nFrac--; + } if( pA->sign!=pB->sign ){ return pA->sign ? -1 : +1; } if( pA->sign ){ - const Decimal *pTemp = pA; + Decimal *pTemp = pA; pA = pB; pB = pTemp; } @@ -4222,7 +4139,8 @@ static void decimalMul(Decimal *pA, Decimal *pB){ ){ goto mul_end; } - acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); + acc = sqlite3_malloc64( (sqlite3_int64)pA->nDigit + + (sqlite3_int64)pB->nDigit + 2 ); if( acc==0 ){ pA->oom = 1; goto mul_end; @@ -4309,7 +4227,7 @@ static Decimal *decimalFromDouble(double r){ isNeg = 0; } memcpy(&a,&r,sizeof(a)); - if( a==0 ){ + if( a==0 || a==(sqlite3_int64)0x8000000000000000LL){ e = 0; m = 0; }else{ @@ -4584,512 +4502,6 @@ int sqlite3_decimal_init( } /************************* End ../ext/misc/decimal.c ********************/ -/************************* Begin ../ext/misc/percentile.c ******************/ -/* -** 2013-05-28 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains code to implement the percentile(Y,P) SQL function -** and similar as described below: -** -** (1) The percentile(Y,P) function is an aggregate function taking -** exactly two arguments. -** -** (2) If the P argument to percentile(Y,P) is not the same for every -** row in the aggregate then an error is thrown. The word "same" -** in the previous sentence means that the value differ by less -** than 0.001. -** -** (3) If the P argument to percentile(Y,P) evaluates to anything other -** than a number in the range of 0.0 to 100.0 inclusive then an -** error is thrown. -** -** (4) If any Y argument to percentile(Y,P) evaluates to a value that -** is not NULL and is not numeric then an error is thrown. -** -** (5) If any Y argument to percentile(Y,P) evaluates to plus or minus -** infinity then an error is thrown. (SQLite always interprets NaN -** values as NULL.) -** -** (6) Both Y and P in percentile(Y,P) can be arbitrary expressions, -** including CASE WHEN expressions. -** -** (7) The percentile(Y,P) aggregate is able to handle inputs of at least -** one million (1,000,000) rows. -** -** (8) If there are no non-NULL values for Y, then percentile(Y,P) -** returns NULL. -** -** (9) If there is exactly one non-NULL value for Y, the percentile(Y,P) -** returns the one Y value. -** -** (10) If there N non-NULL values of Y where N is two or more and -** the Y values are ordered from least to greatest and a graph is -** drawn from 0 to N-1 such that the height of the graph at J is -** the J-th Y value and such that straight lines are drawn between -** adjacent Y values, then the percentile(Y,P) function returns -** the height of the graph at P*(N-1)/100. -** -** (11) The percentile(Y,P) function always returns either a floating -** point number or NULL. -** -** (12) The percentile(Y,P) is implemented as a single C99 source-code -** file that compiles into a shared-library or DLL that can be loaded -** into SQLite using the sqlite3_load_extension() interface. -** -** (13) A separate median(Y) function is the equivalent percentile(Y,50). -** -** (14) A separate percentile_cont(Y,P) function is equivalent to -** percentile(Y,P/100.0). In other words, the fraction value in -** the second argument is in the range of 0 to 1 instead of 0 to 100. -** -** (15) A separate percentile_disc(Y,P) function is like -** percentile_cont(Y,P) except that instead of returning the weighted -** average of the nearest two input values, it returns the next lower -** value. So the percentile_disc(Y,P) will always return a value -** that was one of the inputs. -** -** (16) All of median(), percentile(Y,P), percentile_cont(Y,P) and -** percentile_disc(Y,P) can be used as window functions. -** -** Differences from standard SQL: -** -** * The percentile_cont(X,P) function is equivalent to the following in -** standard SQL: -** -** (percentile_cont(P) WITHIN GROUP (ORDER BY X)) -** -** The SQLite syntax is much more compact. The standard SQL syntax -** is also supported if SQLite is compiled with the -** -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES option. -** -** * No median(X) function exists in the SQL standard. App developers -** are expected to write "percentile_cont(0.5)WITHIN GROUP(ORDER BY X)". -** -** * No percentile(Y,P) function exists in the SQL standard. Instead of -** percential(Y,P), developers must write this: -** "percentile_cont(P/100.0) WITHIN GROUP (ORDER BY Y)". Note that -** the fraction parameter to percentile() goes from 0 to 100 whereas -** the fraction parameter in SQL standard percentile_cont() goes from -** 0 to 1. -** -** Implementation notes as of 2024-08-31: -** -** * The regular aggregate-function versions of these routines work -** by accumulating all values in an array of doubles, then sorting -** that array using quicksort before computing the answer. Thus -** the runtime is O(NlogN) where N is the number of rows of input. -** -** * For the window-function versions of these routines, the array of -** inputs is sorted as soon as the first value is computed. Thereafter, -** the array is kept in sorted order using an insert-sort. This -** results in O(N*K) performance where K is the size of the window. -** One can imagine alternative implementations that give O(N*logN*logK) -** performance, but they require more complex logic and data structures. -** The developers have elected to keep the asymptotically slower -** algorithm for now, for simplicity, under the theory that window -** functions are seldom used and when they are, the window size K is -** often small. The developers might revisit that decision later, -** should the need arise. -*/ -#if defined(SQLITE3_H) - /* no-op */ -#elif defined(SQLITE_STATIC_PERCENTILE) -/* # include "sqlite3.h" */ -#else -/* # include "sqlite3ext.h" */ - SQLITE_EXTENSION_INIT1 -#endif -#include -#include -#include - -/* The following object is the group context for a single percentile() -** aggregate. Remember all input Y values until the very end. -** Those values are accumulated in the Percentile.a[] array. -*/ -typedef struct Percentile Percentile; -struct Percentile { - unsigned nAlloc; /* Number of slots allocated for a[] */ - unsigned nUsed; /* Number of slots actually used in a[] */ - char bSorted; /* True if a[] is already in sorted order */ - char bKeepSorted; /* True if advantageous to keep a[] sorted */ - char bPctValid; /* True if rPct is valid */ - double rPct; /* Fraction. 0.0 to 1.0 */ - double *a; /* Array of Y values */ -}; - -/* Details of each function in the percentile family */ -typedef struct PercentileFunc PercentileFunc; -struct PercentileFunc { - const char *zName; /* Function name */ - char nArg; /* Number of arguments */ - char mxFrac; /* Maximum value of the "fraction" input */ - char bDiscrete; /* True for percentile_disc() */ -}; -static const PercentileFunc aPercentFunc[] = { - { "median", 1, 1, 0 }, - { "percentile", 2, 100, 0 }, - { "percentile_cont", 2, 1, 0 }, - { "percentile_disc", 2, 1, 1 }, -}; - -/* -** Return TRUE if the input floating-point number is an infinity. -*/ -static int percentIsInfinity(double r){ - sqlite3_uint64 u; - assert( sizeof(u)==sizeof(r) ); - memcpy(&u, &r, sizeof(u)); - return ((u>>52)&0x7ff)==0x7ff; -} - -/* -** Return TRUE if two doubles differ by 0.001 or less. -*/ -static int percentSameValue(double a, double b){ - a -= b; - return a>=-0.001 && a<=0.001; -} - -/* -** Search p (which must have p->bSorted) looking for an entry with -** value y. Return the index of that entry. -** -** If bExact is true, return -1 if the entry is not found. -** -** If bExact is false, return the index at which a new entry with -** value y should be insert in order to keep the values in sorted -** order. The smallest return value in this case will be 0, and -** the largest return value will be p->nUsed. -*/ -static int percentBinarySearch(Percentile *p, double y, int bExact){ - int iFirst = 0; /* First element of search range */ - int iLast = p->nUsed - 1; /* Last element of search range */ - while( iLast>=iFirst ){ - int iMid = (iFirst+iLast)/2; - double x = p->a[iMid]; - if( xy ){ - iLast = iMid - 1; - }else{ - return iMid; - } - } - if( bExact ) return -1; - return iFirst; -} - -/* -** Generate an error for a percentile function. -** -** The error format string must have exactly one occurrence of "%%s()" -** (with two '%' characters). That substring will be replaced by the name -** of the function. -*/ -static void percentError(sqlite3_context *pCtx, const char *zFormat, ...){ - PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); - char *zMsg1; - char *zMsg2; - va_list ap; - - va_start(ap, zFormat); - zMsg1 = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - zMsg2 = zMsg1 ? sqlite3_mprintf(zMsg1, pFunc->zName) : 0; - sqlite3_result_error(pCtx, zMsg2, -1); - sqlite3_free(zMsg1); - sqlite3_free(zMsg2); -} - -/* -** The "step" function for percentile(Y,P) is called once for each -** input row. -*/ -static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ - Percentile *p; - double rPct; - int eType; - double y; - assert( argc==2 || argc==1 ); - - if( argc==1 ){ - /* Requirement 13: median(Y) is the same as percentile(Y,50). */ - rPct = 0.5; - }else{ - /* Requirement 3: P must be a number between 0 and 100 */ - PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); - eType = sqlite3_value_numeric_type(argv[1]); - rPct = sqlite3_value_double(argv[1])/(double)pFunc->mxFrac; - if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) - || rPct<0.0 || rPct>1.0 - ){ - percentError(pCtx, "the fraction argument to %%s()" - " is not between 0.0 and %.1f", - (double)pFunc->mxFrac); - return; - } - } - - /* Allocate the session context. */ - p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); - if( p==0 ) return; - - /* Remember the P value. Throw an error if the P value is different - ** from any prior row, per Requirement (2). */ - if( !p->bPctValid ){ - p->rPct = rPct; - p->bPctValid = 1; - }else if( !percentSameValue(p->rPct,rPct) ){ - percentError(pCtx, "the fraction argument to %%s()" - " is not the same for all input rows"); - return; - } - - /* Ignore rows for which Y is NULL */ - eType = sqlite3_value_type(argv[0]); - if( eType==SQLITE_NULL ) return; - - /* If not NULL, then Y must be numeric. Otherwise throw an error. - ** Requirement 4 */ - if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ - percentError(pCtx, "input to %%s() is not numeric"); - return; - } - - /* Throw an error if the Y value is infinity or NaN */ - y = sqlite3_value_double(argv[0]); - if( percentIsInfinity(y) ){ - percentError(pCtx, "Inf input to %%s()"); - return; - } - - /* Allocate and store the Y */ - if( p->nUsed>=p->nAlloc ){ - unsigned n = p->nAlloc*2 + 250; - double *a = sqlite3_realloc64(p->a, sizeof(double)*n); - if( a==0 ){ - sqlite3_free(p->a); - memset(p, 0, sizeof(*p)); - sqlite3_result_error_nomem(pCtx); - return; - } - p->nAlloc = n; - p->a = a; - } - if( p->nUsed==0 ){ - p->a[p->nUsed++] = y; - p->bSorted = 1; - }else if( !p->bSorted || y>=p->a[p->nUsed-1] ){ - p->a[p->nUsed++] = y; - }else if( p->bKeepSorted ){ - int i; - i = percentBinarySearch(p, y, 0); - if( i<(int)p->nUsed ){ - memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0])); - } - p->a[i] = y; - p->nUsed++; - }else{ - p->a[p->nUsed++] = y; - p->bSorted = 0; - } -} - -/* -** Interchange two doubles. -*/ -#define SWAP_DOUBLE(X,Y) {double ttt=(X);(X)=(Y);(Y)=ttt;} - -/* -** Sort an array of doubles. -** -** Algorithm: quicksort -** -** This is implemented separately rather than using the qsort() routine -** from the standard library because: -** -** (1) To avoid a dependency on qsort() -** (2) To avoid the function call to the comparison routine for each -** comparison. -*/ -static void percentSort(double *a, unsigned int n){ - int iLt; /* Entries before a[iLt] are less than rPivot */ - int iGt; /* Entries at or after a[iGt] are greater than rPivot */ - int i; /* Loop counter */ - double rPivot; /* The pivot value */ - - assert( n>=2 ); - if( a[0]>a[n-1] ){ - SWAP_DOUBLE(a[0],a[n-1]) - } - if( n==2 ) return; - iGt = n-1; - i = n/2; - if( a[0]>a[i] ){ - SWAP_DOUBLE(a[0],a[i]) - }else if( a[i]>a[iGt] ){ - SWAP_DOUBLE(a[i],a[iGt]) - } - if( n==3 ) return; - rPivot = a[i]; - iLt = i = 1; - do{ - if( a[i]iLt ) SWAP_DOUBLE(a[i],a[iLt]) - iLt++; - i++; - }else if( a[i]>rPivot ){ - do{ - iGt--; - }while( iGt>i && a[iGt]>rPivot ); - SWAP_DOUBLE(a[i],a[iGt]) - }else{ - i++; - } - }while( i=2 ) percentSort(a, iLt); - if( n-iGt>=2 ) percentSort(a+iGt, n-iGt); - -/* Uncomment for testing */ -#if 0 - for(i=0; ibSorted==0 ){ - assert( p->nUsed>1 ); - percentSort(p->a, p->nUsed); - p->bSorted = 1; - } - p->bKeepSorted = 1; - - /* Find and remove the row */ - i = percentBinarySearch(p, y, 1); - if( i>=0 ){ - p->nUsed--; - if( i<(int)p->nUsed ){ - memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0])); - } - } -} - -/* -** Compute the final output of percentile(). Clean up all allocated -** memory if and only if bIsFinal is true. -*/ -static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ - Percentile *p; - PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); - unsigned i1, i2; - double v1, v2; - double ix, vx; - p = (Percentile*)sqlite3_aggregate_context(pCtx, 0); - if( p==0 ) return; - if( p->a==0 ) return; - if( p->nUsed ){ - if( p->bSorted==0 ){ - assert( p->nUsed>1 ); - percentSort(p->a, p->nUsed); - p->bSorted = 1; - } - ix = p->rPct*(p->nUsed-1); - i1 = (unsigned)ix; - if( pFunc->bDiscrete ){ - vx = p->a[i1]; - }else{ - i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; - v1 = p->a[i1]; - v2 = p->a[i2]; - vx = v1 + (v2-v1)*(ix-i1); - } - sqlite3_result_double(pCtx, vx); - } - if( bIsFinal ){ - sqlite3_free(p->a); - memset(p, 0, sizeof(*p)); - }else{ - p->bKeepSorted = 1; - } -} -static void percentFinal(sqlite3_context *pCtx){ - percentCompute(pCtx, 1); -} -static void percentValue(sqlite3_context *pCtx){ - percentCompute(pCtx, 0); -} - -#if defined(_WIN32) && !defined(SQLITE3_H) && !defined(SQLITE_STATIC_PERCENTILE) - -#endif -int sqlite3_percentile_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - unsigned int i; -#ifdef SQLITE3EXT_H - SQLITE_EXTENSION_INIT2(pApi); -#else - (void)pApi; /* Unused parameter */ -#endif - (void)pzErrMsg; /* Unused parameter */ - for(i=0; i>52; m = a & ((((sqlite3_int64)1)<<52)-1); @@ -6210,19 +5627,20 @@ int sqlite3_ieee_init( ** SELECT * FROM generate_series(0,100,5); ** ** The query above returns integers from 0 through 100 counting by steps -** of 5. +** of 5. In other words, 0, 5, 10, 15, ..., 90, 95, 100. There are a total +** of 21 rows. ** ** SELECT * FROM generate_series(0,100); ** -** Integers from 0 through 100 with a step size of 1. +** Integers from 0 through 100 with a step size of 1. 101 rows. ** ** SELECT * FROM generate_series(20) LIMIT 10; ** -** Integers 20 through 29. +** Integers 20 through 29. 10 rows. ** ** SELECT * FROM generate_series(0,-100,-5); ** -** Integers 0 -5 -10 ... -100. +** Integers 0 -5 -10 ... -100. 21 rows. ** ** SELECT * FROM generate_series(0,-1); ** @@ -6298,140 +5716,89 @@ SQLITE_EXTENSION_INIT1 #include #ifndef SQLITE_OMIT_VIRTUALTABLE -/* -** Return that member of a generate_series(...) sequence whose 0-based -** index is ix. The 0th member is given by smBase. The sequence members -** progress per ix increment by smStep. -*/ -static sqlite3_int64 genSeqMember( - sqlite3_int64 smBase, - sqlite3_int64 smStep, - sqlite3_uint64 ix -){ - static const sqlite3_uint64 mxI64 = - ((sqlite3_uint64)0x7fffffff)<<32 | 0xffffffff; - if( ix>=mxI64 ){ - /* Get ix into signed i64 range. */ - ix -= mxI64; - /* With 2's complement ALU, this next can be 1 step, but is split into - * 2 for UBSAN's satisfaction (and hypothetical 1's complement ALUs.) */ - smBase += (mxI64/2) * smStep; - smBase += (mxI64 - mxI64/2) * smStep; - } - /* Under UBSAN (or on 1's complement machines), must do this last term - * in steps to avoid the dreaded (and harmless) signed multiply overflow. */ - if( ix>=2 ){ - sqlite3_int64 ix2 = (sqlite3_int64)ix/2; - smBase += ix2*smStep; - ix -= ix2; - } - return smBase + ((sqlite3_int64)ix)*smStep; -} - -/* typedef unsigned char u8; */ - -typedef struct SequenceSpec { - sqlite3_int64 iOBase; /* Original starting value ("start") */ - sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ - sqlite3_int64 iBase; /* Starting value to actually use */ - sqlite3_int64 iTerm; /* Terminal value to actually use */ - sqlite3_int64 iStep; /* Increment ("step") */ - sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */ - sqlite3_uint64 uSeqIndexNow; /* Current index during generation */ - sqlite3_int64 iValueNow; /* Current value during generation */ - u8 isNotEOF; /* Sequence generation not exhausted */ - u8 isReversing; /* Sequence is being reverse generated */ -} SequenceSpec; - -/* -** Prepare a SequenceSpec for use in generating an integer series -** given initialized iBase, iTerm and iStep values. Sequence is -** initialized per given isReversing. Other members are computed. -*/ -static void setupSequence( SequenceSpec *pss ){ - int bSameSigns; - pss->uSeqIndexMax = 0; - pss->isNotEOF = 0; - bSameSigns = (pss->iBase < 0)==(pss->iTerm < 0); - if( pss->iTerm < pss->iBase ){ - sqlite3_uint64 nuspan = 0; - if( bSameSigns ){ - nuspan = (sqlite3_uint64)(pss->iBase - pss->iTerm); - }else{ - /* Under UBSAN (or on 1's complement machines), must do this in steps. - * In this clause, iBase>=0 and iTerm<0 . */ - nuspan = 1; - nuspan += pss->iBase; - nuspan += -(pss->iTerm+1); - } - if( pss->iStep<0 ){ - pss->isNotEOF = 1; - if( nuspan==ULONG_MAX ){ - pss->uSeqIndexMax = ( pss->iStep>LLONG_MIN )? nuspan/-pss->iStep : 1; - }else if( pss->iStep>LLONG_MIN ){ - pss->uSeqIndexMax = nuspan/-pss->iStep; - } - } - }else if( pss->iTerm > pss->iBase ){ - sqlite3_uint64 puspan = 0; - if( bSameSigns ){ - puspan = (sqlite3_uint64)(pss->iTerm - pss->iBase); - }else{ - /* Under UBSAN (or on 1's complement machines), must do this in steps. - * In this clause, iTerm>=0 and iBase<0 . */ - puspan = 1; - puspan += pss->iTerm; - puspan += -(pss->iBase+1); - } - if( pss->iStep>0 ){ - pss->isNotEOF = 1; - pss->uSeqIndexMax = puspan/pss->iStep; - } - }else if( pss->iTerm == pss->iBase ){ - pss->isNotEOF = 1; - pss->uSeqIndexMax = 0; - } - pss->uSeqIndexNow = (pss->isReversing)? pss->uSeqIndexMax : 0; - pss->iValueNow = (pss->isReversing) - ? genSeqMember(pss->iBase, pss->iStep, pss->uSeqIndexMax) - : pss->iBase; -} - -/* -** Progress sequence generator to yield next value, if any. -** Leave its state to either yield next value or be at EOF. -** Return whether there is a next value, or 0 at EOF. -*/ -static int progressSequence( SequenceSpec *pss ){ - if( !pss->isNotEOF ) return 0; - if( pss->isReversing ){ - if( pss->uSeqIndexNow > 0 ){ - pss->uSeqIndexNow--; - pss->iValueNow -= pss->iStep; - }else{ - pss->isNotEOF = 0; - } - }else{ - if( pss->uSeqIndexNow < pss->uSeqIndexMax ){ - pss->uSeqIndexNow++; - pss->iValueNow += pss->iStep; - }else{ - pss->isNotEOF = 0; - } - } - return pss->isNotEOF; -} /* series_cursor is a subclass of sqlite3_vtab_cursor which will ** serve as the underlying representation of a cursor that scans -** over rows of the result +** over rows of the result. +** +** iOBase, iOTerm, and iOStep are the original values of the +** start=, stop=, and step= constraints on the query. These are +** the values reported by the start, stop, and step columns of the +** virtual table. +** +** iBase, iTerm, iStep, and bDescp are the actual values used to generate +** the sequence. These might be different from the iOxxxx values. +** For example in +** +** SELECT value FROM generate_series(1,11,2) +** WHERE value BETWEEN 4 AND 8; +** +** The iOBase is 1, but the iBase is 5. iOTerm is 11 but iTerm is 7. +** Another example: +** +** SELECT value FROM generate_series(1,15,3) ORDER BY value DESC; +** +** The cursor initialization for the above query is: +** +** iOBase = 1 iBase = 13 +** iOTerm = 15 iTerm = 1 +** iOStep = 3 iStep = 3 bDesc = 1 +** +** The actual step size is unsigned so that can have a value of +** +9223372036854775808 which is needed for querys like this: +** +** SELECT value +** FROM generate_series(9223372036854775807, +** -9223372036854775808, +** -9223372036854775808) +** ORDER BY value ASC; +** +** The setup for the previous query will be: +** +** iOBase = 9223372036854775807 iBase = -1 +** iOTerm = -9223372036854775808 iTerm = 9223372036854775807 +** iOStep = -9223372036854775808 iStep = 9223372036854775808 bDesc = 0 */ +/* typedef unsigned char u8; */ typedef struct series_cursor series_cursor; struct series_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - SequenceSpec ss; /* (this) Derived class data */ + sqlite3_int64 iOBase; /* Original starting value ("start") */ + sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ + sqlite3_int64 iOStep; /* Original step value */ + sqlite3_int64 iBase; /* Starting value to actually use */ + sqlite3_int64 iTerm; /* Terminal value to actually use */ + sqlite3_uint64 iStep; /* The step size */ + sqlite3_int64 iValue; /* Current value */ + u8 bDesc; /* iStep is really negative */ + u8 bDone; /* True if stepped past last element */ }; +/* +** Computed the difference between two 64-bit signed integers using a +** convoluted computation designed to work around the silly restriction +** against signed integer overflow in C. +*/ +static sqlite3_uint64 span64(sqlite3_int64 a, sqlite3_int64 b){ + assert( a>=b ); + return (*(sqlite3_uint64*)&a) - (*(sqlite3_uint64*)&b); +} + +/* +** Add or substract an unsigned 64-bit integer from a signed 64-bit integer +** and return the new signed 64-bit integer. +*/ +static sqlite3_int64 add64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x += b; + return *(sqlite3_int64*)&x; +} +static sqlite3_int64 sub64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x -= b; + return *(sqlite3_int64*)&x; +} + /* ** The seriesConnect() method is invoked to create a new ** series_vtab that describes the generate_series virtual table. @@ -6512,7 +5879,15 @@ static int seriesClose(sqlite3_vtab_cursor *cur){ */ static int seriesNext(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - progressSequence( & pCur->ss ); + if( pCur->iValue==pCur->iTerm ){ + pCur->bDone = 1; + }else if( pCur->bDesc ){ + pCur->iValue = sub64(pCur->iValue, pCur->iStep); + assert( pCur->iValue>=pCur->iTerm ); + }else{ + pCur->iValue = add64(pCur->iValue, pCur->iStep); + assert( pCur->iValue<=pCur->iTerm ); + } return SQLITE_OK; } @@ -6528,19 +5903,19 @@ static int seriesColumn( series_cursor *pCur = (series_cursor*)cur; sqlite3_int64 x = 0; switch( i ){ - case SERIES_COLUMN_START: x = pCur->ss.iOBase; break; - case SERIES_COLUMN_STOP: x = pCur->ss.iOTerm; break; - case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; - default: x = pCur->ss.iValueNow; break; + case SERIES_COLUMN_START: x = pCur->iOBase; break; + case SERIES_COLUMN_STOP: x = pCur->iOTerm; break; + case SERIES_COLUMN_STEP: x = pCur->iOStep; break; + default: x = pCur->iValue; break; } sqlite3_result_int64(ctx, x); return SQLITE_OK; } #ifndef LARGEST_UINT64 -#define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -#define LARGEST_UINT64 (0xffffffff|(((sqlite3_uint64)0xffffffff)<<32)) -#define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#define LARGEST_INT64 ((sqlite3_int64)0x7fffffffffffffffLL) +#define LARGEST_UINT64 ((sqlite3_uint64)0xffffffffffffffffULL) +#define SMALLEST_INT64 ((sqlite3_int64)0x8000000000000000LL) #endif /* @@ -6548,7 +5923,7 @@ static int seriesColumn( */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; - *pRowid = pCur->ss.iValueNow; + *pRowid = pCur->iValue; return SQLITE_OK; } @@ -6558,7 +5933,7 @@ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int seriesEof(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - return !pCur->ss.isNotEOF; + return pCur->bDone; } /* True to cause run-time checking of the start=, stop=, and/or step= @@ -6569,6 +5944,59 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ # define SQLITE_SERIES_CONSTRAINT_VERIFY 0 #endif +/* +** Return the number of steps between pCur->iBase and pCur->iTerm if +** the step width is pCur->iStep. +*/ +static sqlite3_uint64 seriesSteps(series_cursor *pCur){ + if( pCur->bDesc ){ + assert( pCur->iBase >= pCur->iTerm ); + return span64(pCur->iBase, pCur->iTerm)/pCur->iStep; + }else{ + assert( pCur->iBase <= pCur->iTerm ); + return span64(pCur->iTerm, pCur->iBase)/pCur->iStep; + } +} + +#if defined(SQLITE_ENABLE_MATH_FUNCTIONS) || defined(_WIN32) +/* +** Case 1 (the most common case): +** The standard math library is available so use ceil() and floor() from there. +*/ +static double seriesCeil(double r){ return ceil(r); } +static double seriesFloor(double r){ return floor(r); } +#elif defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +/* +** Case 2 (2nd most common): Use GCC/Clang builtins +*/ +static double seriesCeil(double r){ return __builtin_ceil(r); } +static double seriesFloor(double r){ return __builtin_floor(r); } +#else +/* +** Case 3 (rarely happens): Use home-grown ceil() and floor() routines. +*/ +static double seriesCeil(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r>(double)x ) x++; + return (double)x; +} +static double seriesFloor(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r<(double)x ) x--; + return (double)x; +} +#endif + /* ** This method is called to "rewind" the series_cursor object back ** to the first row of output. This method is always called at least @@ -6602,33 +6030,42 @@ static int seriesFilter( int argc, sqlite3_value **argv ){ series_cursor *pCur = (series_cursor *)pVtabCursor; - int i = 0; - int returnNoRows = 0; - sqlite3_int64 iMin = SMALLEST_INT64; - sqlite3_int64 iMax = LARGEST_INT64; - sqlite3_int64 iLimit = 0; - sqlite3_int64 iOffset = 0; + int iArg = 0; /* Arguments used so far */ + int i; /* Loop counter */ + sqlite3_int64 iMin = SMALLEST_INT64; /* Smallest allowed output value */ + sqlite3_int64 iMax = LARGEST_INT64; /* Largest allowed output value */ + sqlite3_int64 iLimit = 0; /* if >0, the value of the LIMIT */ + sqlite3_int64 iOffset = 0; /* if >0, the value of the OFFSET */ (void)idxStrUnused; + + /* If any constraints have a NULL value, then return no rows. + ** See ticket https://sqlite.org/src/info/fac496b61722daf2 + */ + for(i=0; i