From 35bad4c170ba1acc18d26ebef6bcbccc9317955e Mon Sep 17 00:00:00 2001 From: automated-signal <37887102+automated-signal@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:53:58 -0500 Subject: [PATCH] Storage Service: A few correctness fixes Co-authored-by: Scott Nonnenberg --- DATABASE_SCHEMA.md | 21 +- pnpm-lock.yaml | 8 +- .../CallLinkFinalizeDeleteManager.preload.ts | 206 ------- ts/jobs/callLinkRefreshJobQueue.preload.ts | 14 +- ts/jobs/initializeAllJobQueues.preload.ts | 25 +- ts/services/backups/export.preload.ts | 16 + .../callLinkCleanupService.preload.ts | 36 ++ .../defunctCallLinkCleanupService.preload.ts | 39 ++ ts/services/storage.preload.ts | 263 +++++---- ts/services/storageRecordOps.preload.ts | 162 ++++-- ts/sql/Interface.std.ts | 28 +- ts/sql/Server.node.ts | 54 +- ...leted-fields-for-defunct-call-links.std.ts | 48 ++ ts/sql/migrations/index.node.ts | 3 + ts/sql/server/callLinks.node.ts | 198 ++++--- ts/sql/server/chatFolders.std.ts | 9 + ts/state/ducks/calling.preload.ts | 15 +- ts/state/ducks/conversations.preload.ts | 5 +- .../ducks/notificationProfiles.preload.ts | 26 +- .../ducks/storyDistributionLists.preload.ts | 36 +- .../state/ducks/conversations_test.preload.ts | 10 +- ts/test-mock/storage/delete_test.node.ts | 501 ++++++++++++++++++ ts/test-node/sql/migration_1800_test.node.ts | 113 ++++ ts/textsecure/MessageReceiver.preload.ts | 1 + ts/textsecure/messageReceiverEvents.std.ts | 1 + ts/types/CallLink.std.ts | 3 + ts/util/callDisposition.preload.ts | 12 +- ts/util/callLinksRingrtc.node.ts | 2 + ts/util/onCallLinkUpdateSync.preload.ts | 37 +- 29 files changed, 1355 insertions(+), 537 deletions(-) delete mode 100644 ts/jobs/CallLinkFinalizeDeleteManager.preload.ts create mode 100644 ts/services/expiring/callLinkCleanupService.preload.ts create mode 100644 ts/services/expiring/defunctCallLinkCleanupService.preload.ts create mode 100644 ts/sql/migrations/1800-deleted-fields-for-defunct-call-links.std.ts create mode 100644 ts/test-mock/storage/delete_test.node.ts create mode 100644 ts/test-node/sql/migration_1800_test.node.ts diff --git a/DATABASE_SCHEMA.md b/DATABASE_SCHEMA.md index 7e639723ad..2dd70b2c3d 100644 --- a/DATABASE_SCHEMA.md +++ b/DATABASE_SCHEMA.md @@ -348,6 +348,15 @@ CREATE INDEX callLinks_deleted ON callLinks (deleted, roomId) +
+Index: callLinks → callLinks_expiring + +```sql +CREATE INDEX callLinks_expiring ON callLinks (deletedAt) +``` + +
+
Index: callLinks → sqlite_autoindex_callLinks_1 @@ -601,10 +610,20 @@ CREATE TABLE defunctCallLinks ( storageID TEXT, storageVersion INTEGER, storageUnknownFields BLOB, - storageNeedsSync INTEGER NOT NULL DEFAULT 0 + storageNeedsSync INTEGER NOT NULL DEFAULT 0, + addedAt INTEGER NOT NULL ) STRICT ``` +
+Index: defunctCallLinks → defunctCallLinks_expiring + +```sql +CREATE INDEX defunctCallLinks_expiring ON defunctCallLinks (addedAt) +``` + +
+
Index: defunctCallLinks → sqlite_autoindex_defunctCallLinks_1 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0a6313b05f..fc025e435f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7393,8 +7393,8 @@ packages: fast-string-width@3.0.2: resolution: {integrity: sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==, tarball: https://registry.npmjs.org/fast-string-width/-/fast-string-width-3.0.2.tgz} - fast-uri@3.1.5: - resolution: {integrity: sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==, tarball: https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz} + fast-uri@3.1.6: + resolution: {integrity: sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==, tarball: https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz} fast-wrap-ansi@0.2.2: resolution: {integrity: sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==, tarball: https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.2.tgz} @@ -16838,7 +16838,7 @@ snapshots: ajv@8.18.0: dependencies: fast-deep-equal: 3.1.3 - fast-uri: 3.1.5 + fast-uri: 3.1.6 json-schema-traverse: 1.0.0 require-from-string: 2.0.2 @@ -18825,7 +18825,7 @@ snapshots: dependencies: fast-string-truncated-width: 3.0.3 - fast-uri@3.1.5: {} + fast-uri@3.1.6: {} fast-wrap-ansi@0.2.2: dependencies: diff --git a/ts/jobs/CallLinkFinalizeDeleteManager.preload.ts b/ts/jobs/CallLinkFinalizeDeleteManager.preload.ts deleted file mode 100644 index c6e98cda53..0000000000 --- a/ts/jobs/CallLinkFinalizeDeleteManager.preload.ts +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2024 Signal Messenger, LLC -// SPDX-License-Identifier: AGPL-3.0-only -import * as durations from '../util/durations/index.std.ts'; -import { createLogger } from '../logging/log.std.ts'; -import { DataReader, DataWriter } from '../sql/Client.preload.ts'; -import { - JobManager, - type JobManagerParamsType, - type JobManagerJobResultType, - type JobManagerJobType, -} from './JobManager.std.ts'; - -const log = createLogger('CallLinkFinalizeDeleteManager'); - -export type CoreCallLinkDeleteJobType = { - roomId: string; -}; - -export type CallLinkDeleteJobType = CoreCallLinkDeleteJobType & - JobManagerJobType; - -const MAX_CONCURRENT_JOBS = 5; - -const DEFAULT_RETRY_CONFIG = { - maxAttempts: 10, - backoffConfig: { - // 1 min, 5 min, 25 min, (max) 1 day - multiplier: 5, - firstBackoffs: [durations.MINUTE], - maxBackoffTime: durations.DAY, - }, -}; - -type CallLinkFinalizeDeleteManagerParamsType = - JobManagerParamsType; - -function getJobId(job: CoreCallLinkDeleteJobType): string { - return job.roomId; -} - -// The purpose of this job is to finalize local DB delete of call links and -// associated call history, after we confirm storage sync. -// It does *not* delete the call link from the server -- this should be done -// synchronously and prior to running this job, so we can show confirmation -// or error to the user. -export class CallLinkFinalizeDeleteManager extends JobManager { - jobs = new Map(); - private static _instance: CallLinkFinalizeDeleteManager | undefined; - override logPrefix = 'CallLinkFinalizeDeleteManager'; - - static defaultParams: CallLinkFinalizeDeleteManagerParamsType = { - markAllJobsInactive: () => Promise.resolve(), - getNextJobs, - saveJob, - removeJob, - runJob, - getJobId, - getJobIdForLogging: getJobId, - getRetryConfig: () => DEFAULT_RETRY_CONFIG, - maxConcurrentJobs: MAX_CONCURRENT_JOBS, - }; - - constructor(params: CallLinkFinalizeDeleteManagerParamsType) { - super({ - ...params, - getNextJobs: ({ limit, timestamp }) => - params.getNextJobs.call(this, { limit, timestamp }), - saveJob: (job: CallLinkDeleteJobType) => params.saveJob.call(this, job), - removeJob: (job: CallLinkDeleteJobType) => - params.removeJob.call(this, job), - }); - } - - override async addJob( - jobData: CoreCallLinkDeleteJobType, - options?: { delay: number } - ): Promise { - const { delay } = options || {}; - if (delay) { - log.info( - `CallLinkDeleteJobType/addJob/${getJobId(jobData)}: Adding with delay ${delay}` - ); - const job: CallLinkDeleteJobType = { - ...jobData, - attempts: 0, - retryAfter: Date.now() + delay, - lastAttemptTimestamp: null, - active: false, - }; - await this.params.saveJob(job); - return; - } - - await this._addJob(jobData); - } - - async enqueueAllDeletedCallLinks(options?: { delay: number }): Promise { - const roomIds = await DataReader.getAllMarkedDeletedCallLinkRoomIds(); - log.info( - `CallLinkDeleteJobType/enqueueAllDeletedCallLinks: Found ${roomIds.length} call links to delete` - ); - roomIds.forEach(roomId => this.addJob({ roomId }, options)); - } - - static get instance(): CallLinkFinalizeDeleteManager { - if (!CallLinkFinalizeDeleteManager._instance) { - CallLinkFinalizeDeleteManager._instance = - new CallLinkFinalizeDeleteManager( - CallLinkFinalizeDeleteManager.defaultParams - ); - } - return CallLinkFinalizeDeleteManager._instance; - } - - static async start(): Promise { - await CallLinkFinalizeDeleteManager.instance.enqueueAllDeletedCallLinks(); - await CallLinkFinalizeDeleteManager.instance.start(); - } - - static async stop(): Promise { - return CallLinkFinalizeDeleteManager._instance?.stop(); - } - - static async addJob( - newJob: CoreCallLinkDeleteJobType, - options?: { delay: number } - ): Promise { - return CallLinkFinalizeDeleteManager.instance.addJob(newJob, options); - } - - static async enqueueAllDeletedCallLinks(options?: { - delay: number; - }): Promise { - return CallLinkFinalizeDeleteManager.instance.enqueueAllDeletedCallLinks( - options - ); - } -} - -async function getNextJobs( - this: CallLinkFinalizeDeleteManager, - { - limit, - timestamp, - }: { - limit: number; - timestamp: number; - } -): Promise> { - let countRemaining = limit; - const nextJobs: Array = []; - for (const job of this.jobs.values()) { - if (job.active || (job.retryAfter && job.retryAfter > timestamp)) { - continue; - } - - nextJobs.push(job); - countRemaining -= 1; - if (countRemaining <= 0) { - break; - } - } - return nextJobs; -} - -async function saveJob( - this: CallLinkFinalizeDeleteManager, - job: CallLinkDeleteJobType -): Promise { - const { roomId } = job; - this.jobs.set(roomId, job); -} - -async function removeJob( - this: CallLinkFinalizeDeleteManager, - job: CallLinkDeleteJobType -): Promise { - this.jobs.delete(job.roomId); -} - -async function runJob( - job: CallLinkDeleteJobType, - _options: { isLastAttempt: boolean; abortSignal: AbortSignal } -): Promise> { - const logId = `CallLinkDeleteJobType/runJob/${getJobId(job)}`; - - const callLinkRecord = await DataReader.getCallLinkRecordByRoomId(job.roomId); - if (callLinkRecord == null) { - log.warn(`${logId}: Call link gone from DB`); - return { status: 'finished' }; - } - if (callLinkRecord.deleted !== 1) { - log.error(`${logId}: Call link not marked deleted. Giving up.`); - return { status: 'finished' }; - } - - // For consistency between devices, wait for storage sync - if (callLinkRecord.storageNeedsSync !== 0) { - log.info(`${logId}: Call link storage needs sync; retrying later`); - return { status: 'retry' }; - } - - await DataWriter.finalizeDeleteCallLink(job.roomId); - log.info(`${logId}: Finalized local delete`); - return { status: 'finished' }; -} diff --git a/ts/jobs/callLinkRefreshJobQueue.preload.ts b/ts/jobs/callLinkRefreshJobQueue.preload.ts index d7adce3e2d..53e737fe77 100644 --- a/ts/jobs/callLinkRefreshJobQueue.preload.ts +++ b/ts/jobs/callLinkRefreshJobQueue.preload.ts @@ -24,6 +24,8 @@ import { parseUnknown } from '../util/schemas.std.ts'; import { getRoomIdFromRootKey } from '../util/callLinksRingrtc.node.ts'; import { toCallHistoryFromUnusedCallLink } from '../util/callLinks.std.ts'; import type { StorageServiceFieldsType } from '../sql/Interface.std.ts'; +import { defunctCallLinkCleanupService } from '../services/expiring/defunctCallLinkCleanupService.preload.ts'; +import { drop } from '../util/drop.std.ts'; const globalLogger = createLogger('callLinkRefreshJobQueue'); @@ -137,10 +139,10 @@ class CallLinkRefreshJobQueue extends JobQueue { } protected getPendingCallLinkStorageFields( - storageID: string, + rootKey: string, jobData: CallLinkRefreshJobData ): StorageServiceFieldsType | undefined { - const storageFields = this.#pendingCallLinks.get(storageID); + const storageFields = this.#pendingCallLinks.get(rootKey); if (storageFields) { return { storageID: storageFields.storageID, @@ -240,9 +242,15 @@ class CallLinkRefreshJobQueue extends JobQueue { roomId, rootKey, adminKey: data.adminKey ?? null, + addedAt: Date.now(), ...storageFields, - storageNeedsSync: false, + storageNeedsSync: true, }); + drop( + defunctCallLinkCleanupService.trigger( + 'callLinkRefreshJobQueue: added defunct call link' + ) + ); } else { log.info( `${logId}: Call link not found on server but present locally, ignoring` diff --git a/ts/jobs/initializeAllJobQueues.preload.ts b/ts/jobs/initializeAllJobQueues.preload.ts index 32537f8d1c..672f1ab17f 100644 --- a/ts/jobs/initializeAllJobQueues.preload.ts +++ b/ts/jobs/initializeAllJobQueues.preload.ts @@ -3,20 +3,21 @@ import type { reportMessage, isOnline } from '../textsecure/WebAPI.preload.ts'; import { drop } from '../util/drop.std.ts'; -import { CallLinkFinalizeDeleteManager } from './CallLinkFinalizeDeleteManager.preload.ts'; -import { chatFolderCleanupService } from '../services/expiring/chatFolderCleanupService.preload.ts'; -import { pinnedMessagesCleanupService } from '../services/expiring/pinnedMessagesCleanupService.preload.ts'; -import { callLinkRefreshJobQueue } from './callLinkRefreshJobQueue.preload.ts'; import { conversationJobQueue } from './conversationJobQueue.preload.ts'; -import { deleteDownloadsJobQueue } from './deleteDownloadsJobQueue.preload.ts'; import { groupAvatarJobQueue } from './groupAvatarJobQueue.preload.ts'; +import { singleProtoJobQueue } from './singleProtoJobQueue.preload.ts'; import { readSyncJobQueue } from './readSyncJobQueue.preload.ts'; +import { viewSyncJobQueue } from './viewSyncJobQueue.preload.ts'; +import { viewOnceOpenJobQueue } from './viewOnceOpenJobQueue.preload.ts'; +import { deleteDownloadsJobQueue } from './deleteDownloadsJobQueue.preload.ts'; +import { registrationJobQueue } from './registrationJobQueue.preload.ts'; import { removeStorageKeyJobQueue } from './removeStorageKeyJobQueue.preload.ts'; import { reportSpamJobQueue } from './reportSpamJobQueue.preload.ts'; -import { singleProtoJobQueue } from './singleProtoJobQueue.preload.ts'; -import { viewOnceOpenJobQueue } from './viewOnceOpenJobQueue.preload.ts'; -import { viewSyncJobQueue } from './viewSyncJobQueue.preload.ts'; -import { registrationJobQueue } from './registrationJobQueue.preload.ts'; +import { callLinkRefreshJobQueue } from './callLinkRefreshJobQueue.preload.ts'; +import { callLinkCleanupService } from '../services/expiring/callLinkCleanupService.preload.ts'; +import { defunctCallLinkCleanupService } from '../services/expiring/defunctCallLinkCleanupService.preload.ts'; +import { chatFolderCleanupService } from '../services/expiring/chatFolderCleanupService.preload.ts'; +import { pinnedMessagesCleanupService } from '../services/expiring/pinnedMessagesCleanupService.preload.ts'; type ServerType = { reportMessage: typeof reportMessage; @@ -53,7 +54,8 @@ export function initializeAllJobQueues({ drop(removeStorageKeyJobQueue.streamJobs()); drop(reportSpamJobQueue.streamJobs()); drop(callLinkRefreshJobQueue.streamJobs()); - drop(CallLinkFinalizeDeleteManager.start()); + drop(callLinkCleanupService.start('initializeAllJobQueues')); + drop(defunctCallLinkCleanupService.start('initializeAllJobQueues')); drop(chatFolderCleanupService.start('initializeAllJobQueues')); drop(pinnedMessagesCleanupService.start('initializeAllJobQueues')); } @@ -71,7 +73,8 @@ export async function shutdownAllJobQueues(): Promise { removeStorageKeyJobQueue.shutdown(), reportSpamJobQueue.shutdown(), callLinkRefreshJobQueue.shutdown(), - CallLinkFinalizeDeleteManager.stop(), + callLinkCleanupService.stop('shutdownAllJobQueues'), + defunctCallLinkCleanupService.stop('shutdownAllJobQueues'), chatFolderCleanupService.stop('shutdownAllJobQueues'), pinnedMessagesCleanupService.stop('shutdownAllJobQueues'), ]); diff --git a/ts/services/backups/export.preload.ts b/ts/services/backups/export.preload.ts index 362dcccafa..73a7156dea 100644 --- a/ts/services/backups/export.preload.ts +++ b/ts/services/backups/export.preload.ts @@ -760,6 +760,10 @@ export class BackupExportStream extends Readable { const allNotificationProfiles = await DataReader.getAllNotificationProfiles(); + const isNotificationProfileSyncDisabled = itemStorage.get( + 'notificationProfileSyncDisabled', + false + ); for (const profile of allNotificationProfiles) { const { @@ -768,6 +772,7 @@ export class BackupExportStream extends Readable { emoji = null, color, createdAtMs, + deletedAtTimestampMs, allowAllCalls, allowAllMentions, allowedMembers, @@ -775,8 +780,19 @@ export class BackupExportStream extends Readable { scheduleStartTime = null, scheduleEndTime = null, scheduleDaysEnabled, + storageID, } = profile; + // Skipping deleted profile + if (isNumber(deletedAtTimestampMs) && deletedAtTimestampMs > 0) { + continue; + } + + // sync=OFF, and so only exporting profiles with storageID (from Primary) + if (isNotificationProfileSyncDisabled && !storageID) { + continue; + } + const allowedRecipients = Array.from(allowedMembers) .map(conversationId => this.#getRecipientByConversationId( diff --git a/ts/services/expiring/callLinkCleanupService.preload.ts b/ts/services/expiring/callLinkCleanupService.preload.ts new file mode 100644 index 0000000000..031225c044 --- /dev/null +++ b/ts/services/expiring/callLinkCleanupService.preload.ts @@ -0,0 +1,36 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { DataReader, DataWriter } from '../../sql/Client.preload.ts'; +import { getMessageQueueTime } from '../../util/getMessageQueueTime.dom.ts'; +import { createExpiringEntityCleanupService } from './createExpiringEntityCleanupService.std.ts'; +import * as RemoteConfig from '../../RemoteConfig.dom.ts'; + +export const callLinkCleanupService = createExpiringEntityCleanupService({ + logPrefix: 'CallLink', + getNextExpiringEntity: async () => { + const item = await DataReader.getTimestampOfOldestDeletedCallLink(); + if (!item) { + return null; + } + + const messageQueueTime = getMessageQueueTime(); + const expiresAtMs = item.deletedAt + messageQueueTime; + return { id: item.roomId, expiresAtMs }; + }, + cleanupExpiredEntities: async () => { + const messageQueueTime = getMessageQueueTime(); + const ids = await DataWriter.deleteExpiredCallLinks(messageQueueTime); + return ids; + }, + subscribeToTriggers: trigger => { + let prevMessageQueueTime = getMessageQueueTime(); + return RemoteConfig.onChange(['global.messageQueueTimeInSeconds'], () => { + const messageQueueTime = getMessageQueueTime(); + if (messageQueueTime !== prevMessageQueueTime) { + trigger('messageQueueTime changed'); + } + prevMessageQueueTime = getMessageQueueTime(); + }); + }, +}); diff --git a/ts/services/expiring/defunctCallLinkCleanupService.preload.ts b/ts/services/expiring/defunctCallLinkCleanupService.preload.ts new file mode 100644 index 0000000000..b1c0641a62 --- /dev/null +++ b/ts/services/expiring/defunctCallLinkCleanupService.preload.ts @@ -0,0 +1,39 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { DataReader, DataWriter } from '../../sql/Client.preload.ts'; +import { getMessageQueueTime } from '../../util/getMessageQueueTime.dom.ts'; +import { createExpiringEntityCleanupService } from './createExpiringEntityCleanupService.std.ts'; +import * as RemoteConfig from '../../RemoteConfig.dom.ts'; + +export const defunctCallLinkCleanupService = createExpiringEntityCleanupService( + { + logPrefix: 'DefunctCallLink', + getNextExpiringEntity: async () => { + const item = await DataReader.getTimestampOfOldestDefunctCallLink(); + if (!item) { + return null; + } + + const messageQueueTime = getMessageQueueTime(); + const expiresAtMs = item.addedAt + messageQueueTime; + return { id: item.roomId, expiresAtMs }; + }, + cleanupExpiredEntities: async () => { + const messageQueueTime = getMessageQueueTime(); + const ids = + await DataWriter.deleteExpiredDefunctCallLinks(messageQueueTime); + return ids; + }, + subscribeToTriggers: trigger => { + let prevMessageQueueTime = getMessageQueueTime(); + return RemoteConfig.onChange(['global.messageQueueTimeInSeconds'], () => { + const messageQueueTime = getMessageQueueTime(); + if (messageQueueTime !== prevMessageQueueTime) { + trigger('messageQueueTime changed'); + } + prevMessageQueueTime = getMessageQueueTime(); + }); + }, + } +); diff --git a/ts/services/storage.preload.ts b/ts/services/storage.preload.ts index 45af2c61d5..4836b7490b 100644 --- a/ts/services/storage.preload.ts +++ b/ts/services/storage.preload.ts @@ -31,8 +31,9 @@ import { toStickerPackRecord, toCallLinkRecord, mergeCallLinkRecord, - toDefunctOrPendingCallLinkRecord, toChatFolderRecord, + toDefunctCallLinkRecord, + toPendingCallLinkRecord, mergeChatFolderRecord, mergeNotificationProfileRecord, } from './storageRecordOps.preload.ts'; @@ -201,7 +202,8 @@ async function generateManifest( previousManifest?: Proto.ManifestRecord, isNewManifest = false ): Promise { - log.info(`upload(${version}): generating manifest new=${isNewManifest}`); + const logId = `generateManifest(${version})`; + log.info(`${logId}: generating manifest new=${isNewManifest}`); await window.ConversationController.checkForConflicts(); @@ -254,8 +256,7 @@ async function generateManifest( const newRedactedID = redactStorageID(storageID, version, conversation); if (currentStorageID) { log.info( - `upload(${version}): ` + - `updating from=${currentRedactedID} ` + + `${logId}: updating from=${currentRedactedID} ` + `to=${newRedactedID}` ); deleteKeys.add(currentStorageID); @@ -364,11 +365,7 @@ async function generateManifest( conversation ); - log.warn( - `generateManifest(${version}): ` + - `dropping contact=${recordID} ` + - `due to ${dropReason}` - ); + log.warn(`${logId}: dropping contact=${recordID} due to ${dropReason}`); conversation.set({ storageID: undefined }); deleteKeys.add(droppedID); continue; @@ -386,8 +383,7 @@ async function generateManifest( // first before syncing it to storage service. if (conversation.get('needsGroupUpdate') === true) { log.warn( - `upload(${version}): ` + - `dropping group=${conversation.idForLogging()} until it is updated` + `${logId}: dropping group=${conversation.idForLogging()} until it is updated` ); continue; } @@ -406,10 +402,7 @@ async function generateManifest( }; identifierType = ITEM_TYPE.GROUPV1; } else { - log.warn( - `upload(${version}): ` + - `unknown conversation=${conversation.idForLogging()}` - ); + log.warn(`${logId}: unknown conversation=${conversation.idForLogging()}`); } if (!storageRecord || !identifierType) { @@ -449,8 +442,7 @@ async function generateManifest( } = await getNonConversationRecords(); log.info( - `upload(${version}): ` + - `adding storyDistributionLists=${storyDistributionLists.length}` + `${logId}: adding storyDistributionLists=${storyDistributionLists.length}` ); for (const storyDistributionList of storyDistributionLists) { @@ -478,8 +470,7 @@ async function generateManifest( const recordID = redactStorageID(droppedID, droppedVersion); log.warn( - `generateManifest(${version}): ` + - `dropping storyDistributionList=${recordID} ` + + `${logId}: dropping storyDistributionList=${recordID} ` + `due to expired deleted timestamp=${storyDistributionList.deletedAtTimestamp}` ); deleteKeys.add(droppedID); @@ -515,13 +506,11 @@ async function generateManifest( const localOnlyCount = notificationProfiles.length - notificationProfilesToUpload.length; log.info( - `upload(${version}): ` + - `sync=OFF; adding notificationProfiles=${notificationProfilesToUpload.length}, excluding ${localOnlyCount} local profiles` + `${logId}: sync=OFF; adding notificationProfiles=${notificationProfilesToUpload.length}, excluding ${localOnlyCount} local profiles` ); } else { log.info( - `upload(${version}): ` + - `sync=ON, adding notificationProfiles=${notificationProfilesToUpload.length}` + `${logId}: sync=ON, adding notificationProfiles=${notificationProfilesToUpload.length}` ); } for (const notificationProfile of notificationProfilesToUpload) { @@ -548,8 +537,7 @@ async function generateManifest( const recordID = redactStorageID(droppedID, droppedVersion); log.info( - `generateManifest(${version}): ` + - `dropping notificationProfile=${recordID} ` + + `${logId}: dropping notificationProfile=${recordID} ` + `due to expired deleted timestamp=${notificationProfile.deletedAtTimestampMs}` ); deleteKeys.add(droppedID); @@ -604,15 +592,13 @@ async function generateManifest( const recordID = redactStorageID(droppedID, droppedVersion); log.info( - `generateManifest(${version}): ` + - `dropping stickerPack=${recordID} ` + + `${logId}: dropping stickerPack=${recordID} ` + `due to expired deleted timestamp=${stickerPack.uninstalledAt}` ); deleteKeys.add(droppedID); } else { log.info( - `generateManifest(${version}): ` + - `dropping never uploaded stickerPack=${stickerPack.id}` + + `${logId}: dropping never uploaded stickerPack=${stickerPack.id}` + `due to expired deleted timestamp=${stickerPack.uninstalledAt}` ); } @@ -689,7 +675,7 @@ async function generateManifest( }); log.info( - `upload(${version}): stickerPacks ` + + `${logId}: stickerPacks ` + `installed=${newlyInstalledPacks}/${installedStickerPacks.length} ` + `uninstalled=${newlyUninstalledPacks}/${uninstalledStickerPacks.length}` ); @@ -701,7 +687,7 @@ async function generateManifest( for (const callLinkDbRecord of callLinkDbRecords) { const { roomId } = callLinkDbRecord; if (callLinkDbRecord.adminKey == null || callLinkDbRecord.rootKey == null) { - log.warn(`upload(${version}): call link ${roomId} has empty rootKey`); + log.warn(`${logId}: call link ${roomId} has empty rootKey`); continue; } @@ -733,8 +719,7 @@ async function generateManifest( const freshCallLink = await DataReader.getCallLinkByRoomId(roomId); if (freshCallLink == null) { log.warn( - `upload(${version}): ` + - `call link ${roomId} removed locally from DB while we were uploading to storage` + `${logId}: call link ${roomId} removed locally from DB while we were uploading to storage` ); return; } @@ -746,15 +731,12 @@ async function generateManifest( } } - log.info( - `upload(${version}): ` + - `adding defunctCallLinks=${defunctCallLinks.length}` - ); + log.info(`${logId}: adding defunctCallLinks=${defunctCallLinks.length}`); defunctCallLinks.forEach(defunctCallLink => { const storageRecord: Proto.StorageRecord.Params = { record: { - callLink: toDefunctOrPendingCallLinkRecord(defunctCallLink), + callLink: toDefunctCallLinkRecord(defunctCallLink), }, }; @@ -782,15 +764,12 @@ async function generateManifest( } }); - log.info( - `upload(${version}): ` + - `adding pendingCallLinks=${pendingCallLinks.length}` - ); + log.info(`${logId}: adding pendingCallLinks=${pendingCallLinks.length}`); pendingCallLinks.forEach(pendingCallLink => { const storageRecord: Proto.StorageRecord.Params = { record: { - callLink: toDefunctOrPendingCallLinkRecord(pendingCallLink), + callLink: toPendingCallLinkRecord(pendingCallLink), }, }; @@ -822,7 +801,7 @@ async function generateManifest( } }); - log.info(`upload(${version}): adding chatFolders=${chatFolders.length}`); + log.info(`${logId}: adding chatFolders=${chatFolders.length}`); chatFolders.forEach(chatFolder => { const { isNewItem, storageID } = processStorageRecord({ @@ -858,7 +837,7 @@ async function generateManifest( const redactedUnknowns = unknownRecordsArray.map(redactExtendedStorageID); log.info( - `upload(${version}): adding unknown ` + + `${logId}: adding unknown ` + `records=${JSON.stringify(redactedUnknowns)} ` + `count=${redactedUnknowns.length}` ); @@ -876,7 +855,7 @@ async function generateManifest( const redactedErrors = recordsWithErrors.map(redactExtendedStorageID); log.info( - `upload(${version}): adding error ` + + `${logId}: adding error ` + `records=${JSON.stringify(redactedErrors)} count=${redactedErrors.length}` ); @@ -895,8 +874,7 @@ async function generateManifest( redactExtendedStorageID ); log.info( - `upload(${version}): ` + - `deleting extra keys=${JSON.stringify(redactedPendingDeletes)} ` + + `${logId}: deleting extra keys=${JSON.stringify(redactedPendingDeletes)} ` + `count=${redactedPendingDeletes.length}` ); @@ -917,8 +895,7 @@ async function generateManifest( const typeAndID = `${itemType}+${storageID}`; if (duplicates.has(storageID) || typeDuplicates.has(typeAndID)) { log.warn( - `upload(${version}): removing from duplicate item ` + - 'from the manifest', + `${logId}: removing from duplicate item from the manifest`, redactStorageID(storageID), itemType ); @@ -931,7 +908,7 @@ async function generateManifest( const hasDeleteKey = deleteKeys.has(storageID); if (hasDeleteKey) { log.warn( - `upload(${version}): removing key which has been deleted`, + `${logId}: removing key which has been deleted`, redactStorageID(storageID), itemType ); @@ -942,7 +919,7 @@ async function generateManifest( if (itemType === ITEM_TYPE.ACCOUNT) { if (hasAccountType) { log.warn( - `upload(${version}): removing duplicate account`, + `${logId}: removing duplicate account`, redactStorageID(storageID) ); recordsByID.delete(storageID); @@ -960,7 +937,7 @@ async function generateManifest( // Ensure there are no duplicate StorageIdentifiers in your list of inserts if (storageKeyDuplicates.has(storageID)) { log.warn( - `upload(${version}): removing duplicate identifier from inserts`, + `${logId}: removing duplicate identifier from inserts`, redactStorageID(storageID) ); insertKeys.delete(storageID); @@ -981,7 +958,7 @@ async function generateManifest( const remoteKeys = new Set(); (previousManifest.identifiers ?? []).forEach( (identifier: IManifestRecordIdentifier) => { - strictAssert(identifier.raw, 'Identifier without raw field'); + strictAssert(identifier.raw, `${logId}: Identifier without raw field`); const storageID = Bytes.toBase64(identifier.raw); remoteKeys.add(storageID); } @@ -1021,28 +998,32 @@ async function generateManifest( return redactStorageID(id); }); log.error( - `upload(${version}): delete key sizes do not match`, + `${logId}: delete key sizes do not match`, 'local', localDeletes.join(','), 'remote', remoteDeletes.join(',') ); - throw new Error('invalid write delete keys length do not match'); + throw new Error( + `${logId}: invalid write delete keys length do not match` + ); } if (insertKeys.size !== pendingInserts.size) { - throw new Error('invalid write insert items length do not match'); + throw new Error( + `${logId}: invalid write insert items length do not match` + ); } for (const storageID of deleteKeys) { if (!pendingDeletes.has(storageID)) { throw new Error( - 'invalid write delete key missing from pending deletes' + `${logId}: invalid write delete key missing from pending deletes` ); } } for (const storageID of insertKeys) { if (!pendingInserts.has(storageID)) { throw new Error( - 'invalid write insert key missing from pending inserts' + `${logId}: invalid write insert key missing from pending inserts` ); } } @@ -1628,13 +1609,13 @@ async function mergeRecord( type NonConversationRecordsResultType = Readonly<{ callLinkDbRecords: ReadonlyArray; + chatFolders: ReadonlyArray; defunctCallLinks: ReadonlyArray; + installedStickerPacks: ReadonlyArray; notificationProfiles: ReadonlyArray; pendingCallLinks: ReadonlyArray; - installedStickerPacks: ReadonlyArray; - uninstalledStickerPacks: ReadonlyArray; storyDistributionLists: ReadonlyArray; - chatFolders: ReadonlyArray; + uninstalledStickerPacks: ReadonlyArray; }>; // TODO: DESKTOP-3929 @@ -1649,8 +1630,8 @@ async function getNonConversationRecords(): Promise { + const logId = `processManifest/${version}`; const remoteKeysTypeMap = new Map(); (manifest.identifiers || []).forEach( ({ raw, type }: IManifestRecordIdentifier) => { - strictAssert(raw, 'Identifier without raw field'); + strictAssert(raw, `${logId}: Identifier without raw field`); remoteKeysTypeMap.set(Bytes.toBase64(raw), type); } ); @@ -1780,18 +1762,16 @@ async function processManifest( ); log.info( - `process(${version}): localRecords=${localRecordCount} ` + + `${logId}: localRecords=${localRecordCount} ` + `localKeys=${localVersions.size} unknownKeys=${stillUnknown.length} ` + `remoteKeys=${remoteKeys.size}` ); log.info( - `process(${version}): ` + - `remoteOnlyCount=${remoteOnlySet.size} ` + + `${logId}: remoteOnlyCount=${remoteOnlySet.size} ` + `remoteOnlyKeys=${JSON.stringify(redactedRemoteOnly)}` ); log.info( - `process(${version}): ` + - `localOnlyCount=${localOnlySet.size} ` + + `${logId}: localOnlyCount=${localOnlySet.size} ` + `localOnlyKeys=${JSON.stringify(redactedLocalOnly)}` ); @@ -1835,8 +1815,7 @@ async function processManifest( conversation.isUnregistered() ) { log.info( - `process(${version}): localKey=${missingKey} is ` + - 'unregistered and not in remote manifest' + `${logId}/conversation: localKey=${missingKey} is unregistered and not in remote manifest` ); conversation.setUnregistered({ timestamp: Date.now() - getMessageQueueTime(), @@ -1847,8 +1826,7 @@ async function processManifest( }); } else { log.info( - `process(${version}): localKey=${missingKey} ` + - 'was not in remote manifest' + `${logId}/conversation: localKey=${missingKey} was not in remote manifest` ); } conversation.set({ storageID: undefined, storageVersion: undefined }); @@ -1861,30 +1839,26 @@ async function processManifest( { const { callLinkDbRecords, + chatFolders, defunctCallLinks, + installedStickerPacks, + notificationProfiles, pendingCallLinks, storyDistributionLists, - installedStickerPacks, uninstalledStickerPacks, - chatFolders, } = await getNonConversationRecords(); uninstalledStickerPacks.forEach(stickerPack => { - const { storageID, storageVersion } = stickerPack; + const { id, storageID, storageVersion } = stickerPack; if (!storageID || remoteKeys.has(storageID)) { return; } const missingKey = redactStorageID(storageID, storageVersion); log.info( - `process(${version}): localKey=${missingKey} was not ` + - 'in remote manifest' + `${logId}/uninstalledStickerPack: localKey=${missingKey} was not in remote manifest. Removing.` ); - void DataWriter.addUninstalledStickerPack({ - ...stickerPack, - storageID: undefined, - storageVersion: undefined, - }); + drop(DataWriter.removeUninstalledStickerPack(id)); }); installedStickerPacks.forEach(stickerPack => { @@ -1895,8 +1869,7 @@ async function processManifest( const missingKey = redactStorageID(storageID, storageVersion); log.info( - `process(${version}): localKey=${missingKey} was not ` + - 'in remote manifest' + `${logId}/installedStickerPack: localKey=${missingKey} was not in remote manifest` ); void DataWriter.updateStickerPackInfo({ id: stickerPack.id, @@ -1911,15 +1884,26 @@ async function processManifest( }); storyDistributionLists.forEach(storyDistributionList => { - const { storageID, storageVersion } = storyDistributionList; + const { id, deletedAtTimestamp, storageID, storageVersion } = + storyDistributionList; if (!storageID || remoteKeys.has(storageID)) { return; } const missingKey = redactStorageID(storageID, storageVersion); + if (deletedAtTimestamp) { + log.info( + `${logId}/storyDistributionList: localKey=${missingKey} was not in remote manifest, but deleted locally. Removing.` + ); + drop(DataWriter.deleteStoryDistribution(id)); + window.reduxActions.storyDistributionLists.distributionListWasDeleted( + id + ); + return; + } + log.info( - `process(${version}): localKey=${missingKey} was not ` + - 'in remote manifest' + `${logId}/storyDistributionList: localKey=${missingKey} was not in remote manifest` ); void DataWriter.modifyStoryDistribution({ ...storyDistributionList, @@ -1934,7 +1918,7 @@ async function processManifest( ); if (!myStories) { - log.info(`process(${version}): creating my stories`); + log.info(`${logId}: creating my stories`); const storyDistribution: StoryDistributionWithMembersType = { allowsReplies: true, id: MY_STORY_ID, @@ -1957,18 +1941,22 @@ async function processManifest( } callLinkDbRecords.forEach(callLinkDbRecord => { - const { storageID, storageVersion } = callLinkDbRecord; + const { deletedAt, roomId, storageID, storageVersion } = callLinkDbRecord; if (!storageID || remoteKeys.has(storageID)) { return; } - const missingKey = redactStorageID( - storageID, - storageVersion || undefined - ); + const missingKey = redactStorageID(storageID, storageVersion); + if (deletedAt) { + log.info( + `${logId}/callLinkDbRecord: localKey=${missingKey} was not in remote manifest, but deleted locally. Removing.` + ); + drop(DataWriter.deleteCallLink(roomId)); + return; + } + log.info( - `process(${version}): localKey=${missingKey} was not ` + - 'in remote manifest' + `${logId}/callLinkDbRecord: localKey=${missingKey} was not in remote manifest` ); const callLink = callLinkFromRecord(callLinkDbRecord); drop( @@ -1981,23 +1969,16 @@ async function processManifest( }); defunctCallLinks.forEach(defunctCallLink => { - const { storageID, storageVersion } = defunctCallLink; + const { roomId, storageID, storageVersion } = defunctCallLink; if (!storageID || remoteKeys.has(storageID)) { return; } const missingKey = redactStorageID(storageID, storageVersion); log.info( - `process(${version}): localKey=${missingKey} was not ` + - 'in remote manifest' - ); - drop( - DataWriter.updateDefunctCallLink({ - ...defunctCallLink, - storageID: undefined, - storageVersion: undefined, - }) + `${logId}/defunctCallLink: localKey=${missingKey} was not in remote manifest. Removing.` ); + drop(DataWriter.deleteDefunctCallLink(roomId)); }); pendingCallLinks.forEach(pendingCallLink => { @@ -2008,8 +1989,7 @@ async function processManifest( const missingKey = redactStorageID(storageID, storageVersion); log.info( - `process(${version}): localKey=${missingKey} was not ` + - 'in remote manifest' + `${logId}/pendingCallLink: localKey=${missingKey} was not in remote manifest` ); callLinkRefreshJobQueue.updatePendingCallLinkStorageFields( pendingCallLink.rootKey, @@ -2022,15 +2002,24 @@ async function processManifest( }); chatFolders.forEach(chatFolder => { - const { storageID, storageVersion } = chatFolder; + const { deletedAtTimestampMs, id, storageID, storageVersion } = + chatFolder; if (!storageID || remoteKeys.has(storageID)) { return; } const missingKey = redactStorageID(storageID, storageVersion); + if (deletedAtTimestampMs) { + log.info( + `${logId}/chatFolder: localKey=${missingKey} was not in remote manifest, but deleted locally. Removing.` + ); + drop(DataWriter.deleteChatFolderById(id)); + window.reduxActions.chatFolders.refetchChatFolders(); + return; + } + log.info( - `process(${version}): localKey=${missingKey} was not ` + - 'in remote manifest' + `${logId}/chatFolder: localKey=${missingKey} was not in remote manifest` ); void DataWriter.updateChatFolder({ @@ -2047,12 +2036,54 @@ async function processManifest( }); if (!hasCurrentAllChatFolder) { - log.info(`process(${version}): creating all chats chat folder`); + log.info(`${logId}: creating all chats chat folder`); window.reduxActions.chatFolders.createAllChatsChatFolder(); } + + const isNotificationProfileSyncDisabled = itemStorage.get( + 'notificationProfileSyncDisabled', + false + ); + notificationProfiles.forEach(notificationProfile => { + const { deletedAtTimestampMs, id, storageID, storageVersion } = + notificationProfile; + if (!storageID || remoteKeys.has(storageID)) { + return; + } + + const missingKey = redactStorageID(storageID, storageVersion); + + if (deletedAtTimestampMs) { + log.info( + `${logId}/notificationProfile: localKey=${missingKey} was not in remote manifest, but deleted locally. Removing.` + ); + drop(DataWriter.deleteNotificationProfileById(id)); + window.reduxActions.notificationProfiles.profileWasRemoved(id); + return; + } + if (isNotificationProfileSyncDisabled) { + log.info( + `${logId}/notificationProfile: localKey=${missingKey} was not in remote manifest, but sync=OFF. Removing.` + ); + drop(DataWriter.deleteNotificationProfileById(id)); + window.reduxActions.notificationProfiles.profileWasRemoved(id); + return; + } + + log.info( + `${logId}/notificationProfile: localKey=${missingKey} was not in remote manifest` + ); + const update = { + ...notificationProfile, + storageID: undefined, + storageVersion: undefined, + }; + drop(DataWriter.updateNotificationProfile(update)); + window.reduxActions.notificationProfiles.profileWasUpdated(update); + }); } - log.info(`process(${version}): done`); + log.info(`${logId}: done`); } export type FetchRemoteRecordsResultType = Readonly<{ diff --git a/ts/services/storageRecordOps.preload.ts b/ts/services/storageRecordOps.preload.ts index 11393a5478..40c15d0739 100644 --- a/ts/services/storageRecordOps.preload.ts +++ b/ts/services/storageRecordOps.preload.ts @@ -138,6 +138,8 @@ import { isKnownProtoEnumMember } from '../util/isKnownProtoEnumMember.std.ts'; import { Emoji } from '../axo/emoji.std.ts'; import { getOurAddress } from '../util/sendToGroup.preload.ts'; import { QualifiedAddress } from '../types/QualifiedAddress.std.ts'; +import { callLinkCleanupService } from './expiring/callLinkCleanupService.preload.ts'; +import { defunctCallLinkCleanupService } from './expiring/defunctCallLinkCleanupService.preload.ts'; const { isEqual } = lodash; @@ -842,19 +844,16 @@ export function toCallLinkRecord( }; } -export function toDefunctOrPendingCallLinkRecord( - callLink: DefunctCallLinkType | PendingCallLinkType +export function toPendingCallLinkRecord( + callLink: PendingCallLinkType ): Proto.CallLinkRecord.Params { const rootKey = toRootKeyBytes(callLink.rootKey); const adminPasskey = callLink.adminKey ? toAdminKeyBytes(callLink.adminKey) : null; - strictAssert(rootKey, 'toDefunctOrPendingCallLinkRecord: no rootKey'); - strictAssert( - adminPasskey, - 'toDefunctOrPendingCallLinkRecord: no adminPasskey' - ); + strictAssert(rootKey, 'toPendingCallLinkRecord: no rootKey'); + strictAssert(adminPasskey, 'toPendingCallLinkRecord: no adminPasskey'); return { rootKey, @@ -864,6 +863,22 @@ export function toDefunctOrPendingCallLinkRecord( }; } +export function toDefunctCallLinkRecord( + callLink: DefunctCallLinkType +): Proto.CallLinkRecord.Params { + const rootKey = toRootKeyBytes(callLink.rootKey); + const deletedAtTimestampMs = BigInt(callLink.addedAt); + + strictAssert(rootKey, 'toDefunctCallLinkRecord: no rootKey'); + + return { + rootKey, + adminPasskey: null, + deletedAtTimestampMs, + $unknown: fromStorageUnknownFields(callLink.storageUnknownFields), + }; +} + function toRecipient( conversationId: string, logPrefix: string @@ -2530,6 +2545,19 @@ export async function mergeStickerPackRecord( }; } +function getEarliestTimestamp( + ...timestamps: Array +): number | undefined { + const normalized = timestamps.filter( + (timestamp): timestamp is number => timestamp != null && timestamp > 0 + ); + if (normalized.length === 0) { + return undefined; + } + + return Math.min(...normalized); +} + export async function mergeCallLinkRecord( storageID: string, storageVersion: number, @@ -2554,6 +2582,7 @@ export async function mergeCallLinkRecord( const localCallLinkDbRecord = await DataReader.getCallLinkRecordByRoomId(roomId); + const defunctCallLink = await DataReader.getDefunctCallLinkByRoomId(roomId); const details = logRecordChanges( localCallLinkDbRecord == null @@ -2562,9 +2591,12 @@ export async function mergeCallLinkRecord( callLinkRecord ); - // Note deletedAtTimestampMs can be 0 - const deletedAtTimestampMs = toNumber(callLinkRecord.deletedAtTimestampMs); - const deletedAt = deletedAtTimestampMs || null; + const deletedAt = + getEarliestTimestamp( + toNumber(callLinkRecord.deletedAtTimestampMs), + localCallLinkDbRecord?.deletedAt, + defunctCallLink?.addedAt + ) || null; const shouldDrop = Boolean( deletedAt && isOlderThan(deletedAt, getMessageQueueTime()) ); @@ -2593,12 +2625,39 @@ export async function mergeCallLinkRecord( if (!localCallLinkDbRecord) { if (deletedAt) { - details.push( - `skipping deleted call link with no matching local record deletedAt=${deletedAt}` - ); - } else if (await DataReader.defunctCallLinkExists(roomId)) { - details.push('skipping known defunct call link'); - } else if (callLinkRefreshJobQueue.hasPendingCallLink(storageID)) { + if (defunctCallLink) { + details.push( + `updating known defunct call link; deletedAt=${deletedAt}` + ); + await DataWriter.updateDefunctCallLink({ + roomId, + rootKey: rootKeyString, + adminKey: null, + addedAt: deletedAt, + storageID, + storageVersion, + storageUnknownFields: callLinkDbRecord.storageUnknownFields, + storageNeedsSync: false, + }); + } else { + details.push( + `creating defunct call link, given no matching local record; deletedAt=${deletedAt}` + ); + await DataWriter.insertDefunctCallLink({ + roomId, + rootKey: rootKeyString, + adminKey: null, + addedAt: deletedAt, + storageID, + storageVersion, + storageUnknownFields: callLinkDbRecord.storageUnknownFields, + storageNeedsSync: false, + }); + drop( + defunctCallLinkCleanupService.trigger('just added defunct call link') + ); + } + } else if (callLinkRefreshJobQueue.hasPendingCallLink(rootKeyString)) { details.push('pending call link refresh, updating storage fields'); callLinkRefreshJobQueue.updatePendingCallLinkStorageFields( rootKeyString, @@ -2658,11 +2717,12 @@ export async function mergeCallLinkRecord( // Deleted in storage but we have it locally: Delete locally too and update redux if (deletedAt && localCallLinkDbRecord.deleted !== 1) { // Another device deleted the link and uploaded to storage, and we learned about it - log.info(`${logId}: Discovered deleted call link, deleting locally`); - details.push('deleting locally'); + log.info(`${logId}: Discovered deleted call link, marking deleted locally`); + details.push('marking deleted locally'); // No need to delete via RingRTC as we assume the originating device did that already - await DataWriter.deleteCallLinkAndHistory(roomId); + await DataWriter.markCallLinkDeleted(roomId, deletedAt); window.reduxActions.calling.handleCallLinkDelete({ roomId }); + drop(callLinkCleanupService.trigger('marked call link deleted')); } else if (!deletedAt && localCallLinkDbRecord.deleted === 1) { // Not deleted in storage, but we've marked it as deleted locally. // Skip doing anything, we will update things locally after sync. @@ -2761,6 +2821,14 @@ export async function mergeChatFolderRecord( const idString = bytesToUuid(remoteChatFolderRecord.id) as ChatFolderId; const logPrefix = `mergeChatFolderRecord(${redactedStorageID}, idString)`; + const localChatFolder = await DataReader.getChatFolder(idString); + + const localDeletedAt = localChatFolder?.deletedAtTimestampMs ?? 0; + const deletedAtTimestampMs: number = + getEarliestTimestamp( + toNumber(remoteChatFolderRecord.deletedAtTimestampMs), + localChatFolder?.deletedAtTimestampMs + ) || 0; const remoteChatFolder: ChatFolder = { id: idString, @@ -2783,8 +2851,8 @@ export async function mergeChatFolderRecord( remoteChatFolderRecord.excludedRecipients ?? [], logPrefix ), - deletedAtTimestampMs: - toNumber(remoteChatFolderRecord.deletedAtTimestampMs) ?? 0, + deletedAtTimestampMs, + storageID, storageVersion, storageUnknownFields: @@ -2794,34 +2862,19 @@ export async function mergeChatFolderRecord( storageNeedsSync: false, }; - const localChatFolder = await DataReader.getChatFolder(remoteChatFolder.id); - - let deletedAtTimestampMs: number; - - const remoteDeletedAt = remoteChatFolder.deletedAtTimestampMs; - const localDeletedAt = localChatFolder?.deletedAtTimestampMs ?? 0; - - if (remoteDeletedAt > 0 && localDeletedAt > 0) { - if (remoteDeletedAt < localDeletedAt) { - deletedAtTimestampMs = remoteDeletedAt; - } else { - deletedAtTimestampMs = localDeletedAt; - } - } else if (remoteDeletedAt > 0) { - deletedAtTimestampMs = remoteDeletedAt; - } else if (localDeletedAt > 0) { - deletedAtTimestampMs = localDeletedAt; - } else { - deletedAtTimestampMs = remoteDeletedAt; - } - if (remoteChatFolder.folderType === ChatFolderType.ALL) { log.info(`${logPrefix}: Updating or inserting all chats folder`); await DataWriter.upsertAllChatsChatFolderFromSync(remoteChatFolder); } else if (deletedAtTimestampMs > 0) { if (localChatFolder == null) { log.info( - `${logPrefix}: skipping deleted chat folder, no local record found` + `${logPrefix}: creating deleted chat folder with deletedAtTimestampMs=${deletedAtTimestampMs}` + ); + await DataWriter.createChatFolder(remoteChatFolder); + drop( + chatFolderCleanupService.trigger( + 'mergeChatFolderRecord: created deleted chat folder' + ) ); } else if (localDeletedAt === deletedAtTimestampMs) { log.info( @@ -2905,8 +2958,9 @@ export function prepareForDisabledNotificationProfileSync(): { const notDeletedProfiles = profiles.filter( profile => - (profile.storageID && profile.deletedAtTimestampMs == null) || - profile.deletedAtTimestampMs === 0 + profile.storageID && + (profile.deletedAtTimestampMs == null || + profile.deletedAtTimestampMs === 0) ); const toAdd: Array = []; @@ -3116,9 +3170,7 @@ export async function mergeNotificationProfileRecord( : Proto.NotificationProfile.DayOfWeek.UNKNOWN; }) ), - deletedAtTimestampMs: localDeletedAt - ? Math.min(localDeletedAt, deletedAt ?? Number.MAX_SAFE_INTEGER) - : dropNull(deletedAt), + deletedAtTimestampMs: getEarliestTimestamp(localDeletedAt, deletedAt), storageID, storageVersion, storageUnknownFields: @@ -3130,15 +3182,11 @@ export async function mergeNotificationProfileRecord( window.reduxActions.notificationProfiles; if (!localProfile) { - if (deletedAt) { - details.push( - `skipping deleted notification profile with no matching local record deletedAt=${deletedAt}` - ); - } else { - details.push('created new notification profile'); - await DataWriter.createNotificationProfile(newProfile); - profileWasCreated(newProfile); - } + details.push( + `created new notification profile; deletedAtTimestampMs=${newProfile.deletedAtTimestampMs}` + ); + await DataWriter.createNotificationProfile(newProfile); + profileWasCreated(newProfile); return { details, diff --git a/ts/sql/Interface.std.ts b/ts/sql/Interface.std.ts index 596c509811..ad68d7d3dd 100644 --- a/ts/sql/Interface.std.ts +++ b/ts/sql/Interface.std.ts @@ -1022,14 +1022,21 @@ type ReadableInterface = { receivedAt: number ) => string | null; callLinkExists: (roomId: string) => boolean; - defunctCallLinkExists: (roomId: string) => boolean; getAllCallLinks: () => ReadonlyArray; getCallLinkByRoomId: (roomId: string) => CallLinkType | undefined; getCallLinkRecordByRoomId: (roomId: string) => CallLinkRecord | undefined; + getDefunctCallLinkByRoomId: ( + roomId: string + ) => DefunctCallLinkType | undefined; getAllAdminCallLinks: () => ReadonlyArray; - getAllCallLinkRecordsWithAdminKey: () => ReadonlyArray; - getAllDefunctCallLinksWithAdminKey: () => ReadonlyArray; - getAllMarkedDeletedCallLinkRoomIds: () => ReadonlyArray; + getAllCallLinkRecordsForStorageService: () => ReadonlyArray; + getAllDefunctCallLinksForStorageService: () => ReadonlyArray; + getTimestampOfOldestDefunctCallLink: () => + | { roomId: string; addedAt: number } + | undefined; + getTimestampOfOldestDeletedCallLink(): + | { roomId: string; deletedAt: number } + | undefined; getMessagesBetween: ( conversationId: string, options: GetMessagesBetweenOptions @@ -1319,15 +1326,19 @@ type WritableInterface = { roomId: string, callLinkState: CallLinkStateType ) => CallLinkType; - beginDeleteAllCallLinks: () => boolean; - beginDeleteCallLink: (roomId: string) => boolean; + markAllCallLinksDeleted: () => boolean; + markCallLinkDeleted: (roomId: string, deletedAt: number) => boolean; + deleteCallLink: (roomId: string) => boolean; + deleteDefunctCallLink: (roomId: string) => boolean; deleteCallHistoryByRoomId: (roomid: string) => void; deleteCallLinkAndHistory: (roomId: string) => void; - finalizeDeleteCallLink: (roomId: string) => void; + deleteExpiredDefunctCallLinks( + messageQueueTime: number + ): ReadonlyArray; + deleteExpiredCallLinks(messageQueueTime: number): ReadonlyArray; _removeAllCallLinks: () => void; insertDefunctCallLink: (defunctCallLink: DefunctCallLinkType) => void; updateDefunctCallLink: (defunctCallLink: DefunctCallLinkType) => void; - deleteCallLinkFromSync: (roomId: string) => void; migrateConversationMessages: (obsoleteId: string, currentId: string) => void; saveEditedMessage: ( mainMessage: ReadonlyDeep, @@ -1498,6 +1509,7 @@ type WritableInterface = { deleteExpiredChatFolders: ( messageQueueTime: number ) => ReadonlyArray; + deleteChatFolderById: (id: ChatFolderId) => void; createMegaphone: (megaphone: RemoteMegaphoneType) => void; updateMegaphone: (megaphone: RemoteMegaphoneType) => void; diff --git a/ts/sql/Server.node.ts b/ts/sql/Server.node.ts index 13daf6a5cf..83637c304d 100644 --- a/ts/sql/Server.node.ts +++ b/ts/sql/Server.node.ts @@ -213,24 +213,27 @@ import { } from './Interface.std.ts'; import { _removeAllCallLinks, - beginDeleteAllCallLinks, - beginDeleteCallLink, callLinkExists, - defunctCallLinkExists, deleteCallHistoryByRoomId, deleteCallLinkAndHistory, - deleteCallLinkFromSync, - finalizeDeleteCallLink, + deleteCallLink, + deleteDefunctCallLink, + deleteExpiredCallLinks, + deleteExpiredDefunctCallLinks, getAllAdminCallLinks, - getAllCallLinkRecordsWithAdminKey, + getAllCallLinkRecordsForStorageService, getAllCallLinks, - getAllDefunctCallLinksWithAdminKey, - getAllMarkedDeletedCallLinkRoomIds, + getAllDefunctCallLinksForStorageService, getCallLinkByRoomId, getCallLinkRecordByRoomId, + getDefunctCallLinkByRoomId, + getTimestampOfOldestDefunctCallLink, + getTimestampOfOldestDeletedCallLink, insertCallLink, insertDefunctCallLink, insertOrUpdateCallLinkFromSync, + markAllCallLinksDeleted, + markCallLinkDeleted, updateCallLink, updateCallLinkState, updateDefunctCallLink, @@ -264,6 +267,7 @@ import { updateChatFolderPositions, updateChatFolderDeletedAtTimestampMsFromSync, deleteExpiredChatFolders, + deleteChatFolderById, } from './server/chatFolders.std.ts'; import { getAllPinnedMessages, @@ -529,14 +533,16 @@ export const DataReader: ServerReadableInterface = { getNextExpiringPinnedMessageAcrossConversations, callLinkExists, - defunctCallLinkExists, getAllCallLinks, getCallLinkByRoomId, getCallLinkRecordByRoomId, + getDefunctCallLinkByRoomId, getAllAdminCallLinks, - getAllCallLinkRecordsWithAdminKey, - getAllDefunctCallLinksWithAdminKey, - getAllMarkedDeletedCallLinkRoomIds, + getAllCallLinkRecordsForStorageService, + getAllDefunctCallLinksForStorageService, + getTimestampOfOldestDefunctCallLink, + getTimestampOfOldestDeletedCallLink, + getMessagesBetween, getNearbyMessageFromDeletedSet, getMostRecentAddressableMessages, @@ -685,15 +691,22 @@ export const DataWriter: ServerWritableInterface = { insertOrUpdateCallLinkFromSync, updateCallLink, updateCallLinkState, - beginDeleteAllCallLinks, - beginDeleteCallLink, + markAllCallLinksDeleted, + markCallLinkDeleted, + deleteCallLink, + deleteDefunctCallLink, + deleteCallHistoryByRoomId, deleteCallLinkAndHistory, - finalizeDeleteCallLink, + _removeAllCallLinks, - deleteCallLinkFromSync, + insertDefunctCallLink, updateDefunctCallLink, + + deleteExpiredCallLinks, + deleteExpiredDefunctCallLinks, + migrateConversationMessages, saveEditedMessage, saveEditedMessages, @@ -791,6 +804,7 @@ export const DataWriter: ServerWritableInterface = { updateChatFolderDeletedAtTimestampMsFromSync, markChatFolderDeleted, deleteExpiredChatFolders, + deleteChatFolderById, createMegaphone, updateMegaphone, @@ -8585,6 +8599,14 @@ function markNotificationProfileDeleted( ): number | undefined { const now = new Date().getTime(); + const existing = getNotificationProfileById(db, id); + if (existing && existing.deletedAtTimestampMs) { + logger.warn( + `markNotificationProfileDeleted: Notification profile ${id} already had deletedAtTimestampMs set` + ); + return existing.deletedAtTimestampMs; + } + const [query, parameters] = sql` UPDATE notificationProfiles SET diff --git a/ts/sql/migrations/1800-deleted-fields-for-defunct-call-links.std.ts b/ts/sql/migrations/1800-deleted-fields-for-defunct-call-links.std.ts new file mode 100644 index 0000000000..f4d6c20a1c --- /dev/null +++ b/ts/sql/migrations/1800-deleted-fields-for-defunct-call-links.std.ts @@ -0,0 +1,48 @@ +// Copyright 2026 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { sql } from '../util.std.ts'; + +import type { WritableDB } from '../Interface.std.ts'; +import type { LoggerType } from '../../types/Logging.std.ts'; + +export default function updateToSchemaVersion1800( + db: WritableDB, + logger: LoggerType +): void { + db.exec(` + ALTER TABLE defunctCallLinks + ADD COLUMN addedAt INTEGER; + `); + + const [deleteQuery] = sql` + DELETE FROM defunctCallLinks + WHERE adminKey IS NULL AND storageID IS NULL; + `; + const deleteResult = db.prepare(deleteQuery).run(); + logger.info( + `updateToSchemaVersion1800: Deleted ${deleteResult.changes} defunct call links` + ); + + const now = Date.now(); + const [updateQuery, updateParams] = sql` + UPDATE defunctCallLinks + SET addedAt = ${now}, storageNeedsSync = 1 + `; + const updateResult = db.prepare(updateQuery).run(updateParams); + logger.info( + `updateToSchemaVersion1800: Updated ${updateResult.changes} defunct call links` + ); + + db.exec(` + ALTER TABLE defunctCallLinks + ALTER COLUMN addedAt SET NOT NULL; + `); + + db.exec(` + CREATE INDEX defunctCallLinks_expiring ON defunctCallLinks(addedAt); + `); + db.exec(` + CREATE INDEX callLinks_expiring ON callLinks(deletedAt); + `); +} diff --git a/ts/sql/migrations/index.node.ts b/ts/sql/migrations/index.node.ts index c02f82597d..306ff34520 100644 --- a/ts/sql/migrations/index.node.ts +++ b/ts/sql/migrations/index.node.ts @@ -156,6 +156,7 @@ import updateToSchemaVersion1760 from './1760-delete-story-reply-attachment.std. import updateToSchemaVersion1770 from './1770-add-blocked-at.std.ts'; import updateToSchemaVersion1780 from './1780-fts-reindex.std.ts'; import updateToSchemaVersion1790 from './1790-notify-for-mentions-if-muted.std.ts'; +import updateToSchemaVersion1800 from './1800-deleted-fields-for-defunct-call-links.std.ts'; import { DataWriter } from '../Server.node.ts'; import { strictAssert } from '../../util/assert.std.ts'; @@ -1676,6 +1677,8 @@ export const SCHEMA_VERSIONS: ReadonlyArray = [ { version: 1770, update: updateToSchemaVersion1770 }, { version: 1780, update: updateToSchemaVersion1780 }, { version: 1790, update: updateToSchemaVersion1790 }, + + { version: 1800, update: updateToSchemaVersion1800 }, ]; class DBVersionFromFutureError extends Error { diff --git a/ts/sql/server/callLinks.node.ts b/ts/sql/server/callLinks.node.ts index edf2585e22..ee073a2bf8 100644 --- a/ts/sql/server/callLinks.node.ts +++ b/ts/sql/server/callLinks.node.ts @@ -253,33 +253,22 @@ export function deleteCallHistoryByRoomId( ); } -// This should only be called from a sync message to avoid accidentally deleting -// on the client but not the server -export function deleteCallLinkFromSync(db: WritableDB, roomId: string): void { - db.transaction(() => { - const [query, params] = sql` - DELETE FROM callLinks - WHERE roomId = ${roomId}; - `; - - db.prepare(query).run(params); - - deleteCallHistoryByRoomId(db, roomId); - })(); -} - /** * Deletes a non-admin call link from the local database, or if it's an admin call link, * then marks it for deletion and storage sync. * * @returns boolean: True if storage sync is needed; False if not */ -export function beginDeleteCallLink(db: WritableDB, roomId: string): boolean { +export function markCallLinkDeleted( + db: WritableDB, + roomId: string, + deletedAt: number +): boolean { return db.transaction(() => { // If adminKey is null, then we should delete the call link const [deleteNonAdminCallLinksQuery, deleteNonAdminCallLinksParams] = sql` DELETE FROM callLinks - WHERE adminKey IS NULL + WHERE (adminKey IS NULL AND storageID IS NULL) AND roomId = ${roomId}; `; @@ -293,8 +282,6 @@ export function beginDeleteCallLink(db: WritableDB, roomId: string): boolean { return false; } - const deletedAt = new Date().getTime(); - // If the admin key is not null, we should mark it for deletion const [markAdminCallLinksDeletedQuery, markAdminCallLinksDeletedParams] = sql` @@ -303,7 +290,7 @@ export function beginDeleteCallLink(db: WritableDB, roomId: string): boolean { deleted = 1, deletedAt = ${deletedAt}, storageNeedsSync = 1 - WHERE adminKey IS NOT NULL + WHERE (adminKey IS NOT NULL OR storageID IS NOT NULL) AND deleted IS NOT 1 AND roomId = ${roomId}; `; @@ -340,27 +327,28 @@ export function deleteCallLinkAndHistory(db: WritableDB, roomId: string): void { * * @returns boolean: True if storage sync is needed; False if not */ -export function beginDeleteAllCallLinks(db: WritableDB): boolean { +export function markAllCallLinksDeleted(db: WritableDB): boolean { const deletedAt = new Date().getTime(); return db.transaction(() => { const [markAdminCallLinksDeletedQuery, markAdminCallLinksDeletedParams] = sql` - UPDATE callLinks - SET - deleted = 1, - deletedAt = ${deletedAt}, - storageNeedsSync = 1 - WHERE adminKey IS NOT NULL - AND deleted IS NOT 1; - `; + UPDATE callLinks + SET + deleted = 1, + deletedAt = ${deletedAt}, + storageNeedsSync = 1 + WHERE (adminKey IS NOT NULL OR storageID IS NOT NULL) + AND deleted IS NOT 1; + `; const markAdminCallLinksDeletedResult = db .prepare(markAdminCallLinksDeletedQuery) .run(markAdminCallLinksDeletedParams); + // We can delete these immediately because they were never synced to Storage Service const [deleteNonAdminCallLinksQuery] = sql` DELETE FROM callLinks - WHERE adminKey IS NULL; + WHERE (adminKey IS NULL AND storageID IS NULL); `; db.prepare(deleteNonAdminCallLinksQuery).run(); @@ -371,12 +359,13 @@ export function beginDeleteAllCallLinks(db: WritableDB): boolean { } // When you need to access the deleted field -export function getAllCallLinkRecordsWithAdminKey( +export function getAllCallLinkRecordsForStorageService( db: ReadableDB ): ReadonlyArray { const [query] = sql` SELECT * FROM callLinks - WHERE adminKey IS NOT NULL + WHERE + (adminKey IS NOT NULL OR storageID IS NOT NULL) AND rootKey IS NOT NULL; `; return db @@ -388,35 +377,11 @@ export function getAllCallLinkRecordsWithAdminKey( export function getAllAdminCallLinks( db: ReadableDB ): ReadonlyArray { - return getAllCallLinkRecordsWithAdminKey(db).map((record: CallLinkRecord) => - callLinkFromRecord(record) + return getAllCallLinkRecordsForStorageService(db).map( + (record: CallLinkRecord) => callLinkFromRecord(record) ); } -export function getAllMarkedDeletedCallLinkRoomIds( - db: ReadableDB -): ReadonlyArray { - const [query] = sql` - SELECT roomId FROM callLinks WHERE deleted = 1; - `; - return db - .prepare(query, { - pluck: true, - }) - .all(); -} - -// TODO: Run this after uploading storage records, maybe periodically on startup -export function finalizeDeleteCallLink(db: WritableDB, roomId: string): void { - const [query, params] = sql` - DELETE FROM callLinks - WHERE roomId = ${roomId} - AND deleted = 1 - AND storageNeedsSync = 0; - `; - db.prepare(query).run(params); -} - export function _removeAllCallLinks(db: WritableDB): void { const [query, params] = sql` DELETE FROM callLinks; @@ -424,28 +389,13 @@ export function _removeAllCallLinks(db: WritableDB): void { db.prepare(query).run(params); } -export function defunctCallLinkExists(db: ReadableDB, roomId: string): boolean { - const [query, params] = sql` - SELECT 1 - FROM defunctCallLinks - WHERE roomId = ${roomId}; - `; - return ( - db - .prepare(query, { - pluck: true, - }) - .get(params) === 1 - ); -} - -export function getAllDefunctCallLinksWithAdminKey( +export function getAllDefunctCallLinksForStorageService( db: ReadableDB ): ReadonlyArray { const [query] = sql` SELECT * FROM defunctCallLinks - WHERE adminKey IS NOT NULL; + WHERE (adminKey IS NOT NULL OR storageID IS NOT NULL); `; return db .prepare(query) @@ -455,6 +405,26 @@ export function getAllDefunctCallLinksWithAdminKey( ); } +export function getDefunctCallLinkByRoomId( + db: ReadableDB, + roomId: string +): DefunctCallLinkType | undefined { + const [query, params] = sql` + SELECT * + FROM defunctCallLinks + WHERE roomId = ${roomId} + `; + + const item = db.prepare(query).get(params); + if (!item) { + return undefined; + } + + return defunctCallLinkFromRecord( + parseUnknown(defunctCallLinkRecordSchema, item as unknown) + ); +} + export function insertDefunctCallLink( db: WritableDB, defunctCallLink: DefunctCallLinkType @@ -469,6 +439,7 @@ export function insertDefunctCallLink( roomId, rootKey, adminKey, + addedAt, storageID, storageVersion, storageUnknownFields, @@ -477,6 +448,7 @@ export function insertDefunctCallLink( $roomId, $rootKey, $adminKey, + $addedAt, $storageID, $storageVersion, $storageUnknownFields, @@ -498,8 +470,9 @@ export function updateDefunctCallLink( // Do not write roomId or rootKey since they should never change db.prepare( ` - UPDATE callLinks + UPDATE defunctCallLinks SET + addedAt = $addedAt, storageID = $storageID, storageVersion = $storageVersion, storageUnknownFields = $storageUnknownFields, @@ -508,3 +481,76 @@ export function updateDefunctCallLink( ` ).run(data); } + +export function getTimestampOfOldestDefunctCallLink( + db: ReadableDB +): { roomId: string; addedAt: number } | undefined { + const [query, params] = sql` + SELECT roomId, addedAt FROM defunctCallLinks + ORDER BY addedAt ASC + LIMIT 1 + `; + return db.prepare(query).get(params); +} + +// Note: this should only be used in unusual situations; defunct call links will expire +// normally based on addedAt +export function deleteDefunctCallLink(db: WritableDB, roomId: string): boolean { + const [query, params] = sql` + DELETE FROM defunctCallLinks + WHERE roomId = ${roomId} + `; + const result = db.prepare(query).run(params); + + return result.changes > 0; +} + +export function deleteExpiredDefunctCallLinks( + db: WritableDB, + messageQueueTime: number +): ReadonlyArray { + const before = Date.now() - messageQueueTime; + const [query, params] = sql` + DELETE FROM defunctCallLinks + WHERE addedAt < ${before} + RETURNING roomId + `; + return db.prepare(query, { pluck: true }).all(params); +} + +export function getTimestampOfOldestDeletedCallLink( + db: ReadableDB +): { roomId: string; deletedAt: number } | undefined { + const [query, params] = sql` + SELECT roomId, deletedAt FROM callLinks + WHERE deletedAt > 0 + ORDER BY deletedAt ASC + LIMIT 1 + `; + return db.prepare(query).get(params); +} + +// Note: this should only be used in unusual situations; usually we want to mark deleted. +export function deleteCallLink(db: WritableDB, roomId: string): boolean { + const [query, params] = sql` + DELETE FROM callLinks + WHERE roomId = ${roomId} + `; + const result = db.prepare(query).run(params); + + return result.changes > 0; +} + +export function deleteExpiredCallLinks( + db: WritableDB, + messageQueueTime: number +): ReadonlyArray { + const before = Date.now() - messageQueueTime; + const [query, params] = sql` + DELETE FROM callLinks + WHERE deletedAt > 0 + AND deletedAt < ${before} + RETURNING roomId + `; + return db.prepare(query, { pluck: true }).all(params); +} diff --git a/ts/sql/server/chatFolders.std.ts b/ts/sql/server/chatFolders.std.ts index 8029e7420d..d8be60743e 100644 --- a/ts/sql/server/chatFolders.std.ts +++ b/ts/sql/server/chatFolders.std.ts @@ -387,3 +387,12 @@ export function deleteExpiredChatFolders( `; return db.prepare(query, { pluck: true }).all(params); } + +// Note: this should only be used in unusual situations; usually we want to mark deleted +export function deleteChatFolderById(db: WritableDB, id: ChatFolderId): void { + const [query, params] = sql` + DELETE FROM chatFolders + WHERE id = ${id} + `; + db.prepare(query).run(params); +} diff --git a/ts/state/ducks/calling.preload.ts b/ts/state/ducks/calling.preload.ts index 00b28a582c..796a657df5 100644 --- a/ts/state/ducks/calling.preload.ts +++ b/ts/state/ducks/calling.preload.ts @@ -123,7 +123,6 @@ import { getPresentingSource, } from '../selectors/calling.std.ts'; import { runStorageServiceUploadJob } from '../../services/storage.preload.ts'; -import { CallLinkFinalizeDeleteManager } from '../../jobs/CallLinkFinalizeDeleteManager.preload.ts'; import { callLinkRefreshJobQueue } from '../../jobs/callLinkRefreshJobQueue.preload.ts'; import { isOnline, @@ -135,6 +134,7 @@ import { noopAction, type NoopActionType } from './noop.std.ts'; import type { SignalService } from '../../protobuf/index.std.ts'; import { Emoji } from '../../axo/emoji.std.ts'; import type { ErrorModalDataProps } from '../../components/ErrorModal.dom.tsx'; +import { callLinkCleanupService } from '../../services/expiring/callLinkCleanupService.preload.ts'; const { omit } = lodash; @@ -2360,7 +2360,10 @@ function deleteCallLink( return; } - const isStorageSyncNeeded = await DataWriter.beginDeleteCallLink(roomId); + const isStorageSyncNeeded = await DataWriter.markCallLinkDeleted( + roomId, + Date.now() + ); if (isStorageSyncNeeded) { runStorageServiceUploadJob({ reason: 'deleteCallLink' }); } @@ -2368,13 +2371,7 @@ function deleteCallLink( if (isCallLinkAdmin(callLink)) { // This throws if call link is active or network is unavailable. await calling.deleteCallLink(callLink); - // Wait for storage service sync before finalizing delete. - drop( - CallLinkFinalizeDeleteManager.addJob( - { roomId: callLink.roomId }, - { delay: 10000 } - ) - ); + drop(callLinkCleanupService.trigger('deleted call link')); } await DataWriter.deleteCallHistoryByRoomId(callLink.roomId); diff --git a/ts/state/ducks/conversations.preload.ts b/ts/state/ducks/conversations.preload.ts index a36fec0ba3..488a310a7c 100644 --- a/ts/state/ducks/conversations.preload.ts +++ b/ts/state/ducks/conversations.preload.ts @@ -46,9 +46,10 @@ import { TOGGLE_DISCARD_DRAFT_DIALOG, } from './globalModals.preload.ts'; import { - MODIFY_LIST, DELETE_LIST, HIDE_MY_STORIES_FROM, + MARK_AS_DELETED, + MODIFY_LIST, VIEWERS_CHANGED, } from './storyDistributionLists.preload.ts'; import type { StoryDistributionListsActionType } from './storyDistributionLists.preload.ts'; @@ -6156,7 +6157,7 @@ export function reducer( verificationDataByConversation: nextVerificationData, }; } - if (action.type === DELETE_LIST) { + if (action.type === DELETE_LIST || action.type === MARK_AS_DELETED) { const { listId } = action.payload; const nextVerificationData = visitListsInVerificationData( diff --git a/ts/state/ducks/notificationProfiles.preload.ts b/ts/state/ducks/notificationProfiles.preload.ts index 77a7d87212..3c3a3de777 100644 --- a/ts/state/ducks/notificationProfiles.preload.ts +++ b/ts/state/ducks/notificationProfiles.preload.ts @@ -210,7 +210,7 @@ function markProfileDeleted( // If called based on a local change, this function is run before the storage service // upload. If called based on a storage service update, it is called at the end of // processing, as the AccountRecord is processed. All profiles have been processed at -// that point, and the override from AccountRecord has been processed as well. +// that point, and the override from AccountRecord is just about to be processed. function setIsSyncEnabled( enabled: boolean, { fromStorageService }: { fromStorageService: boolean } @@ -316,8 +316,15 @@ function setProfileOverride( const state = getState(); const currentOverride = getOverride(state); + const isNotificationProfileSyncEnabled = !itemStorage.get( + 'notificationProfileSyncDisabled', + false + ); + const me = window.ConversationController.getOurConversationOrThrow(); - me.captureChange(logId); + if (isNotificationProfileSyncEnabled) { + me.captureChange(logId); + } if (enabled) { if ( @@ -344,7 +351,9 @@ function setProfileOverride( payload: newOverride, }); fastUpdateProfileService(); - updateStorageService(logId); + if (isNotificationProfileSyncEnabled) { + updateStorageService(logId); + } return; } @@ -359,7 +368,9 @@ function setProfileOverride( payload: newOverride, }); fastUpdateProfileService(); - updateStorageService(logId); + if (isNotificationProfileSyncEnabled) { + updateStorageService(logId); + } }; } @@ -400,7 +411,12 @@ function updateOverride( payload, }); - if (!fromStorageService) { + const isNotificationProfileSyncEnabled = !itemStorage.get( + 'notificationProfileSyncDisabled', + false + ); + + if (!fromStorageService && isNotificationProfileSyncEnabled) { const me = window.ConversationController.getOurConversationOrThrow(); me.captureChange(logId); updateStorageService(logId); diff --git a/ts/state/ducks/storyDistributionLists.preload.ts b/ts/state/ducks/storyDistributionLists.preload.ts index 9c942b4b64..b0f2b0aabf 100644 --- a/ts/state/ducks/storyDistributionLists.preload.ts +++ b/ts/state/ducks/storyDistributionLists.preload.ts @@ -47,6 +47,7 @@ export type StoryDistributionListStateType = ReadonlyDeep<{ const ALLOW_REPLIES_CHANGED = 'storyDistributionLists/ALLOW_REPLIES_CHANGED'; const CREATE_LIST = 'storyDistributionLists/CREATE_LIST'; +export const MARK_AS_DELETED = 'storyDistributionLists/MARK_AS_DELETED'; export const DELETE_LIST = 'storyDistributionLists/DELETE_LIST'; export const HIDE_MY_STORIES_FROM = 'storyDistributionLists/HIDE_MY_STORIES_FROM'; @@ -67,11 +68,18 @@ type CreateListActionType = ReadonlyDeep<{ payload: StoryDistributionListDataType; }>; +type MarkAsDeletedActionType = ReadonlyDeep<{ + type: typeof MARK_AS_DELETED; + payload: { + listId: string; + deletedAtTimestamp: number; + }; +}>; + type DeleteListActionType = ReadonlyDeep<{ type: typeof DELETE_LIST; payload: { listId: string; - deletedAtTimestamp: number; }; }>; @@ -108,6 +116,7 @@ export type StoryDistributionListsActionType = ReadonlyDeep< | AllowRepliesChangedActionType | CreateListActionType | DeleteListActionType + | MarkAsDeletedActionType | HideMyStoriesFromActionType | ModifyListActionType | ResetMyStoriesActionType @@ -208,7 +217,7 @@ function createDistributionList( function deleteDistributionList( listId: string -): ThunkAction { +): ThunkAction { return async (dispatch, getState) => { const deletedAtTimestamp = Date.now(); @@ -257,7 +266,7 @@ function deleteDistributionList( runStorageServiceUploadJob({ reason: 'deleteDistributionList' }); dispatch({ - type: DELETE_LIST, + type: MARK_AS_DELETED, payload: { listId, deletedAtTimestamp, @@ -266,6 +275,15 @@ function deleteDistributionList( }; } +function distributionListWasDeleted(listId: string): DeleteListActionType { + return { + type: DELETE_LIST, + payload: { + listId, + }, + }; +} + function modifyDistributionList( distributionList: ModifyDistributionListType ): ModifyListActionType { @@ -515,6 +533,7 @@ export const actions = { allowsRepliesChanged, createDistributionList, deleteDistributionList, + distributionListWasDeleted, hideMyStoriesFrom, modifyDistributionList, removeMembersFromDistributionList, @@ -607,7 +626,7 @@ export function reducer( }; } - if (action.type === DELETE_LIST) { + if (action.type === MARK_AS_DELETED) { const distributionLists = replaceDistributionListData( state.distributionLists, action.payload.listId, @@ -621,6 +640,15 @@ export function reducer( return distributionLists ? { distributionLists } : state; } + if (action.type === DELETE_LIST) { + const { listId } = action.payload; + const distributionLists = state.distributionLists.filter( + item => item.id !== listId + ); + + return { distributionLists }; + } + if (action.type === HIDE_MY_STORIES_FROM) { const distributionLists = replaceDistributionListData( state.distributionLists, diff --git a/ts/test-electron/state/ducks/conversations_test.preload.ts b/ts/test-electron/state/ducks/conversations_test.preload.ts index 55f39f060a..ab2798ebfb 100644 --- a/ts/test-electron/state/ducks/conversations_test.preload.ts +++ b/ts/test-electron/state/ducks/conversations_test.preload.ts @@ -55,8 +55,8 @@ import type { ShowSendAnywayDialogActionType } from '../../../state/ducks/global import { SHOW_SEND_ANYWAY_DIALOG } from '../../../state/ducks/globalModals.preload.ts'; import type { StoryDistributionListsActionType } from '../../../state/ducks/storyDistributionLists.preload.ts'; import { - DELETE_LIST, HIDE_MY_STORIES_FROM, + MARK_AS_DELETED, MODIFY_LIST, VIEWERS_CHANGED, } from '../../../state/ducks/storyDistributionLists.preload.ts'; @@ -2376,7 +2376,7 @@ describe('both/state/ducks/conversations', () => { }); }); }); - describe('DELETE_LIST', () => { + describe('MARK_AS_DELETED', () => { const state: ConversationsStateType = { ...getEmptyState(), verificationDataByConversation: { @@ -2401,7 +2401,7 @@ describe('both/state/ducks/conversations', () => { it('eliminates deleted list entirely', async () => { const action: StoryDistributionListsActionType = { - type: DELETE_LIST, + type: MARK_AS_DELETED, payload: { deletedAtTimestamp: Date.now(), listId: LIST_ID_1, @@ -2443,7 +2443,7 @@ describe('both/state/ducks/conversations', () => { }; const action: StoryDistributionListsActionType = { - type: DELETE_LIST, + type: MARK_AS_DELETED, payload: { deletedAtTimestamp: Date.now(), listId: LIST_ID_1, @@ -2475,7 +2475,7 @@ describe('both/state/ducks/conversations', () => { }; const action: StoryDistributionListsActionType = { - type: DELETE_LIST, + type: MARK_AS_DELETED, payload: { deletedAtTimestamp: Date.now(), listId: LIST_ID_1, diff --git a/ts/test-mock/storage/delete_test.node.ts b/ts/test-mock/storage/delete_test.node.ts new file mode 100644 index 0000000000..7c0b983f7d --- /dev/null +++ b/ts/test-mock/storage/delete_test.node.ts @@ -0,0 +1,501 @@ +// Copyright 2026 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { assert } from 'chai'; +import { Proto } from '@signalapp/mock-server'; + +import * as durations from '../../util/durations/index.std.ts'; +import { initStorage } from './fixtures.node.ts'; +import { debug } from './fixtures.node.ts'; +import { constantTimeEqual, getRandomBytes } from '../../Crypto.node.ts'; + +import type { Bootstrap } from './fixtures.node.ts'; +import type { App } from './fixtures.node.ts'; + +const IdentifierType = Proto.ManifestRecord.Identifier.Type; + +describe('storage service/delete', function (this: Mocha.Suite) { + this.timeout(durations.MINUTE); + + let bootstrap: Bootstrap; + let app: App; + + beforeEach(async () => { + ({ bootstrap, app } = await initStorage()); + }); + + afterEach(async function (this: Mocha.Context) { + if (!bootstrap) { + return; + } + + await bootstrap.maybeSaveLogs(this.currentTest, app); + await app.close(); + await bootstrap.teardown(); + }); + + it('should roundtrip records even if they were deleted when first discovered', async () => { + const { phone, contacts } = bootstrap; + + const alice = contacts[0]; + assert.exists(alice); + + let state = await phone.expectStorageState('initial state'); + + debug('adding deleted records to storage service via phone'); + const deletedAtTimestamp = BigInt(Date.now() + durations.DAY); + + const storyDistributionList = { + type: IdentifierType.STORY_DISTRIBUTION_LIST, + key: Buffer.from(getRandomBytes(16)), + record: { + // if deletedAtTimestamp is set, name and members should not be + storyDistributionList: { + identifier: getRandomBytes(16), + name: null, + deletedAtTimestamp, + allowsReplies: null, + isBlockList: null, + recipientServiceIdsBinary: null, + }, + }, + }; + state = state.addRecord(storyDistributionList); + const stickerPack = { + type: IdentifierType.STICKER_PACK, + key: Buffer.from(getRandomBytes(16)), + record: { + stickerPack: { + packId: getRandomBytes(16), + packKey: getRandomBytes(32), + position: 1, + deletedAtTimestamp, + }, + }, + }; + state = state.addRecord(stickerPack); + const callLink = { + type: IdentifierType.CALL_LINK, + key: Buffer.from(getRandomBytes(16)), + record: { + // if deletedAtTimestampMs is set, adminPassKey should not be + callLink: { + rootKey: getRandomBytes(16), + adminPasskey: null, + deletedAtTimestampMs: deletedAtTimestamp, + }, + }, + }; + state = state.addRecord(callLink); + const chatFolder = { + type: IdentifierType.CHAT_FOLDER, + key: Buffer.from(getRandomBytes(16)), + record: { + chatFolder: { + id: getRandomBytes(16), + name: 'Chat Folder', + position: 4294967295, + showOnlyUnread: null, + showMutedChats: null, + includeAllIndividualChats: null, + includeAllGroupChats: null, + folderType: Proto.ChatFolderRecord.FolderType.CUSTOM, + includedRecipients: null, + excludedRecipients: null, + deletedAtTimestampMs: deletedAtTimestamp, + }, + }, + }; + state = state.addRecord(chatFolder); + const notificationProfile = { + type: IdentifierType.NOTIFICATION_PROFILE, + key: Buffer.from(getRandomBytes(16)), + record: { + notificationProfile: { + id: getRandomBytes(16), + name: 'Notification Profile', + emoji: null, + color: null, + createdAtMs: null, + allowAllCalls: null, + allowAllMentions: null, + allowedMembers: null, + scheduleEnabled: null, + scheduleStartTime: null, + scheduleEndTime: null, + scheduleDaysEnabled: null, + deletedAtTimestampMs: deletedAtTimestamp, + }, + }, + }; + state = state.addRecord(notificationProfile); + + state = state.pin(alice); + + const updatedState = await phone.setStorageState(state); + await phone.sendFetchStorage({ + timestamp: bootstrap.getTimestamp(), + }); + + debug('waiting for Desktop to pick up the change'); + await app.waitForManifestVersion(updatedState.version); + + const window = await app.getWindow(); + const conversationStack = window.locator('.Inbox__conversation-stack'); + + const leftPane = window.locator('#LeftPane'); + + debug('verifying that contact is pinned'); + await leftPane.locator(`[data-testid="${alice.device.aci}"]`).waitFor(); + + debug('unpinning via desktop'); + { + const convo = leftPane.getByTestId(alice.device.aci); + await convo.click(); + + const moreButton = conversationStack.getByRole('button', { + name: 'More Info', + }); + await moreButton.click(); + + const pinButton = window.getByRole('menuitem', { + name: 'Unpin chat', + exact: true, + }); + await pinButton.click(); + } + + debug("waiting for desktop's storage service update to get back to phone"); + + const newState = await phone.waitForStorageState({ + after: updatedState, + predicate: maybeState => !maybeState.isPinned(alice), + }); + + debug( + "validating what's in storage service - deleted items should still be there" + ); + + assert.isTrue( + newState.hasRecord( + item => + constantTimeEqual(item.key, storyDistributionList.key) && + item.record.storyDistributionList?.deletedAtTimestamp === + storyDistributionList.record.storyDistributionList + ?.deletedAtTimestamp + ), + 'looking for deleted storyDistribution list' + ); + assert.isTrue( + newState.hasRecord( + item => + constantTimeEqual(item.key, stickerPack.key) && + item.record.stickerPack?.deletedAtTimestamp === + stickerPack.record.stickerPack?.deletedAtTimestamp + ), + 'looking for deleted stickerPack list' + ); + assert.isTrue( + newState.hasRecord( + item => + constantTimeEqual(item.key, callLink.key) && + item.record.callLink?.deletedAtTimestampMs === + callLink.record.callLink?.deletedAtTimestampMs + ), + 'looking for deleted callLink list' + ); + assert.isTrue( + newState.hasRecord( + item => + constantTimeEqual(item.key, chatFolder.key) && + item.record.chatFolder?.deletedAtTimestampMs === + chatFolder.record.chatFolder?.deletedAtTimestampMs + ), + 'looking for deleted chatFolder list' + ); + assert.isTrue( + newState.hasRecord( + item => + constantTimeEqual(item.key, notificationProfile.key) && + item.record.notificationProfile?.deletedAtTimestampMs === + notificationProfile.record.notificationProfile?.deletedAtTimestampMs + ), + 'looking for deleted notificationProfile list' + ); + }); + + it('should not restore records if they were deleted and removed from storage service', async () => { + const { phone, contacts } = bootstrap; + + const alice = contacts[0]; + assert.exists(alice); + + let initialState = await phone.expectStorageState('initial state'); + + debug('adding deleted records to storage service via phone'); + const deletedAtTimestamp = BigInt(Date.now() + durations.DAY); + + const storyDistributionList = { + type: IdentifierType.STORY_DISTRIBUTION_LIST, + key: Buffer.from(getRandomBytes(16)), + record: { + // if deletedAtTimestamp is set, name and members should not be + storyDistributionList: { + identifier: getRandomBytes(16), + name: null, + deletedAtTimestamp, + allowsReplies: null, + isBlockList: null, + recipientServiceIdsBinary: null, + }, + }, + }; + initialState = initialState.addRecord(storyDistributionList); + const stickerPack = { + type: IdentifierType.STICKER_PACK, + key: Buffer.from(getRandomBytes(16)), + record: { + stickerPack: { + packId: getRandomBytes(16), + packKey: getRandomBytes(32), + position: 1, + deletedAtTimestamp, + }, + }, + }; + initialState = initialState.addRecord(stickerPack); + const callLink = { + type: IdentifierType.CALL_LINK, + key: Buffer.from(getRandomBytes(16)), + record: { + // if deletedAtTimestampMs is set, adminPassKey should not be + callLink: { + rootKey: getRandomBytes(16), + adminPasskey: null, + deletedAtTimestampMs: deletedAtTimestamp, + }, + }, + }; + initialState = initialState.addRecord(callLink); + const chatFolder = { + type: IdentifierType.CHAT_FOLDER, + key: Buffer.from(getRandomBytes(16)), + record: { + chatFolder: { + id: getRandomBytes(16), + name: 'Chat Folder', + position: 4294967295, + showOnlyUnread: null, + showMutedChats: null, + includeAllIndividualChats: null, + includeAllGroupChats: null, + folderType: Proto.ChatFolderRecord.FolderType.CUSTOM, + includedRecipients: null, + excludedRecipients: null, + deletedAtTimestampMs: deletedAtTimestamp, + }, + }, + }; + initialState = initialState.addRecord(chatFolder); + const notificationProfile = { + type: IdentifierType.NOTIFICATION_PROFILE, + key: Buffer.from(getRandomBytes(16)), + record: { + notificationProfile: { + id: getRandomBytes(16), + name: 'Notification Profile', + emoji: null, + color: null, + createdAtMs: null, + allowAllCalls: null, + allowAllMentions: null, + allowedMembers: null, + scheduleEnabled: null, + scheduleStartTime: null, + scheduleEndTime: null, + scheduleDaysEnabled: null, + deletedAtTimestampMs: deletedAtTimestamp, + }, + }, + }; + initialState = initialState.addRecord(notificationProfile); + + initialState = initialState.pin(alice); + + const firstPhoneState = await phone.setStorageState(initialState); + await phone.sendFetchStorage({ + timestamp: bootstrap.getTimestamp(), + }); + + debug('waiting for Desktop to pick up the first change'); + await app.waitForManifestVersion(firstPhoneState.version); + + const window = await app.getWindow(); + const conversationStack = window.locator('.Inbox__conversation-stack'); + + const leftPane = window.locator('#LeftPane'); + + debug('verifying that contact is pinned'); + await leftPane.locator(`[data-testid="${alice.device.aci}"]`).waitFor(); + + debug('unpinning via desktop'); + { + const convo = leftPane.getByTestId(alice.device.aci); + await convo.click(); + + const moreButton = conversationStack.getByRole('button', { + name: 'More Info', + }); + await moreButton.click(); + + const pinButton = window.getByRole('menuitem', { + name: 'Unpin chat', + exact: true, + }); + await pinButton.click(); + } + + debug("waiting for desktop's storage service update to get back to phone"); + + let updateState = await phone.waitForStorageState({ + after: firstPhoneState, + predicate: maybeState => !maybeState.isPinned(alice), + }); + + debug('now removing items from storage service via phone'); + + updateState = updateState.removeRecord( + item => + constantTimeEqual(item.key, storyDistributionList.key) && + item.record.storyDistributionList?.deletedAtTimestamp === + storyDistributionList.record.storyDistributionList?.deletedAtTimestamp + ); + updateState = updateState.removeRecord( + item => + constantTimeEqual(item.key, stickerPack.key) && + item.record.stickerPack?.deletedAtTimestamp === + stickerPack.record.stickerPack?.deletedAtTimestamp + ); + updateState = updateState.removeRecord( + item => + constantTimeEqual(item.key, callLink.key) && + item.record.callLink?.deletedAtTimestampMs === + callLink.record.callLink?.deletedAtTimestampMs + ); + updateState = updateState.removeRecord( + item => + constantTimeEqual(item.key, chatFolder.key) && + item.record.chatFolder?.deletedAtTimestampMs === + chatFolder.record.chatFolder?.deletedAtTimestampMs + ); + updateState = updateState.removeRecord( + item => + constantTimeEqual(item.key, notificationProfile.key) && + item.record.notificationProfile?.deletedAtTimestampMs === + notificationProfile.record.notificationProfile?.deletedAtTimestampMs + ); + + updateState = updateState.pin(alice); + + const secondPhoneState = await phone.setStorageState(updateState); + await phone.sendFetchStorage({ + timestamp: bootstrap.getTimestamp(), + }); + + debug('waiting for Desktop to pick up the second change'); + await app.waitForManifestVersion(secondPhoneState.version); + + debug('verifying that contact is pinned'); + await leftPane.locator(`[data-testid="${alice.device.aci}"]`).waitFor(); + + debug('unpinning via desktop'); + { + const convo = leftPane.getByTestId(alice.device.aci); + await convo.click(); + + const moreButton = conversationStack.getByRole('button', { + name: 'More Info', + }); + await moreButton.click(); + + const pinButton = window.getByRole('menuitem', { + name: 'Unpin chat', + exact: true, + }); + await pinButton.click(); + } + + debug("waiting for desktop's storage service update to get back to phone"); + + const thirdPhoneState = await phone.waitForStorageState({ + after: secondPhoneState, + predicate: maybeState => !maybeState.isPinned(alice), + }); + + debug( + "validating what's in storage service - deleted items should be removed" + ); + + assert.isFalse( + thirdPhoneState.hasRecord(item => { + const itemId = item.record.storyDistributionList?.identifier; + const expectedId = + storyDistributionList.record.storyDistributionList?.identifier; + + if (!itemId || !expectedId) { + return false; + } + return constantTimeEqual(itemId, expectedId); + }), + "don't want to find deleted storyDistribution list" + ); + assert.isFalse( + thirdPhoneState.hasRecord(item => { + const itemId = item.record.stickerPack?.packId; + const expectedId = stickerPack.record.stickerPack?.packId; + + if (!itemId || !expectedId) { + return false; + } + return constantTimeEqual(itemId, expectedId); + }), + "don't want to find deleted stickerPack list" + ); + assert.isFalse( + thirdPhoneState.hasRecord(item => { + const itemId = item.record.callLink?.rootKey; + const expectedId = callLink.record.callLink?.rootKey; + + if (!itemId || !expectedId) { + return false; + } + return constantTimeEqual(itemId, expectedId); + }), + "don't want to find deleted callLink list" + ); + assert.isFalse( + thirdPhoneState.hasRecord(item => { + const itemId = item.record.chatFolder?.id; + const expectedId = chatFolder.record.chatFolder?.id; + + if (!itemId || !expectedId) { + return false; + } + return constantTimeEqual(itemId, expectedId); + }), + "don't want to find deleted chatFolder list" + ); + assert.isFalse( + thirdPhoneState.hasRecord(item => { + const itemId = item.record.notificationProfile?.id; + const expectedId = notificationProfile.record.notificationProfile?.id; + + if (!itemId || !expectedId) { + return false; + } + return constantTimeEqual(itemId, expectedId); + }), + "don't want to find deleted notificationProfile list" + ); + }); +}); diff --git a/ts/test-node/sql/migration_1800_test.node.ts b/ts/test-node/sql/migration_1800_test.node.ts new file mode 100644 index 0000000000..0ee0ec9822 --- /dev/null +++ b/ts/test-node/sql/migration_1800_test.node.ts @@ -0,0 +1,113 @@ +// Copyright 2026 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { assert } from 'chai'; +import { sortBy } from 'lodash'; + +import type { WritableDB } from '../../sql/Interface.std.ts'; +import { + createDB, + updateToVersion, + insertData, + getTableData, +} from './helpers.node.ts'; +import { getRandomBytes } from '../../Crypto.node.ts'; + +describe('SQL/updateToSchemaVersion1800', () => { + let db: WritableDB; + + beforeEach(() => { + db = createDB(); + }); + + afterEach(() => { + db.close(); + }); + + it('adds and sets addedAt, sets storageNeedsSync, deletes if missing both adminKey and storageId', () => { + updateToVersion(db, 1790); + const initialData = [ + { + roomId: 'roomId1 (to be deleted)', + rootKey: Buffer.from(getRandomBytes(32)), + adminKey: null, + storageID: null, + storageVersion: null, + storageUnknownFields: null, + storageNeedsSync: 0, + }, + { + roomId: 'roomId2', + rootKey: Buffer.from(getRandomBytes(32)), + adminKey: Buffer.from(getRandomBytes(32)), + storageID: null, + storageVersion: null, + storageUnknownFields: null, + storageNeedsSync: 0, + }, + { + roomId: 'roomId3', + rootKey: Buffer.from(getRandomBytes(32)), + adminKey: Buffer.from(getRandomBytes(32)), + storageID: 'storageId3', + storageVersion: 3, + storageUnknownFields: Buffer.from(getRandomBytes(32)), + storageNeedsSync: 0, + }, + { + roomId: 'roomId4', + rootKey: Buffer.from(getRandomBytes(32)), + adminKey: null, + storageID: 'storageId4', + storageVersion: 4, + storageUnknownFields: null, + storageNeedsSync: 0, + }, + ]; + insertData(db, 'defunctCallLinks', initialData); + + const now = Date.now(); + updateToVersion(db, 1800); + + const actual = sortBy(getTableData(db, 'defunctCallLinks')); + + assert.lengthOf(actual, 3); + + const item1 = actual[0]; + assert.strictEqual(item1?.roomId, 'roomId2'); + assert.isAtLeast(item1?.addedAt as number, now); + assert.isAtLeast(item1?.storageNeedsSync as number, 1); + + const item2 = actual[1]; + assert.strictEqual(item2?.roomId, 'roomId3'); + assert.isNumber(item2?.addedAt); + assert.isAtLeast(item2?.addedAt as number, now); + assert.isAtLeast(item1?.storageNeedsSync as number, 1); + + const item3 = actual[2]; + assert.strictEqual(item3?.roomId, 'roomId4'); + assert.isNumber(item3?.addedAt); + assert.isAtLeast(item3?.addedAt as number, now); + assert.isAtLeast(item1?.storageNeedsSync as number, 1); + }); + + it('addes a NOT NULL constraint without a table rebuild', () => { + updateToVersion(db, 1790); + const beforeResult = db + .prepare( + "SELECT sql from sqlite_schema WHERE name = 'defunctCallLinks'", + { pluck: true } + ) + .get(); + assert.notMatch(beforeResult, /addedAt INTEGER NOT NULL/); + + updateToVersion(db, 1800); + const afterResult = db + .prepare( + "SELECT sql from sqlite_schema WHERE name = 'defunctCallLinks'", + { pluck: true } + ) + .get(); + assert.match(afterResult as string, /addedAt INTEGER NOT NULL/); + }); +}); diff --git a/ts/textsecure/MessageReceiver.preload.ts b/ts/textsecure/MessageReceiver.preload.ts index 3cd95a9b24..886c66c9f8 100644 --- a/ts/textsecure/MessageReceiver.preload.ts +++ b/ts/textsecure/MessageReceiver.preload.ts @@ -3558,6 +3558,7 @@ export default class MessageReceiver type: callLinkUpdateSyncType, rootKey, adminKey, + timestamp: envelope.timestamp, }, this.#removeFromCache.bind(this, envelope) ); diff --git a/ts/textsecure/messageReceiverEvents.std.ts b/ts/textsecure/messageReceiverEvents.std.ts index 67f5357547..d0cc0f8e5f 100644 --- a/ts/textsecure/messageReceiverEvents.std.ts +++ b/ts/textsecure/messageReceiverEvents.std.ts @@ -549,6 +549,7 @@ export type CallLinkUpdateSyncEventData = Readonly<{ type: CallLinkUpdateSyncType; rootKey: Uint8Array | undefined; adminKey: Uint8Array | undefined; + timestamp: number; }>; export class CallLinkUpdateSyncEvent extends ConfirmableEvent { diff --git a/ts/types/CallLink.std.ts b/ts/types/CallLink.std.ts index e11b0fb5b9..1b8c6a3dcb 100644 --- a/ts/types/CallLink.std.ts +++ b/ts/types/CallLink.std.ts @@ -90,6 +90,7 @@ export type DefunctCallLinkType = Readonly<{ roomId: string; rootKey: string; adminKey: string | null; + addedAt: number; }> & StorageServiceFieldsType; @@ -97,6 +98,7 @@ export type DefunctCallLinkRecord = Readonly<{ roomId: string; rootKey: Uint8Array; adminKey: Uint8Array | null; + addedAt: number; storageID: string | null; storageVersion: number | null; storageUnknownFields: Uint8Array | null; @@ -107,6 +109,7 @@ export const defunctCallLinkRecordSchema = z.object({ roomId: z.string(), rootKey: z.instanceof(Uint8Array), adminKey: z.instanceof(Uint8Array).nullable(), + addedAt: z.number(), storageID: z.string().nullable(), storageVersion: z.number().int().nullable(), storageUnknownFields: z.instanceof(Uint8Array).nullable(), diff --git a/ts/util/callDisposition.preload.ts b/ts/util/callDisposition.preload.ts index e3f09c092c..44ab383903 100644 --- a/ts/util/callDisposition.preload.ts +++ b/ts/util/callDisposition.preload.ts @@ -64,7 +64,6 @@ import type { ConversationModel } from '../models/conversations.preload.ts'; import { drop } from './drop.std.ts'; import { sendCallLinkUpdateSync } from './sendCallLinkUpdateSync.preload.ts'; import { runStorageServiceUploadJob } from '../services/storage.preload.ts'; -import { CallLinkFinalizeDeleteManager } from '../jobs/CallLinkFinalizeDeleteManager.preload.ts'; import { parseLoose, parseStrict } from './schemas.std.ts'; import { calling } from '../services/calling.preload.ts'; import { cleanupMessages } from './cleanup.preload.ts'; @@ -74,6 +73,7 @@ import { update as updateExpiringMessagesService } from '../services/expiringMes import type { DurationInSeconds } from './durations/duration-in-seconds.std.ts'; import { isFeaturedEnabledNoRedux } from './isFeatureEnabled.dom.ts'; import type { GetUnreadCallMessagesAndMarkReadResult } from '../sql/Interface.std.ts'; +import { callLinkCleanupService } from '../services/expiring/callLinkCleanupService.preload.ts'; const { isEqual } = lodash; @@ -1463,7 +1463,7 @@ export async function clearCallHistoryDataAndSync( ); // This skips call history for admin call links. const messageIds = await DataWriter.clearCallHistory(latestCall); - const isStorageSyncNeeded = await DataWriter.beginDeleteAllCallLinks(); + const isStorageSyncNeeded = await DataWriter.markAllCallLinksDeleted(); if (isStorageSyncNeeded) { runStorageServiceUploadJob({ reason: 'clearCallHistoryDataAndSync' }); } @@ -1489,19 +1489,13 @@ export async function clearCallHistoryDataAndSync( await calling.deleteCallLink(callLink); // oxlint-disable-next-line no-await-in-loop await DataWriter.deleteCallHistoryByRoomId(callLink.roomId); - // Wait for storage service sync before finalizing delete. - drop( - CallLinkFinalizeDeleteManager.addJob( - { roomId: callLink.roomId }, - { delay: 10000 } - ) - ); successCount += 1; } catch (error) { log.warn('clearCallHistory: Failed to delete admin call link', error); failCount += 1; } } + drop(callLinkCleanupService.trigger('deleted all call links')); log.info( `clearCallHistory: Deleted admin call links, success=${successCount} failed=${failCount}` ); diff --git a/ts/util/callLinksRingrtc.node.ts b/ts/util/callLinksRingrtc.node.ts index e5a586011e..dd9ae228ba 100644 --- a/ts/util/callLinksRingrtc.node.ts +++ b/ts/util/callLinksRingrtc.node.ts @@ -146,6 +146,7 @@ export function defunctCallLinkFromRecord( roomId: record.roomId, rootKey, adminKey, + addedAt: record.addedAt, storageID: record.storageID || undefined, storageVersion: record.storageVersion || undefined, storageUnknownFields: record.storageUnknownFields || undefined, @@ -168,6 +169,7 @@ export function defunctCallLinkToRecord( roomId: defunctCallLink.roomId, rootKey, adminKey, + addedAt: defunctCallLink.addedAt, storageID: defunctCallLink.storageID || null, storageVersion: defunctCallLink.storageVersion || null, storageUnknownFields: defunctCallLink.storageUnknownFields || null, diff --git a/ts/util/onCallLinkUpdateSync.preload.ts b/ts/util/onCallLinkUpdateSync.preload.ts index 717cbbd2db..82bdd25cac 100644 --- a/ts/util/onCallLinkUpdateSync.preload.ts +++ b/ts/util/onCallLinkUpdateSync.preload.ts @@ -2,14 +2,18 @@ // SPDX-License-Identifier: AGPL-3.0-only import { CallLinkRootKey } from '@signalapp/ringrtc'; -import type { CallLinkUpdateSyncEvent } from '../textsecure/messageReceiverEvents.std.ts'; import { createLogger } from '../logging/log.std.ts'; import * as Errors from '../types/errors.std.ts'; import { fromAdminKeyBytes } from './callLinks.std.ts'; import { getRoomIdFromRootKey } from './callLinksRingrtc.node.ts'; import { strictAssert } from './assert.std.ts'; import { CallLinkUpdateSyncType } from '../types/CallLink.std.ts'; -import { DataWriter } from '../sql/Client.preload.ts'; +import { DataWriter, DataReader } from '../sql/Client.preload.ts'; +import { drop } from './drop.std.ts'; + +import type { CallLinkUpdateSyncEvent } from '../textsecure/messageReceiverEvents.std.ts'; +import { callLinkCleanupService } from '../services/expiring/callLinkCleanupService.preload.ts'; +import { defunctCallLinkCleanupService } from '../services/expiring/defunctCallLinkCleanupService.preload.ts'; const log = createLogger('onCallLinkUpdateSync'); @@ -17,7 +21,7 @@ export async function onCallLinkUpdateSync( syncEvent: CallLinkUpdateSyncEvent ): Promise { const { callLinkUpdate, confirm } = syncEvent; - const { type, rootKey, adminKey } = callLinkUpdate; + const { type, rootKey, adminKey, timestamp } = callLinkUpdate; if (!rootKey) { log.warn('Missing rootKey, invalid sync message'); @@ -50,8 +54,31 @@ export async function onCallLinkUpdateSync( adminKey: adminKeyString, }); } else if (type === CallLinkUpdateSyncType.Delete) { - log.info(`${logId}: Deleting call link record ${roomId}`); - await DataWriter.deleteCallLinkFromSync(roomId); + if (await DataReader.callLinkExists(roomId)) { + log.info(`${logId}: Marking call link ${roomId} deleted`); + await DataWriter.markCallLinkDeleted(roomId, timestamp); + drop( + callLinkCleanupService.trigger('onCallLinkUpdateSync, marked deleted') + ); + } else { + const defunctCallLink = + await DataReader.getDefunctCallLinkByRoomId(roomId); + if (defunctCallLink && defunctCallLink.addedAt > timestamp) { + log.info( + `${logId}: Updating timestamp for defunct call link ${roomId}` + ); + const updated = { ...defunctCallLink, addedAt: timestamp }; + await DataWriter.updateDefunctCallLink(updated); + drop( + defunctCallLinkCleanupService.trigger( + 'onCallLinkUpdateSync, updated addedAt' + ) + ); + } else if (!defunctCallLink) { + log.info(`${logId}: No local record for deleted call link ${roomId}`); + } + } + window.reduxActions.calling.handleCallLinkDelete({ roomId }); }