diff --git a/test/keychange_listener_test.js b/test/keychange_listener_test.js index 33fe9dbe4c..2189da5949 100644 --- a/test/keychange_listener_test.js +++ b/test/keychange_listener_test.js @@ -38,6 +38,7 @@ describe('KeyChangeListener', () => { after(async () => { await window.Signal.Data.removeAllMessagesInConversation(convo.id, { + logId: phoneNumberWithKeyChange, MessageCollection: Whisper.MessageCollection, }); await window.Signal.Data.removeConversation(convo.id, { @@ -78,6 +79,7 @@ describe('KeyChangeListener', () => { }); after(async () => { await window.Signal.Data.removeAllMessagesInConversation(groupConvo.id, { + logId: phoneNumberWithKeyChange, MessageCollection: Whisper.MessageCollection, }); await window.Signal.Data.removeConversation(groupConvo.id, { diff --git a/ts/background.ts b/ts/background.ts index f4545fbe81..41bd0dddd4 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -240,7 +240,7 @@ type WhatIsThis = import('./window.d').WhatIsThis; if (_.isNumber(preMessageReceiverStatus)) { return preMessageReceiverStatus; } - return -1; + return WebSocket.CLOSED; }; window.Whisper.events = _.clone(window.Backbone.Events); let accountManager: typeof window.textsecure.AccountManager; @@ -1604,7 +1604,17 @@ type WhatIsThis = import('./window.d').WhatIsThis; // Maybe refresh remote configuration when we become active window.registerForActive(async () => { - await window.Signal.RemoteConfig.maybeRefreshRemoteConfig(); + try { + await window.Signal.RemoteConfig.maybeRefreshRemoteConfig(); + } catch (error) { + if (error && window._.isNumber(error.code)) { + window.log.warn( + `registerForActive: Failed to to refresh remote config. Code: ${error.code}` + ); + return; + } + throw error; + } }); // Listen for changes to the `desktop.clientExpiration` remote flag diff --git a/ts/models/conversations.ts b/ts/models/conversations.ts index 525a2535be..7f6583845a 100644 --- a/ts/models/conversations.ts +++ b/ts/models/conversations.ts @@ -4013,6 +4013,7 @@ export class ConversationModel extends window.Backbone.Model< window.Signal.Data.updateConversation(this.attributes); await window.Signal.Data.removeAllMessagesInConversation(this.id, { + logId: this.idForLogging(), MessageCollection: window.Whisper.MessageCollection, }); } diff --git a/ts/sql/Client.ts b/ts/sql/Client.ts index a9bf69d483..13abc372dd 100644 --- a/ts/sql/Client.ts +++ b/ts/sql/Client.ts @@ -150,6 +150,7 @@ const dataInterface: ClientInterface = { saveMessage, saveMessages, removeMessage, + removeMessages, getUnreadByConversation, getMessageBySender, @@ -225,7 +226,6 @@ const dataInterface: ClientInterface = { // Client-side only, and test-only _removeConversations, - _removeMessages, _cleanData, _jobs, }; @@ -903,8 +903,8 @@ async function removeMessage( } // Note: this method will not clean up external files, just delete from SQL -async function _removeMessages(ids: Array) { - await channels.removeMessage(ids); +async function removeMessages(ids: Array) { + await channels.removeMessages(ids); } async function getMessageById( @@ -1074,15 +1074,23 @@ async function migrateConversationMessages( async function removeAllMessagesInConversation( conversationId: string, { + logId, MessageCollection, - }: { MessageCollection: typeof MessageModelCollectionType } + }: { + logId: string; + MessageCollection: typeof MessageModelCollectionType; + } ) { let messages; do { - // Yes, we really want the await in the loop. We're deleting 100 at a + const chunkSize = 20; + window.log.info( + `removeAllMessagesInConversation/${logId}: Fetching chunk of ${chunkSize} messages` + ); + // Yes, we really want the await in the loop. We're deleting a chunk at a // time so we don't use too much memory. messages = await getOlderMessagesByConversation(conversationId, { - limit: 100, + limit: chunkSize, MessageCollection, }); @@ -1092,13 +1100,17 @@ async function removeAllMessagesInConversation( const ids = messages.map((message: MessageModel) => message.id); + window.log.info(`removeAllMessagesInConversation/${logId}: Cleanup...`); // Note: It's very important that these models are fully hydrated because // we need to delete all associated on-disk files along with the database delete. - await Promise.all( - messages.map(async (message: MessageModel) => message.cleanup()) + const queue = new window.PQueue({ concurrency: 3, timeout: 1000 * 60 * 2 }); + queue.addAll( + messages.map((message: MessageModel) => async () => message.cleanup()) ); + await queue.onIdle(); - await channels.removeMessage(ids); + window.log.info(`removeAllMessagesInConversation/${logId}: Deleting...`); + await channels.removeMessages(ids); } while (messages.length > 0); } diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 7a1bf1fcff..16b666a5d2 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -238,7 +238,8 @@ export type ServerInterface = DataInterface & { conversationId: string ) => Promise>; removeConversation: (id: Array | string) => Promise; - removeMessage: (id: Array | string) => Promise; + removeMessage: (id: string) => Promise; + removeMessages: (ids: Array) => Promise; saveMessage: ( data: MessageType, options: { forceSave?: boolean } @@ -266,51 +267,41 @@ export type ServerInterface = DataInterface & { }; export type ClientInterface = DataInterface & { - getAllConversations: ({ - ConversationCollection, - }: { + getAllConversations: (options: { ConversationCollection: typeof ConversationModelCollectionType; }) => Promise; getAllGroupsInvolvingId: ( id: string, - { - ConversationCollection, - }: { + options: { ConversationCollection: typeof ConversationModelCollectionType; } ) => Promise; - getAllPrivateConversations: ({ - ConversationCollection, - }: { + getAllPrivateConversations: (options: { ConversationCollection: typeof ConversationModelCollectionType; }) => Promise; getConversationById: ( id: string, - { Conversation }: { Conversation: typeof ConversationModel } + options: { Conversation: typeof ConversationModel } ) => Promise; - getExpiredMessages: ({ - MessageCollection, - }: { + getExpiredMessages: (options: { MessageCollection: typeof MessageModelCollectionType; }) => Promise; getMessageById: ( id: string, - { Message }: { Message: typeof MessageModel } + options: { Message: typeof MessageModel } ) => Promise; getMessageBySender: ( - options: { + data: { source: string; sourceUuid: string; sourceDevice: string; sent_at: number; }, - { Message }: { Message: typeof MessageModel } + options: { Message: typeof MessageModel } ) => Promise; getMessagesBySentAt: ( sentAt: number, - { - MessageCollection, - }: { MessageCollection: typeof MessageModelCollectionType } + options: { MessageCollection: typeof MessageModelCollectionType } ) => Promise; getOlderMessagesByConversation: ( conversationId: string, @@ -341,39 +332,33 @@ export type ClientInterface = DataInterface & { Message: typeof MessageModel; } ) => Promise; - getNextExpiringMessage: ({ - Message, - }: { + getNextExpiringMessage: (options: { Message: typeof MessageModel; }) => Promise; - getNextTapToViewMessageToAgeOut: ({ - Message, - }: { + getNextTapToViewMessageToAgeOut: (options: { Message: typeof MessageModel; }) => Promise; - getOutgoingWithoutExpiresAt: ({ - MessageCollection, - }: { + getOutgoingWithoutExpiresAt: (options: { MessageCollection: typeof MessageModelCollectionType; }) => Promise; - getTapToViewMessagesNeedingErase: ({ - MessageCollection, - }: { + getTapToViewMessagesNeedingErase: (options: { MessageCollection: typeof MessageModelCollectionType; }) => Promise; getUnreadByConversation: ( conversationId: string, - { - MessageCollection, - }: { MessageCollection: typeof MessageModelCollectionType } + options: { MessageCollection: typeof MessageModelCollectionType } ) => Promise; removeConversation: ( id: string, - { Conversation }: { Conversation: typeof ConversationModel } + options: { Conversation: typeof ConversationModel } ) => Promise; removeMessage: ( id: string, - { Message }: { Message: typeof MessageModel } + options: { Message: typeof MessageModel } + ) => Promise; + removeMessages: ( + ids: Array, + options: { Message: typeof MessageModel } ) => Promise; saveMessage: ( data: MessageType, @@ -383,9 +368,7 @@ export type ClientInterface = DataInterface & { // Test-only - _getAllMessages: ({ - MessageCollection, - }: { + _getAllMessages: (options: { MessageCollection: typeof MessageModelCollectionType; }) => Promise; @@ -394,9 +377,10 @@ export type ClientInterface = DataInterface & { shutdown: () => Promise; removeAllMessagesInConversation: ( conversationId: string, - { - MessageCollection, - }: { MessageCollection: typeof MessageModelCollectionType } + options: { + logId: string; + MessageCollection: typeof MessageModelCollectionType; + } ) => Promise; removeOtherData: () => Promise; cleanupOrphanedAttachments: () => Promise; @@ -405,7 +389,6 @@ export type ClientInterface = DataInterface & { // Client-side only, and test-only _removeConversations: (ids: Array) => Promise; - _removeMessages: (ids: Array) => Promise; _cleanData: (data: any, path?: string) => any; _jobs: { [id: string]: ClientJobType }; }; diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 104c4dd984..bbb5084665 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -129,6 +129,7 @@ const dataInterface: ServerInterface = { saveMessage, saveMessages, removeMessage, + removeMessages, getUnreadByConversation, getMessageBySender, getMessageById, @@ -236,6 +237,7 @@ type PromisifiedSQLDatabase = { statement: string, params?: { [key: string]: any } ) => Promise>; + on: (event: 'trace', handler: (sql: string) => void) => void; }; function promisify(rawInstance: sql.Database): PromisifiedSQLDatabase { @@ -244,6 +246,7 @@ function promisify(rawInstance: sql.Database): PromisifiedSQLDatabase { run: pify(rawInstance.run.bind(rawInstance)), get: pify(rawInstance.get.bind(rawInstance)), all: pify(rawInstance.all.bind(rawInstance)), + on: rawInstance.on.bind(rawInstance), }; } @@ -928,51 +931,51 @@ async function updateToSchemaVersion12( try { await instance.run(`CREATE TABLE sticker_packs( - id TEXT PRIMARY KEY, - key TEXT NOT NULL, + id TEXT PRIMARY KEY, + key TEXT NOT NULL, - author STRING, - coverStickerId INTEGER, - createdAt INTEGER, - downloadAttempts INTEGER, - installedAt INTEGER, - lastUsed INTEGER, - status STRING, - stickerCount INTEGER, - title STRING - );`); + author STRING, + coverStickerId INTEGER, + createdAt INTEGER, + downloadAttempts INTEGER, + installedAt INTEGER, + lastUsed INTEGER, + status STRING, + stickerCount INTEGER, + title STRING + );`); await instance.run(`CREATE TABLE stickers( - id INTEGER NOT NULL, - packId TEXT NOT NULL, + id INTEGER NOT NULL, + packId TEXT NOT NULL, - emoji STRING, - height INTEGER, - isCoverOnly INTEGER, - lastUsed INTEGER, - path STRING, - width INTEGER, + emoji STRING, + height INTEGER, + isCoverOnly INTEGER, + lastUsed INTEGER, + path STRING, + width INTEGER, - PRIMARY KEY (id, packId), - CONSTRAINT stickers_fk - FOREIGN KEY (packId) - REFERENCES sticker_packs(id) - ON DELETE CASCADE - );`); + PRIMARY KEY (id, packId), + CONSTRAINT stickers_fk + FOREIGN KEY (packId) + REFERENCES sticker_packs(id) + ON DELETE CASCADE + );`); await instance.run(`CREATE INDEX stickers_recents - ON stickers ( - lastUsed - ) WHERE lastUsed IS NOT NULL;`); + ON stickers ( + lastUsed + ) WHERE lastUsed IS NOT NULL;`); await instance.run(`CREATE TABLE sticker_references( - messageId STRING, - packId TEXT, - CONSTRAINT sticker_references_fk - FOREIGN KEY(packId) - REFERENCES sticker_packs(id) - ON DELETE CASCADE - );`); + messageId STRING, + packId TEXT, + CONSTRAINT sticker_references_fk + FOREIGN KEY(packId) + REFERENCES sticker_packs(id) + ON DELETE CASCADE + );`); await instance.run('PRAGMA user_version = 12;'); await instance.run('COMMIT TRANSACTION;'); @@ -1685,24 +1688,26 @@ async function initialize({ try { promisified = await openAndSetUpSQLCipher(databaseFilePath, { key }); - // promisified.on('trace', async statement => { - // if ( - // !globalInstance || - // statement.startsWith('--') || - // statement.includes('COMMIT') || - // statement.includes('BEGIN') || - // statement.includes('ROLLBACK') - // ) { - // return; - // } + // if (promisified) { + // promisified.on('trace', async statement => { + // if ( + // !globalInstance || + // statement.startsWith('--') || + // statement.includes('COMMIT') || + // statement.includes('BEGIN') || + // statement.includes('ROLLBACK') + // ) { + // return; + // } - // // Note that this causes problems when attempting to commit transactions - this - // // statement is running, and we get at SQLITE_BUSY error. So we delay. - // await new Promise(resolve => setTimeout(resolve, 1000)); + // // Note that this causes problems when attempting to commit transactions - this + // // statement is running, and we get at SQLITE_BUSY error. So we delay. + // await new Promise(resolve => setTimeout(resolve, 1000)); - // const data = await db.get(`EXPLAIN QUERY PLAN ${statement}`); - // console._log(`EXPLAIN QUERY PLAN ${statement}\n`, data && data.detail); - // }); + // const data = await promisified.get(`EXPLAIN QUERY PLAN ${statement}`); + // console._log(`EXPLAIN QUERY PLAN ${statement}\n`, data && data.detail); + // }); + // } await updateSchema(promisified); @@ -2583,22 +2588,16 @@ async function saveMessages( } saveMessages.needsSerial = true; -async function removeMessage(id: Array | string) { +async function removeMessage(id: string) { const db = getInstance(); - if (!Array.isArray(id)) { - await db.run('DELETE FROM messages WHERE id = $id;', { $id: id }); + await db.run('DELETE FROM messages WHERE id = $id;', { $id: id }); +} - return; - } - - if (!id.length) { - throw new Error('removeMessages: No ids to delete!'); - } - - // Our node interface doesn't seem to allow you to replace one single ? with an array +async function removeMessages(ids: Array) { + const db = getInstance(); await db.run( - `DELETE FROM messages WHERE id IN ( ${id.map(() => '?').join(', ')} );`, - id + `DELETE FROM messages WHERE id IN ( ${ids.map(() => '?').join(', ')} );`, + ids ); } diff --git a/ts/views/conversation_view.ts b/ts/views/conversation_view.ts index f5cebd358b..a277fd8b4b 100644 --- a/ts/views/conversation_view.ts +++ b/ts/views/conversation_view.ts @@ -2873,16 +2873,14 @@ Whisper.ConversationView = Whisper.View.extend({ async destroyMessages() { try { await this.confirm(window.i18n('deleteConversationConfirmation')); - try { - this.model.trigger('unload', 'delete messages'); - await this.model.destroyMessages(); - this.model.updateLastMessage(); - } catch (error) { - window.log.error( - 'destroyMessages: Failed to successfully delete conversation', - error && error.stack ? error.stack : error - ); - } + this.longRunningTaskWrapper({ + name: 'destroymessages', + task: async () => { + this.model.trigger('unload', 'delete messages'); + await this.model.destroyMessages(); + this.model.updateLastMessage(); + }, + }); } catch (error) { // nothing to see here, user canceled out of dialog }