From d6e81eee11802fdf01e1df04ff4d5013217b6e4f Mon Sep 17 00:00:00 2001 From: trevor-signal <131492920+trevor-signal@users.noreply.github.com> Date: Thu, 22 May 2025 21:09:54 -0400 Subject: [PATCH] Normalize message attachments --- app/attachment_channel.ts | 86 +- ts/AttachmentCrypto.ts | 4 +- .../conversation/ImageGrid.stories.tsx | 1 + ts/components/conversation/Quote.stories.tsx | 10 +- ts/components/conversation/Quote.tsx | 7 +- ts/jobs/AttachmentDownloadManager.ts | 17 +- ts/messageModifiers/AttachmentDownloads.ts | 5 +- ts/messages/copyQuote.ts | 34 +- ts/messages/handleDataMessage.ts | 3 - ts/model-types.d.ts | 11 +- ts/sql/Client.ts | 44 +- ts/sql/Interface.ts | 163 +++- ts/sql/Server.ts | 910 +++++++++++++----- ts/sql/hydration.ts | 343 ++++++- ts/sql/mainWorker.ts | 42 +- ts/sql/migrations/1360-attachments.ts | 111 +++ ts/sql/migrations/index.ts | 6 +- ts/sql/sqlLogger.ts | 45 + ts/sql/util.ts | 24 + ts/state/ducks/lightbox.ts | 7 +- ts/state/selectors/message.ts | 14 +- ts/test-electron/backup/attachments_test.ts | 12 +- ts/test-electron/backup/helpers.ts | 3 - .../cleanupOrphanedAttachments_test.ts | 259 +++++ ts/test-electron/models/conversations_test.ts | 3 - .../normalizedAttachments_test.ts | 608 ++++++++++++ .../AttachmentDownloadManager_test.ts | 30 +- ts/test-node/sql/migration_1360_test.ts | 89 ++ ts/test-node/types/Message2_test.ts | 4 +- .../initializeAttachmentMetadata_test.ts | 221 ----- ts/textsecure/downloadAttachment.ts | 25 +- ts/types/Attachment.ts | 101 +- ts/types/Message.ts | 13 - ts/types/Message2.ts | 10 +- ts/types/Stickers.ts | 4 +- ts/types/Util.ts | 3 + .../message/initializeAttachmentMetadata.ts | 48 - ts/util/dropNull.ts | 23 + ts/util/makeQuote.ts | 4 +- 39 files changed, 2540 insertions(+), 807 deletions(-) create mode 100644 ts/sql/migrations/1360-attachments.ts create mode 100644 ts/sql/sqlLogger.ts create mode 100644 ts/test-electron/cleanupOrphanedAttachments_test.ts create mode 100644 ts/test-electron/normalizedAttachments_test.ts create mode 100644 ts/test-node/sql/migration_1360_test.ts delete mode 100644 ts/test-node/types/message/initializeAttachmentMetadata_test.ts delete mode 100644 ts/types/message/initializeAttachmentMetadata.ts diff --git a/app/attachment_channel.ts b/app/attachment_channel.ts index 91fda68608..c2b6e82ecb 100644 --- a/app/attachment_channel.ts +++ b/app/attachment_channel.ts @@ -273,11 +273,13 @@ type DeleteOrphanedAttachmentsOptionsType = Readonly<{ type CleanupOrphanedAttachmentsOptionsType = Readonly<{ sql: MainSQL; userDataPath: string; + _block?: boolean; }>; async function cleanupOrphanedAttachments({ sql, userDataPath, + _block = false, }: CleanupOrphanedAttachmentsOptionsType): Promise { await deleteAllBadges({ userDataPath, @@ -304,8 +306,6 @@ async function cleanupOrphanedAttachments({ attachments: orphanedDraftAttachments, }); - // Delete orphaned attachments from conversations and messages. - const orphanedAttachments = new Set(await getAllAttachments(userDataPath)); console.log( 'cleanupOrphanedAttachments: found ' + @@ -319,21 +319,27 @@ async function cleanupOrphanedAttachments({ ); { - const attachments: Array = await sql.sqlRead( + const conversationAttachments: Array = await sql.sqlRead( 'getKnownConversationAttachments' ); - let missing = 0; - for (const known of attachments) { + let missingConversationAttachments = 0; + for (const known of conversationAttachments) { if (!orphanedAttachments.delete(known)) { - missing += 1; + missingConversationAttachments += 1; } } console.log( - `cleanupOrphanedAttachments: found ${attachments.length} conversation ` + - `attachments (${missing} missing), ${orphanedAttachments.size} remain` + `cleanupOrphanedAttachments: Got ${conversationAttachments.length} conversation attachments,` + + ` ${orphanedAttachments.size} remain` ); + + if (missingConversationAttachments > 0) { + console.warn( + `cleanupOrphanedAttachments: ${missingConversationAttachments} conversation attachments were not found on disk` + ); + } } { @@ -347,20 +353,32 @@ async function cleanupOrphanedAttachments({ } console.log( - `cleanupOrphanedAttachments: found ${downloads.length} downloads ` + - `(${missing} missing), ${orphanedDownloads.size} remain` + `cleanupOrphanedAttachments: found ${downloads.length} known downloads, ` + + `${orphanedDownloads.size} remain` ); + + if (missing > 0) { + console.warn( + `cleanupOrphanedAttachments: ${missing} downloads were not found on disk` + ); + } } // This call is intentionally not awaited. We block the app while running // all fetches above to ensure that there are no in-flight attachments that // are saved to disk, but not put into any message or conversation model yet. - deleteOrphanedAttachments({ + const deletePromise = deleteOrphanedAttachments({ orphanedAttachments, orphanedDownloads, sql, userDataPath, }); + + if (_block) { + await deletePromise; + } else { + drop(deletePromise); + } } function deleteOrphanedAttachments({ @@ -368,14 +386,14 @@ function deleteOrphanedAttachments({ orphanedDownloads, sql, userDataPath, -}: DeleteOrphanedAttachmentsOptionsType): void { +}: DeleteOrphanedAttachmentsOptionsType): Promise { // This function *can* throw. async function runWithPossibleException(): Promise { let cursor: MessageAttachmentsCursorType | undefined; - let totalFound = 0; + let totalAttachmentsFound = 0; let totalMissing = 0; let totalDownloadsFound = 0; - let totalDownloadsMissing = 0; + try { do { let attachments: ReadonlyArray; @@ -387,7 +405,7 @@ function deleteOrphanedAttachments({ cursor )); - totalFound += attachments.length; + totalAttachmentsFound += attachments.length; totalDownloadsFound += downloads.length; for (const known of attachments) { @@ -397,9 +415,7 @@ function deleteOrphanedAttachments({ } for (const known of downloads) { - if (!orphanedDownloads.delete(known)) { - totalDownloadsMissing += 1; - } + orphanedDownloads.delete(known); } if (cursor === undefined) { @@ -418,11 +434,16 @@ function deleteOrphanedAttachments({ } console.log( - `cleanupOrphanedAttachments: found ${totalFound} message ` + - `attachments, (${totalMissing} missing) ` + - `${orphanedAttachments.size} remain` + `cleanupOrphanedAttachments: ${totalAttachmentsFound} message ` + + `attachments; ${orphanedAttachments.size} remain` ); + if (totalMissing > 0) { + console.warn( + `cleanupOrphanedAttachments: ${totalMissing} message attachments were not found on disk` + ); + } + await deleteAllAttachments({ userDataPath, attachments: Array.from(orphanedAttachments), @@ -430,7 +451,6 @@ function deleteOrphanedAttachments({ console.log( `cleanupOrphanedAttachments: found ${totalDownloadsFound} downloads ` + - `(${totalDownloadsMissing} missing) ` + `${orphanedDownloads.size} remain` ); await deleteAllDownloads({ @@ -454,8 +474,7 @@ function deleteOrphanedAttachments({ } } - // Intentionally not awaiting - void runSafe(); + return runSafe(); } let attachmentsDir: string | undefined; @@ -505,12 +524,19 @@ export function initialize({ rmSync(downloadsDir, { recursive: true, force: true }); }); - ipcMain.handle(CLEANUP_ORPHANED_ATTACHMENTS_KEY, async () => { - const start = Date.now(); - await cleanupOrphanedAttachments({ sql, userDataPath: configDir }); - const duration = Date.now() - start; - console.log(`cleanupOrphanedAttachments: took ${duration}ms`); - }); + ipcMain.handle( + CLEANUP_ORPHANED_ATTACHMENTS_KEY, + async (_event, { _block }) => { + const start = Date.now(); + await cleanupOrphanedAttachments({ + sql, + userDataPath: configDir, + _block, + }); + const duration = Date.now() - start; + console.log(`cleanupOrphanedAttachments: took ${duration}ms`); + } + ); ipcMain.handle(CLEANUP_DOWNLOADS_KEY, async () => { const start = Date.now(); diff --git a/ts/AttachmentCrypto.ts b/ts/AttachmentCrypto.ts index 92e4611405..01412642b5 100644 --- a/ts/AttachmentCrypto.ts +++ b/ts/AttachmentCrypto.ts @@ -81,6 +81,7 @@ export type ReencryptedAttachmentV2 = { localKey: string; isReencryptableToSameDigest: boolean; version: 2; + size: number; }; export type ReencryptionInfo = { @@ -583,7 +584,7 @@ export async function decryptAttachmentV2ToSink( export async function decryptAndReencryptLocally( options: DecryptAttachmentOptionsType ): Promise { - const { idForLogging } = options; + const { idForLogging, size } = options; const logId = `reencryptAttachmentV2(${idForLogging})`; // Create random output file @@ -622,6 +623,7 @@ export async function decryptAndReencryptLocally( plaintextHash: result.plaintextHash, isReencryptableToSameDigest: result.isReencryptableToSameDigest, version: 2, + size, }; } catch (error) { log.error( diff --git a/ts/components/conversation/ImageGrid.stories.tsx b/ts/components/conversation/ImageGrid.stories.tsx index fb759df848..e415e04498 100644 --- a/ts/components/conversation/ImageGrid.stories.tsx +++ b/ts/components/conversation/ImageGrid.stories.tsx @@ -913,6 +913,7 @@ export function MixedContentTypes(args: Props): JSX.Element { screenshot: { height: 112, width: 112, + size: 128000, url: '/fixtures/kitten-4-112-112.jpg', contentType: IMAGE_JPEG, path: 'originalpath', diff --git a/ts/components/conversation/Quote.stories.tsx b/ts/components/conversation/Quote.stories.tsx index 49cb4741cf..979ec19753 100644 --- a/ts/components/conversation/Quote.stories.tsx +++ b/ts/components/conversation/Quote.stories.tsx @@ -244,7 +244,7 @@ ImageOnly.args = { width: 100, size: 100, path: pngUrl, - objectUrl: pngUrl, + url: pngUrl, }, }, }; @@ -261,7 +261,7 @@ ImageAttachment.args = { width: 100, size: 100, path: pngUrl, - objectUrl: pngUrl, + url: pngUrl, }, }, }; @@ -331,7 +331,7 @@ VideoOnly.args = { width: 100, size: 100, path: pngUrl, - objectUrl: pngUrl, + url: pngUrl, }, }, text: undefined, @@ -349,7 +349,7 @@ VideoAttachment.args = { width: 100, size: 100, path: pngUrl, - objectUrl: pngUrl, + url: pngUrl, }, }, }; @@ -588,7 +588,7 @@ IsStoryReplyEmoji.args = { width: 100, size: 100, path: pngUrl, - objectUrl: pngUrl, + url: pngUrl, }, }, reactionEmoji: '🏋️', diff --git a/ts/components/conversation/Quote.tsx b/ts/components/conversation/Quote.tsx index 779dc8ccbc..2e90cbb480 100644 --- a/ts/components/conversation/Quote.tsx +++ b/ts/components/conversation/Quote.tsx @@ -33,7 +33,10 @@ import type { QuotedAttachmentType } from '../../model-types'; const EMPTY_OBJECT = Object.freeze(Object.create(null)); -export type QuotedAttachmentForUIType = QuotedAttachmentType & +export type QuotedAttachmentForUIType = Pick< + QuotedAttachmentType, + 'contentType' | 'thumbnail' | 'fileName' +> & Pick; export type Props = { @@ -101,7 +104,7 @@ function getUrl(thumbnail?: ThumbnailType): string | undefined { return; } - return thumbnail.objectUrl || thumbnail.url; + return thumbnail.url; } function getTypeLabel({ diff --git a/ts/jobs/AttachmentDownloadManager.ts b/ts/jobs/AttachmentDownloadManager.ts index 8b6412f7ac..f254ebb0c3 100644 --- a/ts/jobs/AttachmentDownloadManager.ts +++ b/ts/jobs/AttachmentDownloadManager.ts @@ -46,7 +46,7 @@ import { isImageTypeSupported, isVideoTypeSupported, } from '../util/GoogleChrome'; -import type { MIMEType } from '../types/MIME'; +import { IMAGE_JPEG, type MIMEType } from '../types/MIME'; import { AttachmentDownloadSource } from '../sql/Interface'; import { drop } from '../util/drop'; import { @@ -64,6 +64,7 @@ import { isPermanentlyUndownloadableWithoutBackfill, } from './helpers/attachmentBackfill'; import { formatCountForLogging } from '../logging/formatCountForLogging'; +import { strictAssert } from '../util/assert'; export { isPermanentlyUndownloadable }; @@ -648,7 +649,10 @@ export async function runDownloadAttachmentJobInner({ attachmentWithThumbnail, }; } catch (e) { - log.warn(`${logId}: error when trying to download thumbnail`); + log.warn( + `${logId}: error when trying to download thumbnail`, + Errors.toLogFormat(e) + ); } } @@ -836,9 +840,16 @@ async function downloadBackupThumbnail({ }, }); + const calculatedSize = downloadedThumbnail.size; + strictAssert(calculatedSize, 'size must be calculated for backup thumbnails'); + const attachmentWithThumbnail = { ...attachment, - thumbnailFromBackup: downloadedThumbnail, + thumbnailFromBackup: { + contentType: IMAGE_JPEG, + ...downloadedThumbnail, + size: calculatedSize, + }, }; return attachmentWithThumbnail; diff --git a/ts/messageModifiers/AttachmentDownloads.ts b/ts/messageModifiers/AttachmentDownloads.ts index 875c60b89a..d9f2978d49 100644 --- a/ts/messageModifiers/AttachmentDownloads.ts +++ b/ts/messageModifiers/AttachmentDownloads.ts @@ -1,5 +1,6 @@ // Copyright 2019 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only +import { omit } from 'lodash'; import * as log from '../logging/log'; import * as Bytes from '../Bytes'; import type { AttachmentDownloadJobTypeType } from '../types/AttachmentDownload'; @@ -340,7 +341,7 @@ export async function addAttachmentToMessage( if (thumbnail !== newThumbnail) { handledInEditHistory = true; } - return { ...item, thumbnail: newThumbnail }; + return { ...item, thumbnail: omit(newThumbnail, 'thumbnail') }; }), }, }; @@ -362,7 +363,7 @@ export async function addAttachmentToMessage( return { ...item, - thumbnail: maybeReplaceAttachment(thumbnail), + thumbnail: maybeReplaceAttachment(omit(thumbnail, 'thumbnail')), }; }), }; diff --git a/ts/messages/copyQuote.ts b/ts/messages/copyQuote.ts index b52d60db00..0faaaab5da 100644 --- a/ts/messages/copyQuote.ts +++ b/ts/messages/copyQuote.ts @@ -13,8 +13,8 @@ import { strictAssert } from '../util/assert'; import { getQuoteBodyText } from '../util/getQuoteBodyText'; import { isQuoteAMatch, messageHasPaymentEvent } from './helpers'; import * as Errors from '../types/errors'; -import { isDownloadable } from '../types/Attachment'; import type { MessageModel } from '../models/messages'; +import { isDownloadable } from '../types/Attachment'; export type MinimalMessageCache = Readonly<{ findBySentAt( @@ -77,7 +77,7 @@ export const copyQuoteContentFromOriginal = async ( { messageCache = window.MessageCache }: CopyQuoteOptionsType = {} ): Promise => { const { attachments } = quote; - const firstAttachment = attachments ? attachments[0] : undefined; + const quoteAttachment = attachments ? attachments[0] : undefined; if (messageHasPaymentEvent(message.attributes)) { // eslint-disable-next-line no-param-reassign @@ -125,7 +125,7 @@ export const copyQuoteContentFromOriginal = async ( // eslint-disable-next-line no-param-reassign quote.bodyRanges = message.attributes.bodyRanges; - if (!firstAttachment || !firstAttachment.contentType) { + if (!quoteAttachment || !quoteAttachment.contentType) { return; } @@ -150,17 +150,17 @@ export const copyQuoteContentFromOriginal = async ( if (queryAttachments.length > 0) { const queryFirst = queryAttachments[0]; - const { thumbnail } = queryFirst; + const { thumbnail: quotedThumbnail } = queryFirst; - if (thumbnail && thumbnail.path) { - firstAttachment.thumbnail = { - ...thumbnail, + if (quotedThumbnail && quotedThumbnail.path) { + quoteAttachment.thumbnail = { + ...quotedThumbnail, copied: true, }; - } else if (!firstAttachment.thumbnail || !isDownloadable(queryFirst)) { - firstAttachment.contentType = queryFirst.contentType; - firstAttachment.fileName = queryFirst.fileName; - firstAttachment.thumbnail = undefined; + } else if (!quoteAttachment.thumbnail || !isDownloadable(queryFirst)) { + quoteAttachment.contentType = queryFirst.contentType; + quoteAttachment.fileName = queryFirst.fileName; + quoteAttachment.thumbnail = undefined; } else { // there is a thumbnail, but the original message attachment has not been // downloaded yet, so we leave the quote attachment as is for now @@ -168,19 +168,17 @@ export const copyQuoteContentFromOriginal = async ( } if (queryPreview.length > 0) { - const queryFirst = queryPreview[0]; - const { image } = queryFirst; - - if (image && image.path) { - firstAttachment.thumbnail = { - ...image, + const { image: quotedPreviewImage } = queryPreview[0]; + if (quotedPreviewImage && quotedPreviewImage.path) { + quoteAttachment.thumbnail = { + ...quotedPreviewImage, copied: true, }; } } if (sticker && sticker.data && sticker.data.path) { - firstAttachment.thumbnail = { + quoteAttachment.thumbnail = { ...sticker.data, copied: true, }; diff --git a/ts/messages/handleDataMessage.ts b/ts/messages/handleDataMessage.ts index 594c62ddff..a97e9297af 100644 --- a/ts/messages/handleDataMessage.ts +++ b/ts/messages/handleDataMessage.ts @@ -554,9 +554,6 @@ export async function handleDataMessage( errors: [], flags: dataMessage.flags, giftBadge: initialMessage.giftBadge, - hasAttachments: dataMessage.hasAttachments, - hasFileAttachments: dataMessage.hasFileAttachments, - hasVisualMediaAttachments: dataMessage.hasVisualMediaAttachments, isViewOnce: Boolean(dataMessage.isViewOnce), mentionsMe: (dataMessage.bodyRanges ?? []).some(bodyRange => { if (!BodyRange.isMention(bodyRange)) { diff --git a/ts/model-types.d.ts b/ts/model-types.d.ts index b533650c69..3c4f20ba7d 100644 --- a/ts/model-types.d.ts +++ b/ts/model-types.d.ts @@ -15,11 +15,7 @@ import type { ReadStatus } from './messages/MessageReadStatus'; import type { SendStateByConversationId } from './messages/MessageSendState'; import type { GroupNameCollisionsWithIdsByTitle } from './util/groupMemberNameCollisions'; -import type { - AttachmentDraftType, - AttachmentType, - ThumbnailType, -} from './types/Attachment'; +import type { AttachmentDraftType, AttachmentType } from './types/Attachment'; import type { EmbeddedContactType } from './types/EmbeddedContact'; import { SignalService as Proto } from './protobuf'; import type { AvatarDataType, ContactAvatarType } from './types/Avatar'; @@ -82,7 +78,7 @@ export type GroupMigrationType = { export type QuotedAttachmentType = { contentType: MIMEType; fileName?: string; - thumbnail?: ThumbnailType; + thumbnail?: AttachmentType; }; export type QuotedMessageType = { @@ -186,9 +182,6 @@ export type MessageAttributesType = { expireTimer?: DurationInSeconds; groupMigration?: GroupMigrationType; group_update?: GroupV1Update; - hasAttachments?: boolean | 0 | 1; - hasFileAttachments?: boolean | 0 | 1; - hasVisualMediaAttachments?: boolean | 0 | 1; mentionsMe?: boolean | 0 | 1; isErased?: boolean; isTapToViewInvalid?: boolean; diff --git a/ts/sql/Client.ts b/ts/sql/Client.ts index 37480f6260..d5c6e1a7ae 100644 --- a/ts/sql/Client.ts +++ b/ts/sql/Client.ts @@ -49,7 +49,6 @@ import type { ItemType, StoredItemType, MessageType, - MessageTypeUnhydrated, PreKeyIdType, PreKeyType, StoredPreKeyType, @@ -62,7 +61,6 @@ import type { ClientOnlyReadableInterface, ClientOnlyWritableInterface, } from './Interface'; -import { hydrateMessage } from './hydration'; import type { MessageAttributesType } from '../model-types'; import type { AttachmentDownloadJobType } from '../types/AttachmentDownload'; @@ -546,7 +544,6 @@ function handleSearchMessageJSON( messages: Array ): Array { return messages.map(message => { - const parsedMessage = hydrateMessage(message); assertDev( message.ftsSnippet ?? typeof message.mentionStart === 'number', 'Neither ftsSnippet nor matching mention returned from message search' @@ -554,7 +551,7 @@ function handleSearchMessageJSON( const snippet = message.ftsSnippet ?? generateSnippetAroundMention({ - body: parsedMessage.body || '', + body: message.body || '', mentionStart: message.mentionStart ?? 0, mentionLength: message.mentionLength ?? 1, }); @@ -562,7 +559,7 @@ function handleSearchMessageJSON( return { // Empty array is a default value. `message.json` has the real field bodyRanges: [], - ...parsedMessage, + ...message, snippet, }; }); @@ -629,15 +626,17 @@ async function saveMessages( forceSave, ourAci, postSaveUpdates, + _testOnlyAvoidNormalizingAttachments, }: { forceSave?: boolean; ourAci: AciString; postSaveUpdates: () => Promise; + _testOnlyAvoidNormalizingAttachments?: boolean; } ): Promise> { const result = await writableChannel.saveMessages( arrayOfMessages.map(message => _cleanMessageData(message)), - { forceSave, ourAci } + { forceSave, ourAci, _testOnlyAvoidNormalizingAttachments } ); drop(postSaveUpdates?.()); @@ -730,19 +729,13 @@ async function removeMessages( await writableChannel.removeMessages(messageIds); } -function handleMessageJSON( - messages: Array -): Array { - return messages.map(message => hydrateMessage(message)); -} - async function getNewerMessagesByConversation( options: AdjacentMessagesByConversationOptionsType ): Promise> { const messages = await readableChannel.getNewerMessagesByConversation(options); - return handleMessageJSON(messages); + return messages; } async function getRecentStoryReplies( @@ -754,7 +747,7 @@ async function getRecentStoryReplies( options ); - return handleMessageJSON(messages); + return messages; } async function getOlderMessagesByConversation( @@ -763,7 +756,7 @@ async function getOlderMessagesByConversation( const messages = await readableChannel.getOlderMessagesByConversation(options); - return handleMessageJSON(messages); + return messages; } async function getConversationRangeCenteredOnMessage( @@ -772,11 +765,7 @@ async function getConversationRangeCenteredOnMessage( const result = await readableChannel.getConversationRangeCenteredOnMessage(options); - return { - ...result, - older: handleMessageJSON(result.older), - newer: handleMessageJSON(result.newer), - }; + return result; } async function removeMessagesInConversation( @@ -832,9 +821,13 @@ async function saveAttachmentDownloadJob( // Other -async function cleanupOrphanedAttachments(): Promise { +async function cleanupOrphanedAttachments({ + _block = false, +}: { + _block?: boolean; +} = {}): Promise { try { - await invokeWithTimeout(CLEANUP_ORPHANED_ATTACHMENTS_KEY); + await invokeWithTimeout(CLEANUP_ORPHANED_ATTACHMENTS_KEY, { _block }); } catch (error) { log.warn( 'sql/Client: cleanupOrphanedAttachments failure', @@ -859,9 +852,12 @@ async function removeOtherData(): Promise { ]); } -async function invokeWithTimeout(name: string): Promise { +async function invokeWithTimeout( + name: string, + ...args: Array +): Promise { return createTaskWithTimeout( - () => ipc.invoke(name), + () => ipc.invoke(name, ...args), `callChannel call to ${name}` )(); } diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 32d7a55801..773fafc2b6 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -37,7 +37,10 @@ import type { CallLinkType, DefunctCallLinkType, } from '../types/CallLink'; -import type { AttachmentDownloadJobType } from '../types/AttachmentDownload'; +import type { + AttachmentDownloadJobType, + AttachmentDownloadJobTypeType, +} from '../types/AttachmentDownload'; import type { GroupSendEndorsementsData, GroupSendMemberEndorsementRecord, @@ -46,6 +49,7 @@ import type { SyncTaskType } from '../util/syncTasks'; import type { AttachmentBackupJobType } from '../types/AttachmentBackup'; import type { GifType } from '../components/fun/panels/FunPanelGifs'; import type { NotificationProfileType } from '../types/NotificationProfile'; +import { strictAssert } from '../util/assert'; export type ReadableDB = Database & { __readable_db: never }; export type WritableDB = ReadableDB & { __writable_db: never }; @@ -217,7 +221,10 @@ export type StoredPreKeyType = PreKeyType & { privateKey: string; publicKey: string; }; -export type ServerSearchResultMessageType = MessageTypeUnhydrated & { +export type ServerSearchResultMessageType = MessageType & + ServerMessageSearchResultType; + +export type ServerMessageSearchResultType = { // If the FTS matches text in message.body, snippet will be populated ftsSnippet: string | null; @@ -537,6 +544,136 @@ export enum AttachmentDownloadSource { BACKFILL = 'backfill', } +export const MESSAGE_ATTACHMENT_COLUMNS = [ + 'messageId', + 'conversationId', + 'sentAt', + 'attachmentType', + 'orderInMessage', + 'editHistoryIndex', + 'clientUuid', + 'size', + 'contentType', + 'path', + 'localKey', + 'plaintextHash', + 'caption', + 'fileName', + 'blurHash', + 'height', + 'width', + 'digest', + 'key', + 'iv', + 'flags', + 'downloadPath', + 'transitCdnKey', + 'transitCdnNumber', + 'transitCdnUploadTimestamp', + 'backupMediaName', + 'backupCdnNumber', + 'incrementalMac', + 'incrementalMacChunkSize', + 'isReencryptableToSameDigest', + 'reencryptionIv', + 'reencryptionKey', + 'reencryptionDigest', + 'thumbnailPath', + 'thumbnailSize', + 'thumbnailContentType', + 'thumbnailLocalKey', + 'thumbnailVersion', + 'screenshotPath', + 'screenshotSize', + 'screenshotContentType', + 'screenshotLocalKey', + 'screenshotVersion', + 'backupThumbnailPath', + 'backupThumbnailSize', + 'backupThumbnailContentType', + 'backupThumbnailLocalKey', + 'backupThumbnailVersion', + 'storyTextAttachmentJson', + 'localBackupPath', + 'isCorrupted', + 'backfillError', + 'error', + 'wasTooBig', + 'copiedFromQuotedAttachment', + 'version', + 'pending', +] as const satisfies Array; + +export type MessageAttachmentDBType = { + messageId: string; + attachmentType: AttachmentDownloadJobTypeType; + orderInMessage: number; + editHistoryIndex: number | null; + conversationId: string; + sentAt: number; + clientUuid: string | null; + size: number; + contentType: string; + path: string | null; + plaintextHash: string | null; + downloadPath: string | null; + caption: string | null; + blurHash: string | null; + width: number | null; + height: number | null; + flags: number | null; + key: string | null; + iv: string | null; + digest: string | null; + fileName: string | null; + incrementalMac: string | null; + incrementalMacChunkSize: number | null; + localKey: string | null; + version: 1 | 2 | null; + transitCdnKey: string | null; + transitCdnNumber: number | null; + transitCdnUploadTimestamp: number | null; + backupMediaName: string | null; + backupCdnNumber: number | null; + thumbnailPath: string | null; + thumbnailSize: number | null; + thumbnailContentType: string | null; + thumbnailLocalKey: string | null; + thumbnailVersion: 1 | 2 | null; + screenshotPath: string | null; + screenshotSize: number | null; + screenshotContentType: string | null; + screenshotLocalKey: string | null; + screenshotVersion: 1 | 2 | null; + backupThumbnailPath: string | null; + backupThumbnailSize: number | null; + backupThumbnailContentType: string | null; + backupThumbnailLocalKey: string | null; + backupThumbnailVersion: 1 | 2 | null; + reencryptionIv: string | null; + reencryptionKey: string | null; + reencryptionDigest: string | null; + storyTextAttachmentJson: string | null; + localBackupPath: string | null; + isCorrupted: 1 | 0 | null; + backfillError: 1 | 0 | null; + error: 1 | 0 | null; + wasTooBig: 1 | 0 | null; + pending: 1 | 0 | null; + isReencryptableToSameDigest: 1 | 0 | null; + copiedFromQuotedAttachment: 1 | 0 | null; +}; + +// Test to make sure that MESSAGE_ATTACHMENT_COLUMNS & +// MessageAttachmentDBReferenceType remain in sync! +const testDBRefTypeMatchesColumnNames = true as unknown as [ + keyof MessageAttachmentDBType, +] satisfies [(typeof MESSAGE_ATTACHMENT_COLUMNS)[number]]; +strictAssert( + testDBRefTypeMatchesColumnNames, + 'attachment_columns must match DB fields type' +); + type ReadableInterface = { close: () => void; @@ -744,6 +881,9 @@ type ReadableInterface = { getStatisticsForLogging(): Record; getSizeOfPendingBackupAttachmentDownloadJobs(): number; + getAttachmentReferencesForMessages: ( + messageIds: Array + ) => Array; }; type WritableInterface = { @@ -866,6 +1006,7 @@ type WritableInterface = { ) => void; _removeAllReactions: () => void; _removeAllMessages: () => void; + _removeMessage: (id: string) => void; incrementMessagesMigrationAttempts: ( messageIds: ReadonlyArray ) => void; @@ -1072,16 +1213,16 @@ export type ServerReadableDirectInterface = ReadableInterface & { getRecentStoryReplies( storyId: string, options?: GetRecentStoryRepliesOptionsType - ): Array; + ): Array; getOlderMessagesByConversation: ( options: AdjacentMessagesByConversationOptionsType - ) => Array; + ) => Array; getNewerMessagesByConversation: ( options: AdjacentMessagesByConversationOptionsType - ) => Array; + ) => Array; getConversationRangeCenteredOnMessage: ( options: AdjacentMessagesByConversationOptionsType - ) => GetConversationRangeCenteredOnMessageResultType; + ) => GetConversationRangeCenteredOnMessageResultType; getIdentityKeyById: ( id: IdentityKeyIdType @@ -1141,7 +1282,11 @@ export type ServerWritableDirectInterface = WritableInterface & { ) => string; saveMessages: ( arrayOfMessages: ReadonlyArray>, - options: { forceSave?: boolean; ourAci: AciString } + options: { + forceSave?: boolean; + ourAci: AciString; + _testOnlyAvoidNormalizingAttachments?: boolean; + } ) => Array; saveMessagesIndividually: ( arrayOfMessages: ReadonlyArray>, @@ -1241,6 +1386,7 @@ export type ClientOnlyWritableInterface = ClientInterfaceWrap<{ forceSave?: boolean; ourAci: AciString; postSaveUpdates: () => Promise; + _testOnlyAvoidNormalizingAttachments?: boolean; } ) => string; saveMessages: ( @@ -1249,6 +1395,7 @@ export type ClientOnlyWritableInterface = ClientInterfaceWrap<{ forceSave?: boolean; ourAci: AciString; postSaveUpdates: () => Promise; + _testOnlyAvoidNormalizingAttachments?: boolean; } ) => Array; saveMessagesIndividually: ( @@ -1311,7 +1458,7 @@ export type ClientOnlyWritableInterface = ClientInterfaceWrap<{ } ) => void; removeOtherData: () => void; - cleanupOrphanedAttachments: () => void; + cleanupOrphanedAttachments: (options?: { _block: boolean }) => void; ensureFilePermissions: () => void; }>; diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index b979d6c3a8..8ce1906274 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -38,9 +38,10 @@ import type { BadgeImageType, BadgeType } from '../badges/types'; import type { StoredJob } from '../jobs/types'; import { formatCountForLogging } from '../logging/formatCountForLogging'; import { ReadStatus } from '../messages/MessageReadStatus'; -import type { GroupV2MemberType } from '../model-types.d'; -import type { ConversationColorType, CustomColorType } from '../types/Colors'; -import type { LoggerType } from '../types/Logging'; +import type { + GroupV2MemberType, + MessageAttributesType, +} from '../model-types.d'; import type { ReactionType } from '../types/Reactions'; import { ReactionReadStatus } from '../types/Reactions'; import type { AciString, ServiceIdString } from '../types/ServiceId'; @@ -51,9 +52,7 @@ import * as Errors from '../types/errors'; import { assertDev, strictAssert } from '../util/assert'; import { combineNames } from '../util/combineNames'; import { consoleLogger } from '../util/consoleLogger'; -import { dropNull } from '../util/dropNull'; -import * as durations from '../util/durations'; -import { generateMessageId } from '../util/generateMessageId'; +import { dropNull, shallowConvertUndefinedToNull } from '../util/dropNull'; import { isNormalNumber } from '../util/isNormalNumber'; import { isNotNil } from '../util/isNotNil'; import { parseIntOrThrow } from '../util/parseIntOrThrow'; @@ -78,8 +77,14 @@ import { sqlFragment, sqlJoin, QueryFragment, + convertOptionalBooleanToNullableInteger, } from './util'; -import { hydrateMessage } from './hydration'; +import { + hydrateMessage, + hydrateMessages, + getAttachmentReferencesForMessages, + ROOT_MESSAGE_ATTACHMENT_EDIT_HISTORY_INDEX, +} from './hydration'; import { SeenStatus } from '../MessageSeenStatus'; import { @@ -88,6 +93,7 @@ import { } from '../types/AttachmentBackup'; import { attachmentDownloadJobSchema, + type AttachmentDownloadJobTypeType, type AttachmentDownloadJobType, } from '../types/AttachmentDownload'; import type { @@ -139,7 +145,6 @@ import type { MessageCursorType, MessageMetricsType, MessageType, - MessageTypeUnhydrated, PageMessagesCursorType, PageMessagesResultType, PreKeyIdType, @@ -178,8 +183,15 @@ import type { UninstalledStickerPackType, UnprocessedType, WritableDB, + MessageAttachmentDBType, + MessageTypeUnhydrated, + ServerMessageSearchResultType, +} from './Interface'; +import { + AttachmentDownloadSource, + MESSAGE_COLUMNS, + MESSAGE_ATTACHMENT_COLUMNS, } from './Interface'; -import { AttachmentDownloadSource, MESSAGE_COLUMNS } from './Interface'; import { _removeAllCallLinks, beginDeleteAllCallLinks, @@ -214,6 +226,15 @@ import { import { INITIAL_EXPIRE_TIMER_VERSION } from '../util/expirationTimer'; import type { GifType } from '../components/fun/panels/FunPanelGifs'; import type { NotificationProfileType } from '../types/NotificationProfile'; +import * as durations from '../util/durations'; +import { + isFile, + isVisualMedia, + type AttachmentType, +} from '../types/Attachment'; +import { generateMessageId } from '../util/generateMessageId'; +import type { ConversationColorType, CustomColorType } from '../types/Colors'; +import { sqlLogger } from './sqlLogger'; type ConversationRow = Readonly<{ json: string; @@ -412,6 +433,7 @@ export const DataReader: ServerReadableInterface = { getBackupCdnObjectMetadata, getSizeOfPendingBackupAttachmentDownloadJobs, + getAttachmentReferencesForMessages, // Server-only getKnownMessageAttachments, @@ -497,6 +519,7 @@ export const DataWriter: ServerWritableInterface = { removeReactionFromConversation, _removeAllReactions, _removeAllMessages, + _removeMessage: removeMessage, getUnreadEditedMessagesAndMarkRead, clearCallHistory, _removeAllCallHistory, @@ -773,7 +796,7 @@ function openAndSetUpSQLCipher(filePath: string, { key }: { key: string }) { return db; } -let logger = consoleLogger; +let logger = sqlLogger; let databaseFilePath: string | undefined; let indexedDBPath: string | undefined; @@ -781,13 +804,11 @@ export function initialize({ configDir, key, isPrimary, - logger: suppliedLogger, }: { appVersion: string; configDir: string; key: string; isPrimary: boolean; - logger: LoggerType; }): WritableDB { if (!isString(configDir)) { throw new Error('initialize: configDir is required!'); @@ -796,8 +817,6 @@ export function initialize({ throw new Error('initialize: key is required!'); } - logger = suppliedLogger; - indexedDBPath = join(configDir, 'IndexedDB'); const dbDir = join(configDir, 'sql'); @@ -1996,11 +2015,13 @@ function searchMessages( LIMIT ${limit} `; - let result: Array; + let queryResult: Array< + ServerMessageSearchResultType & MessageTypeUnhydrated + >; if (!contactServiceIdsMatchingQuery?.length) { const [sqlQuery, params] = sql`${ftsFragment};`; - result = writable.prepare(sqlQuery).all(params); + queryResult = writable.prepare(sqlQuery).all(params); } else { const coalescedColumns = MESSAGE_COLUMNS_FRAGMENTS.map( name => sqlFragment` @@ -2047,7 +2068,7 @@ function searchMessages( ORDER BY received_at DESC, sent_at DESC LIMIT ${limit}; `; - result = writable.prepare(sqlQuery).all(params); + queryResult = writable.prepare(sqlQuery).all(params); } writable.exec( @@ -2056,7 +2077,16 @@ function searchMessages( DROP TABLE tmp_filtered_results; ` ); - return result; + const hydrated = hydrateMessages(db, queryResult); + return queryResult.map((row, idx) => { + return { + ...hydrated[idx], + ftsSnippet: row.ftsSnippet, + mentionAci: row.mentionAci, + mentionStart: row.mentionStart, + mentionLength: row.mentionLength, + }; + }); })(); } @@ -2129,7 +2159,8 @@ export function getMostRecentAddressableMessages( conversationId: string, limit = 5 ): Array { - const [query, parameters] = sql` + return db.transaction(() => { + const [query, parameters] = sql` SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} FROM messages @@ -2141,9 +2172,10 @@ export function getMostRecentAddressableMessages( LIMIT ${limit}; `; - const rows = db.prepare(query).all(parameters); + const rows = db.prepare(query).all(parameters); - return rows.map(row => hydrateMessage(row)); + return hydrateMessages(db, rows); + })(); } export function getMostRecentAddressableNondisappearingMessages( @@ -2151,7 +2183,8 @@ export function getMostRecentAddressableNondisappearingMessages( conversationId: string, limit = 5 ): Array { - const [query, parameters] = sql` + return db.transaction(() => { + const [query, parameters] = sql` SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} FROM messages @@ -2164,9 +2197,10 @@ export function getMostRecentAddressableNondisappearingMessages( LIMIT ${limit}; `; - const rows = db.prepare(query).all(parameters); + const rows = db.prepare(query).all(parameters); - return rows.map(row => hydrateMessage(row)); + return hydrateMessages(db, rows); + })(); } export function removeSyncTaskById(db: WritableDB, id: string): void { @@ -2342,22 +2376,290 @@ export function dequeueOldestSyncTasks( })(); } -export function saveMessage( +function saveMessageAttachmentsForRootOrEditedVersion( db: WritableDB, - data: ReadonlyDeep, + message: { + id: string; + conversationId: string; + sent_at: number; + } & Pick< + MessageAttributesType, + | 'attachments' + | 'bodyAttachment' + | 'contact' + | 'preview' + | 'quote' + | 'sticker' + >, + { editHistoryIndex }: { editHistoryIndex: number | null } +) { + const { id: messageId, conversationId, sent_at: sentAt } = message; + + const mainAttachments = message.attachments; + if (mainAttachments) { + for (let i = 0; i < mainAttachments.length; i += 1) { + const attachment = mainAttachments[i]; + saveMessageAttachment({ + db, + messageId, + conversationId, + sentAt, + attachmentType: 'attachment', + attachment, + orderInMessage: i, + editHistoryIndex, + }); + } + } + + const { bodyAttachment } = message; + if (bodyAttachment) { + saveMessageAttachment({ + db, + messageId, + conversationId, + sentAt, + attachmentType: 'long-message', + attachment: bodyAttachment, + orderInMessage: 0, + editHistoryIndex, + }); + } + + const previewAttachments = message.preview?.map(preview => preview.image); + if (previewAttachments) { + for (let i = 0; i < previewAttachments.length; i += 1) { + const attachment = previewAttachments[i]; + if (!attachment) { + continue; + } + saveMessageAttachment({ + db, + messageId, + conversationId, + sentAt, + attachmentType: 'preview', + attachment, + orderInMessage: i, + editHistoryIndex, + }); + } + } + + const quoteAttachments = message.quote?.attachments; + if (quoteAttachments) { + for (let i = 0; i < quoteAttachments.length; i += 1) { + const attachment = quoteAttachments[i]; + if (!attachment?.thumbnail) { + continue; + } + saveMessageAttachment({ + db, + messageId, + conversationId, + sentAt, + attachmentType: 'quote', + attachment: attachment.thumbnail, + orderInMessage: i, + editHistoryIndex, + }); + } + } + + const contactAttachments = message.contact?.map( + contact => contact.avatar?.avatar + ); + if (contactAttachments) { + for (let i = 0; i < contactAttachments.length; i += 1) { + const attachment = contactAttachments[i]; + if (!attachment) { + continue; + } + saveMessageAttachment({ + db, + messageId, + conversationId, + sentAt, + attachmentType: 'contact', + attachment, + orderInMessage: i, + editHistoryIndex, + }); + } + } + + const stickerAttachment = message.sticker?.data; + if (stickerAttachment) { + saveMessageAttachment({ + db, + messageId, + conversationId, + sentAt, + attachmentType: 'sticker', + attachment: stickerAttachment, + orderInMessage: 0, + editHistoryIndex, + }); + } +} +function saveMessageAttachments( + db: WritableDB, + message: ReadonlyDeep +) { + const messageId = message.id; + const [deleteQuery, deleteParams] = sql` + DELETE FROM message_attachments + WHERE messageId = ${messageId}; + `; + db.prepare(deleteQuery).run(deleteParams); + + saveMessageAttachmentsForRootOrEditedVersion(db, message, { + editHistoryIndex: null, + }); + + message.editHistory?.forEach((editHistory, idx) => { + saveMessageAttachmentsForRootOrEditedVersion( + db, + { + id: message.id, + conversationId: message.conversationId, + sent_at: editHistory.timestamp, + ...editHistory, + }, + { editHistoryIndex: idx } + ); + }); +} + +function saveMessageAttachment({ + db, + messageId, + conversationId, + sentAt, + attachmentType, + attachment, + orderInMessage, + editHistoryIndex, +}: { + db: WritableDB; + messageId: string; + conversationId: string; + sentAt: number; + attachmentType: AttachmentDownloadJobTypeType; + attachment: AttachmentType; + orderInMessage: number; + editHistoryIndex: number | null; +}) { + const values: MessageAttachmentDBType = shallowConvertUndefinedToNull({ + messageId, + editHistoryIndex: + editHistoryIndex ?? ROOT_MESSAGE_ATTACHMENT_EDIT_HISTORY_INDEX, + attachmentType, + orderInMessage, + conversationId, + sentAt, + clientUuid: attachment.clientUuid, + size: attachment.size, + contentType: attachment.contentType, + path: attachment.path, + localKey: attachment.localKey, + plaintextHash: attachment.plaintextHash, + caption: attachment.caption, + blurHash: attachment.blurHash, + height: attachment.height, + width: attachment.width, + digest: attachment.digest, + key: attachment.key, + iv: attachment.iv, + fileName: attachment.fileName, + downloadPath: attachment.downloadPath, + transitCdnKey: attachment.cdnKey ?? attachment.cdnId, + transitCdnNumber: attachment.cdnNumber, + transitCdnUploadTimestamp: isNumber(attachment.uploadTimestamp) + ? attachment.uploadTimestamp + : null, + backupMediaName: attachment.backupLocator?.mediaName, + backupCdnNumber: attachment.backupLocator?.cdnNumber, + incrementalMac: attachment.incrementalMac, + incrementalMacChunkSize: attachment.chunkSize, + isReencryptableToSameDigest: convertOptionalBooleanToNullableInteger( + attachment.isReencryptableToSameDigest + ), + reencryptionIv: + attachment.isReencryptableToSameDigest === false + ? attachment.reencryptionInfo?.iv + : null, + reencryptionKey: + attachment.isReencryptableToSameDigest === false + ? attachment.reencryptionInfo?.key + : null, + reencryptionDigest: + attachment.isReencryptableToSameDigest === false + ? attachment.reencryptionInfo?.digest + : null, + thumbnailPath: attachment.thumbnail?.path, + thumbnailSize: attachment.thumbnail?.size, + thumbnailContentType: attachment.thumbnail?.contentType, + thumbnailLocalKey: attachment.thumbnail?.localKey, + thumbnailVersion: attachment.thumbnail?.version, + screenshotPath: attachment.screenshot?.path, + screenshotSize: attachment.screenshot?.size, + screenshotContentType: attachment.screenshot?.contentType, + screenshotLocalKey: attachment.screenshot?.localKey, + screenshotVersion: attachment.screenshot?.version, + backupThumbnailPath: attachment.thumbnailFromBackup?.path, + backupThumbnailSize: attachment.thumbnailFromBackup?.size, + backupThumbnailContentType: attachment.thumbnailFromBackup?.contentType, + backupThumbnailLocalKey: attachment.thumbnailFromBackup?.localKey, + backupThumbnailVersion: attachment.thumbnailFromBackup?.version, + storyTextAttachmentJson: attachment.textAttachment + ? objectToJSON(attachment.textAttachment) + : null, + localBackupPath: attachment.localBackupPath, + flags: attachment.flags, + error: convertOptionalBooleanToNullableInteger(attachment.error), + wasTooBig: convertOptionalBooleanToNullableInteger(attachment.wasTooBig), + backfillError: convertOptionalBooleanToNullableInteger( + attachment.backfillError + ), + isCorrupted: convertOptionalBooleanToNullableInteger( + attachment.isCorrupted + ), + copiedFromQuotedAttachment: + 'copied' in attachment + ? convertOptionalBooleanToNullableInteger(attachment.copied) + : null, + version: attachment.version, + pending: convertOptionalBooleanToNullableInteger(attachment.pending), + }); + + db.prepare( + ` + INSERT OR REPLACE INTO message_attachments + (${MESSAGE_ATTACHMENT_COLUMNS.join(', ')}) + VALUES + (${MESSAGE_ATTACHMENT_COLUMNS.map(name => `$${name}`).join(', ')}) + RETURNING rowId; + ` + ).run(values); +} + +function saveMessage( + db: WritableDB, + message: ReadonlyDeep, options: { alreadyInTransaction?: boolean; forceSave?: boolean; jobToInsert?: StoredJob; ourAci: AciString; + _testOnlyAvoidNormalizingAttachments?: boolean; } ): string { // NB: `saveMessagesIndividually` relies on `saveMessage` being atomic const { alreadyInTransaction, forceSave, jobToInsert, ourAci } = options; - if (!alreadyInTransaction) { return db.transaction(() => { - return saveMessage(db, data, { + return saveMessage(db, message, { ...options, alreadyInTransaction: true, }); @@ -2367,9 +2669,6 @@ export function saveMessage( const { body, conversationId, - hasAttachments, - hasFileAttachments, - hasVisualMediaAttachments, id, isErased, isViewOnce, @@ -2393,10 +2692,10 @@ export function saveMessage( unidentifiedDeliveryReceived, ...json - } = data; + } = message; // Extracted separately since we store this field in JSON - const { attachments, groupV2Change } = data; + const { attachments, groupV2Change } = message; let seenStatus = originalSeenStatus; @@ -2419,23 +2718,91 @@ export function saveMessage( ); // eslint-disable-next-line no-param-reassign - data = { - ...data, + message = { + ...message, seenStatus: SeenStatus.Unseen, }; seenStatus = SeenStatus.Unseen; } + const dataToSaveAsJSON = { ...json }; + + const hasRequiredFields = conversationId != null && sent_at != null; + if (!hasRequiredFields) { + logger.error( + 'saveMessage: saving message without conversationId or sent_at!', + { conversationId, sent_at } + ); + } + + const normalizeAttachmentData = + hasRequiredFields && options._testOnlyAvoidNormalizingAttachments !== true; + + if (normalizeAttachmentData) { + // Remove attachments from json data + delete dataToSaveAsJSON.attachments; + delete dataToSaveAsJSON.bodyAttachment; + delete dataToSaveAsJSON.preview; + delete dataToSaveAsJSON.quote; + delete dataToSaveAsJSON.contact; + delete dataToSaveAsJSON.sticker; + delete dataToSaveAsJSON.editHistory; + + dataToSaveAsJSON.preview = message.preview?.map(preview => + omit(preview, 'image') + ); + dataToSaveAsJSON.quote = message.quote + ? { + ...message.quote, + attachments: message.quote.attachments.map(quoteAttachment => + omit(quoteAttachment, 'thumbnail') + ), + } + : undefined; + dataToSaveAsJSON.contact = message.contact?.map(contact => ({ + ...contact, + avatar: omit(contact.avatar, 'avatar'), + })); + dataToSaveAsJSON.sticker = message.sticker + ? omit(message.sticker, 'data') + : undefined; + + dataToSaveAsJSON.editHistory = message.editHistory?.map(editHistory => { + const editHistoryWithoutAttachments = { ...editHistory }; + delete editHistoryWithoutAttachments.attachments; + delete editHistoryWithoutAttachments.bodyAttachment; + editHistoryWithoutAttachments.quote = editHistory.quote + ? { + ...editHistory.quote, + attachments: editHistory.quote.attachments.map(quoteAttachment => + omit(quoteAttachment, 'thumbnail') + ), + } + : undefined; + editHistoryWithoutAttachments.preview = editHistory.preview?.map( + preview => omit(preview, 'image') + ); + + return editHistoryWithoutAttachments; + }); + } + + const downloadedAttachments = message.attachments?.filter( + attachment => attachment.path != null + ); + const payloadWithoutJson = { id, - body: body || null, conversationId, expirationStartTimestamp: expirationStartTimestamp || null, expireTimer: expireTimer || null, - hasAttachments: hasAttachments ? 1 : 0, - hasFileAttachments: hasFileAttachments ? 1 : 0, - hasVisualMediaAttachments: hasVisualMediaAttachments ? 1 : 0, + // TODO (DESKTOP-8711) + hasAttachments: (downloadedAttachments?.length ?? 0) > 0 ? 1 : 0, + hasFileAttachments: downloadedAttachments?.some(isFile) ? 1 : 0, + hasVisualMediaAttachments: downloadedAttachments?.some(isVisualMedia) + ? 1 + : 0, isChangeCreatedByUs: groupV2Change?.from === ourAci ? 1 : 0, isErased: isErased ? 1 : 0, isViewOnce: isViewOnce ? 1 : 0, @@ -2464,16 +2831,19 @@ export function saveMessage( ${MESSAGE_COLUMNS.map(name => `${name} = $${name}`).join(', ')} WHERE id = $id; ` - ).run({ ...payloadWithoutJson, json: objectToJSON(json) }); + ).run({ ...payloadWithoutJson, json: objectToJSON(dataToSaveAsJSON) }); if (jobToInsert) { insertJob(db, jobToInsert); } + if (normalizeAttachmentData) { + saveMessageAttachments(db, message); + } return id; } - const createdId = id || generateMessageId(data.received_at).id; + const createdId = id || generateMessageId(message.received_at).id; db.prepare( ` @@ -2486,9 +2856,13 @@ export function saveMessage( ).run({ ...payloadWithoutJson, id: createdId, - json: objectToJSON(json), + json: objectToJSON(dataToSaveAsJSON), }); + if (normalizeAttachmentData) { + saveMessageAttachments(db, message); + } + if (jobToInsert) { insertJob(db, jobToInsert); } @@ -2499,7 +2873,11 @@ export function saveMessage( function saveMessages( db: WritableDB, arrayOfMessages: ReadonlyArray>, - options: { forceSave?: boolean; ourAci: AciString } + options: { + forceSave?: boolean; + ourAci: AciString; + _testOnlyAvoidNormalizingAttachments?: boolean; + } ): Array { return db.transaction(() => { const result = new Array(); @@ -2568,60 +2946,70 @@ export function getMessageById( db: ReadableDB, id: string ): MessageType | undefined { - const row = db - .prepare( - ` + return db.transaction(() => { + const row = db + .prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE id = $id; ` - ) - .get({ - id, - }); + ) + .get({ + id, + }); - if (!row) { - return undefined; - } + if (!row) { + return undefined; + } - return hydrateMessage(row); + return hydrateMessage(db, row); + })(); } function getMessagesById( db: ReadableDB, messageIds: ReadonlyArray ): Array { - return batchMultiVarQuery( - db, - messageIds, - (batch: ReadonlyArray, persistent: boolean): Array => { - const query = db.prepare( - ` + return db.transaction(() => + batchMultiVarQuery( + db, + messageIds, + ( + batch: ReadonlyArray, + persistent: boolean + ): Array => { + const query = db.prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE id IN ( ${Array(batch.length).fill('?').join(',')} );`, - { persistent } - ); - const rows: Array = query.all(batch); - return rows.map(row => hydrateMessage(row)); - } - ); + { persistent } + ); + const rows: Array = query.all(batch); + return hydrateMessages(db, rows); + } + ) + )(); } function _getAllMessages(db: ReadableDB): Array { - const rows: Array = db - .prepare( - ` + return db.transaction(() => { + const rows: Array = db + .prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages ORDER BY id ASC ` - ) - .all(); + ) + .all(); - return rows.map(row => hydrateMessage(row)); + return hydrateMessages(db, rows); + })(); } + function _removeAllMessages(db: WritableDB): void { db.exec(` DELETE FROM messages; @@ -2651,37 +3039,39 @@ function getMessageBySender( sent_at: number; } ): MessageType | undefined { - const rows: Array = db - .prepare( - ` + return db.transaction(() => { + const rows: Array = db + .prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE (source = $source OR sourceServiceId = $sourceServiceId) AND sourceDevice = $sourceDevice AND sent_at = $sent_at LIMIT 2; ` - ) - .all({ - source: source || null, - sourceServiceId: sourceServiceId || null, - sourceDevice: sourceDevice || null, - sent_at, - }); + ) + .all({ + source: source || null, + sourceServiceId: sourceServiceId || null, + sourceDevice: sourceDevice || null, + sent_at, + }); - if (rows.length > 1) { - logger.warn('getMessageBySender: More than one message found for', { - sent_at, - source, - sourceServiceId, - sourceDevice, - }); - } + if (rows.length > 1) { + logger.warn('getMessageBySender: More than one message found for', { + sent_at, + source, + sourceServiceId, + sourceDevice, + }); + } - if (rows.length < 1) { - return undefined; - } + if (rows.length < 1) { + return undefined; + } - return hydrateMessage(rows[0]); + return hydrateMessage(db, rows[0]); + })(); } export function _storyIdPredicate( @@ -2781,14 +3171,12 @@ function getUnreadByConversationAndMarkRead( `; db.prepare(updateStatusQuery).run(updateStatusParams); - - return rows.map(row => { - const json = hydrateMessage(row); + return hydrateMessages(db, rows).map(msg => { return { - originalReadStatus: json.readStatus, + originalReadStatus: msg.readStatus, readStatus: ReadStatus.Read, seenStatus: SeenStatus.Seen, - ...pick(json, [ + ...pick(msg, [ 'expirationStartTimestamp', 'id', 'sent_at', @@ -3009,7 +3397,7 @@ function getRecentStoryReplies( receivedAt = Number.MAX_VALUE, sentAt = Number.MAX_VALUE, }: GetRecentStoryRepliesOptionsType = {} -): Array { +): Array { const timeFilters = { first: sqlFragment`received_at = ${receivedAt} AND sent_at < ${sentAt}`, second: sqlFragment`received_at < ${receivedAt}`, @@ -3037,7 +3425,10 @@ function getRecentStoryReplies( const [query, params] = sql`${template} LIMIT ${limit}`; - return db.prepare(query).all(params); + return db.transaction(() => { + const rows: Array = db.prepare(query).all(params); + return hydrateMessages(db, rows); + })(); } function getAdjacentMessagesByConversation( @@ -3054,7 +3445,7 @@ function getAdjacentMessagesByConversation( requireFileAttachments, storyId, }: AdjacentMessagesByConversationOptionsType -): Array { +): Array { let timeFilters: { first: QueryFragment; second: QueryFragment }; let timeOrder: QueryFragment; @@ -3105,61 +3496,28 @@ function getAdjacentMessagesByConversation( ORDER BY received_at ${timeOrder}, sent_at ${timeOrder} `; - let template = sqlFragment` + const [query, params] = sql` SELECT first.* FROM (${createQuery(timeFilters.first)}) as first UNION ALL SELECT second.* FROM (${createQuery(timeFilters.second)}) as second + LIMIT ${limit} `; - // See `filterValidAttachments` in ts/state/ducks/lightbox.ts - if (requireVisualMediaAttachments) { - template = sqlFragment` - SELECT messages.* - FROM (${template}) as messages - WHERE - ( - SELECT COUNT(*) - FROM json_each(messages.json ->> 'attachments') AS attachment - WHERE - attachment.value ->> 'thumbnail' IS NOT NULL AND - attachment.value ->> 'pending' IS NOT 1 AND - attachment.value ->> 'error' IS NULL - ) > 0 - LIMIT ${limit}; - `; - } else if (requireFileAttachments) { - template = sqlFragment` - SELECT messages.* - FROM (${template}) as messages - WHERE - ( - SELECT COUNT(*) - FROM json_each(messages.json ->> 'attachments') AS attachment - WHERE - attachment.value ->> 'pending' IS NOT 1 AND - attachment.value ->> 'error' IS NULL - ) > 0 - LIMIT ${limit}; - `; - } else { - template = sqlFragment`${template} LIMIT ${limit}`; - } + return db.transaction(() => { + const results = db.prepare(query).all(params); - const [query, params] = sql`${template}`; + if (direction === AdjacentDirection.Older) { + results.reverse(); + } - const results = db.prepare(query).all(params); - - if (direction === AdjacentDirection.Older) { - results.reverse(); - } - - return results; + return hydrateMessages(db, results); + })(); } function getOlderMessagesByConversation( db: ReadableDB, options: AdjacentMessagesByConversationOptionsType -): Array { +): Array { return getAdjacentMessagesByConversation( db, AdjacentDirection.Older, @@ -3177,7 +3535,8 @@ function getAllStories( sourceServiceId?: ServiceIdString; } ): GetAllStoriesResultType { - const [storiesQuery, storiesParams] = sql` + return db.transaction(() => { + const [storiesQuery, storiesParams] = sql` SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} FROM messages WHERE @@ -3188,20 +3547,20 @@ function getAllStories( sourceServiceId IS ${sourceServiceId ?? null}) ORDER BY received_at ASC, sent_at ASC; `; - const rows = db - .prepare(storiesQuery) - .all(storiesParams); + const rows = db + .prepare(storiesQuery) + .all(storiesParams); - const [repliesQuery, repliesParams] = sql` + const [repliesQuery, repliesParams] = sql` SELECT DISTINCT storyId FROM messages WHERE storyId IS NOT NULL `; - const replies = db - .prepare(repliesQuery, { pluck: true }) - .all(repliesParams); + const replies = db + .prepare(repliesQuery, { pluck: true }) + .all(repliesParams); - const [repliesFromSelfQuery, repliesFromSelfParams] = sql` + const [repliesFromSelfQuery, repliesFromSelfParams] = sql` SELECT DISTINCT storyId FROM messages WHERE ( @@ -3209,26 +3568,27 @@ function getAllStories( type IS 'outgoing' ) `; - const repliesFromSelf = db - .prepare(repliesFromSelfQuery, { - pluck: true, - }) - .all(repliesFromSelfParams); + const repliesFromSelf = db + .prepare(repliesFromSelfQuery, { + pluck: true, + }) + .all(repliesFromSelfParams); - const repliesLookup = new Set(replies); - const repliesFromSelfLookup = new Set(repliesFromSelf); + const repliesLookup = new Set(replies); + const repliesFromSelfLookup = new Set(repliesFromSelf); - return rows.map(row => ({ - ...hydrateMessage(row), - hasReplies: Boolean(repliesLookup.has(row.id)), - hasRepliesFromSelf: Boolean(repliesFromSelfLookup.has(row.id)), - })); + return hydrateMessages(db, rows).map(msg => ({ + ...msg, + hasReplies: Boolean(repliesLookup.has(msg.id)), + hasRepliesFromSelf: Boolean(repliesFromSelfLookup.has(msg.id)), + })); + })(); } function getNewerMessagesByConversation( db: ReadableDB, options: AdjacentMessagesByConversationOptionsType -): Array { +): Array { return getAdjacentMessagesByConversation( db, AdjacentDirection.Newer, @@ -3403,9 +3763,10 @@ function getLastConversationActivity( includeStoryReplies: boolean; } ): MessageType | undefined { - const row = db - .prepare( - ` + return db.transaction(() => { + const row = db + .prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages INDEXED BY messages_activity WHERE @@ -3417,16 +3778,17 @@ function getLastConversationActivity( ORDER BY received_at DESC, sent_at DESC LIMIT 1; ` - ) - .get({ - conversationId, - }); + ) + .get({ + conversationId, + }); - if (!row) { - return undefined; - } + if (!row) { + return undefined; + } - return hydrateMessage(row); + return hydrateMessage(db, row); + })(); } function getLastConversationPreview( db: ReadableDB, @@ -3442,9 +3804,10 @@ function getLastConversationPreview( ? 'messages_preview' : 'messages_preview_without_story'; - const row: MessageTypeUnhydrated | undefined = db - .prepare( - ` + return db.transaction(() => { + const row: MessageTypeUnhydrated | undefined = db + .prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')}, expiresAt FROM ( SELECT ${MESSAGE_COLUMNS.join(', ')}, expiresAt FROM messages INDEXED BY ${index} @@ -3458,13 +3821,14 @@ function getLastConversationPreview( WHERE likely(expiresAt > $now) LIMIT 1 ` - ) - .get({ - conversationId, - now: Date.now(), - }); + ) + .get({ + conversationId, + now: Date.now(), + }); - return row ? hydrateMessage(row) : undefined; + return row ? hydrateMessage(db, row) : undefined; + })(); } function getConversationMessageStats( @@ -3500,24 +3864,26 @@ function getLastConversationMessage( conversationId: string; } ): MessageType | undefined { - const row = db - .prepare( - ` + return db.transaction(() => { + const row = db + .prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE conversationId = $conversationId ORDER BY received_at DESC, sent_at DESC LIMIT 1; ` - ) - .get({ - conversationId, - }); + ) + .get({ + conversationId, + }); - if (!row) { - return undefined; - } + if (!row) { + return undefined; + } - return hydrateMessage(row); + return hydrateMessage(db, row); + })(); } function getOldestUnseenMessageForConversation( @@ -3693,7 +4059,7 @@ function getMessageMetricsForConversation( function getConversationRangeCenteredOnMessage( db: ReadableDB, options: AdjacentMessagesByConversationOptionsType -): GetConversationRangeCenteredOnMessageResultType { +): GetConversationRangeCenteredOnMessageResultType { return db.transaction(() => { return { older: getAdjacentMessagesByConversation( @@ -3852,18 +4218,20 @@ function getCallHistoryMessageByCallId( callId: string; } ): MessageType | undefined { - const [query, params] = sql` + return db.transaction(() => { + const [query, params] = sql` SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} FROM messages WHERE conversationId = ${options.conversationId} AND type = 'call-history' AND callId = ${options.callId} `; - const row = db.prepare(query).get(params); - if (row == null) { - return; - } - return hydrateMessage(row); + const row = db.prepare(query).get(params); + if (row == null) { + return; + } + return hydrateMessage(db, row); + })(); } function getCallHistory( @@ -4666,15 +5034,16 @@ function getMessagesBySentAt( db: ReadableDB, sentAt: number ): Array { - // Make sure to preserve order of columns - const editedColumns = MESSAGE_COLUMNS_FRAGMENTS.map(name => { - if (name.fragment === 'received_at' || name.fragment === 'sent_at') { - return name; - } - return sqlFragment`messages.${name}`; - }); + return db.transaction(() => { + // Make sure to preserve order of columns + const editedColumns = MESSAGE_COLUMNS_FRAGMENTS.map(name => { + if (name.fragment === 'received_at' || name.fragment === 'sent_at') { + return name; + } + return sqlFragment`messages.${name}`; + }); - const [query, params] = sql` + const [query, params] = sql` SELECT ${sqlJoin(editedColumns)} FROM edited_messages INNER JOIN messages ON @@ -4687,35 +5056,39 @@ function getMessagesBySentAt( ORDER BY messages.received_at DESC, messages.sent_at DESC; `; - const rows = db.prepare(query).all(params); + const rows = db.prepare(query).all(params); - return rows.map(row => hydrateMessage(row)); + return hydrateMessages(db, rows); + })(); } function getExpiredMessages(db: ReadableDB): Array { - const now = Date.now(); + return db.transaction(() => { + const now = Date.now(); - const rows: Array = db - .prepare( - ` + const rows: Array = db + .prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')}, expiresAt FROM messages WHERE expiresAt <= $now ORDER BY expiresAt ASC; ` - ) - .all({ now }); + ) + .all({ now }); - return rows.map(row => hydrateMessage(row)); + return hydrateMessages(db, rows); + })(); } function getMessagesUnexpectedlyMissingExpirationStartTimestamp( db: ReadableDB ): Array { - const rows: Array = db - .prepare( - ` + return db.transaction(() => { + const rows: Array = db + .prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages INDEXED BY messages_unexpectedly_missing_expiration_start_timestamp WHERE @@ -4730,10 +5103,11 @@ function getMessagesUnexpectedlyMissingExpirationStartTimestamp( )) ); ` - ) - .all(); + ) + .all(); - return rows.map(row => hydrateMessage(row)); + return hydrateMessages(db, rows); + })(); } function getSoonestMessageExpiry(db: ReadableDB): undefined | number { @@ -4760,9 +5134,10 @@ function getSoonestMessageExpiry(db: ReadableDB): undefined | number { function getNextTapToViewMessageTimestampToAgeOut( db: ReadableDB ): undefined | number { - const row = db - .prepare( - ` + return db.transaction(() => { + const row = db + .prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE -- we want this query to use the messages_view_once index rather than received_at @@ -4771,24 +5146,26 @@ function getNextTapToViewMessageTimestampToAgeOut( ORDER BY received_at ASC, sent_at ASC LIMIT 1; ` - ) - .get(); + ) + .get(); - if (!row) { - return undefined; - } - const data = hydrateMessage(row); - const result = data.received_at_ms; - return isNormalNumber(result) ? result : undefined; + if (!row) { + return undefined; + } + const data = hydrateMessage(db, row); + const result = data.received_at_ms; + return isNormalNumber(result) ? result : undefined; + })(); } function getTapToViewMessagesNeedingErase( db: ReadableDB, maxTimestamp: number ): Array { - const rows: Array = db - .prepare( - ` + return db.transaction(() => { + const rows: Array = db + .prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE @@ -4798,12 +5175,13 @@ function getTapToViewMessagesNeedingErase( IFNULL(received_at_ms, 0) <= $maxTimestamp ) ` - ) - .all({ - maxTimestamp, - }); + ) + .all({ + maxTimestamp, + }); - return rows.map(row => hydrateMessage(row)); + return hydrateMessages(db, rows); + })(); } const MAX_UNPROCESSED_ATTEMPTS = 10; @@ -5424,8 +5802,10 @@ function getBackupCdnObjectMetadata( db: ReadableDB, mediaId: string ): BackupCdnMediaObjectType | undefined { - const [query, params] = - sql`SELECT * from backup_cdn_object_metadata WHERE mediaId = ${mediaId}`; + const [query, params] = sql` + SELECT * FROM backup_cdn_object_metadata + WHERE mediaId = ${mediaId} + `; return db.prepare(query).get(params); } @@ -7131,6 +7511,7 @@ function removeAll(db: WritableDB): void { DELETE FROM syncTasks; DELETE FROM unprocessed; DELETE FROM uninstalled_sticker_packs; + DELETE FROM message_attachments; INSERT INTO messages_fts(messages_fts) VALUES('optimize'); @@ -7259,9 +7640,10 @@ function getMessagesNeedingUpgrade( limit: number, { maxVersion }: { maxVersion: number } ): Array { - const rows: Array = db - .prepare( - ` + return db.transaction(() => { + const rows: Array = db + .prepare( + ` SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE @@ -7272,14 +7654,15 @@ function getMessagesNeedingUpgrade( ) < $maxAttempts LIMIT $limit; ` - ) - .all({ - maxVersion, - maxAttempts: MAX_MESSAGE_MIGRATION_ATTEMPTS, - limit, - }); + ) + .all({ + maxVersion, + maxAttempts: MAX_MESSAGE_MIGRATION_ATTEMPTS, + limit, + }); - return rows.map(row => hydrateMessage(row)); + return hydrateMessages(db, rows); + })(); } // Exported for tests @@ -7585,7 +7968,7 @@ function pageMessages( { persistent } ); const rows: Array = query.all(batch); - return rows.map(row => hydrateMessage(row)); + return hydrateMessages(db, rows); } ); @@ -8050,13 +8433,12 @@ function getUnreadEditedMessagesAndMarkRead( db.prepare(updateStatusQuery).run(updateStatusParams); } - return rows.map(row => { - const json = hydrateMessage(row); + return hydrateMessages(db, rows).map(msg => { return { - originalReadStatus: row.readStatus ?? undefined, + originalReadStatus: msg.readStatus ?? undefined, readStatus: ReadStatus.Read, seenStatus: SeenStatus.Seen, - ...pick(json, [ + ...pick(msg, [ 'conversationId', 'expirationStartTimestamp', 'id', diff --git a/ts/sql/hydration.ts b/ts/sql/hydration.ts index 162109351d..e3d088592b 100644 --- a/ts/sql/hydration.ts +++ b/ts/sql/hydration.ts @@ -1,10 +1,11 @@ // Copyright 2025 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only +import { groupBy } from 'lodash'; import type { ReadStatus } from '../messages/MessageReadStatus'; import type { SeenStatus } from '../MessageSeenStatus'; import type { ServiceIdString } from '../types/ServiceId'; -import { dropNull } from '../util/dropNull'; +import { dropNull, shallowDropNull } from '../util/dropNull'; /* eslint-disable camelcase */ @@ -12,7 +13,23 @@ import type { MessageTypeUnhydrated, MessageType, MESSAGE_COLUMNS, + ReadableDB, + MessageAttachmentDBType, } from './Interface'; +import { + batchMultiVarQuery, + convertOptionalIntegerToBoolean, + jsonToObject, + sql, + sqlJoin, +} from './util'; +import type { AttachmentType } from '../types/Attachment'; +import { IMAGE_JPEG, stringToMIMEType } from '../types/MIME'; +import { strictAssert } from '../util/assert'; +import { sqlLogger } from './sqlLogger'; +import type { MessageAttributesType } from '../model-types'; + +export const ROOT_MESSAGE_ATTACHMENT_EDIT_HISTORY_INDEX = -1; function toBoolean(value: number | null): boolean | undefined { if (value == null) { @@ -21,7 +38,27 @@ function toBoolean(value: number | null): boolean | undefined { return value === 1; } -export function hydrateMessage(row: MessageTypeUnhydrated): MessageType { +export function hydrateMessage( + db: ReadableDB, + row: MessageTypeUnhydrated +): MessageType { + return hydrateMessages(db, [row])[0]; +} + +export function hydrateMessages( + db: ReadableDB, + unhydratedMessages: Array +): Array { + const messagesWithColumnsHydrated = unhydratedMessages.map( + hydrateMessageTableColumns + ); + + return hydrateMessagesWithAttachments(db, messagesWithColumnsHydrated); +} + +export function hydrateMessageTableColumns( + row: MessageTypeUnhydrated +): MessageType { const { json, id, @@ -29,9 +66,6 @@ export function hydrateMessage(row: MessageTypeUnhydrated): MessageType { conversationId, expirationStartTimestamp, expireTimer, - hasAttachments, - hasFileAttachments, - hasVisualMediaAttachments, isErased, isViewOnce, mentionsMe, @@ -63,9 +97,6 @@ export function hydrateMessage(row: MessageTypeUnhydrated): MessageType { conversationId: conversationId || '', expirationStartTimestamp: dropNull(expirationStartTimestamp), expireTimer: dropNull(expireTimer) as MessageType['expireTimer'], - hasAttachments: toBoolean(hasAttachments), - hasFileAttachments: toBoolean(hasFileAttachments), - hasVisualMediaAttachments: toBoolean(hasVisualMediaAttachments), isErased: toBoolean(isErased), isViewOnce: toBoolean(isViewOnce), mentionsMe: toBoolean(mentionsMe), @@ -86,3 +117,299 @@ export function hydrateMessage(row: MessageTypeUnhydrated): MessageType { unidentifiedDeliveryReceived: toBoolean(unidentifiedDeliveryReceived), }; } + +export function getAttachmentReferencesForMessages( + db: ReadableDB, + messageIds: Array +): Array { + return batchMultiVarQuery( + db, + messageIds, + ( + messageIdBatch: ReadonlyArray, + persistent: boolean + ): Array => { + const [query, params] = sql` + SELECT * FROM message_attachments + WHERE messageId IN (${sqlJoin(messageIdBatch)}); + `; + + return db + .prepare(query, { persistent }) + .all(params); + } + ); +} + +function hydrateMessagesWithAttachments( + db: ReadableDB, + messagesWithoutAttachments: Array +): Array { + const attachmentReferencesForAllMessages = getAttachmentReferencesForMessages( + db, + messagesWithoutAttachments.map(msg => msg.id) + ); + const attachmentReferencesByMessage = groupBy( + attachmentReferencesForAllMessages, + 'messageId' + ); + + return messagesWithoutAttachments.map(msg => { + const attachmentReferences = attachmentReferencesByMessage[msg.id] ?? []; + if (!attachmentReferences.length) { + return msg; + } + + const attachmentsByEditHistoryIndex = groupBy( + attachmentReferences, + 'editHistoryIndex' + ); + + const message = hydrateMessageRootOrRevisionWithAttachments( + msg, + attachmentsByEditHistoryIndex[ + ROOT_MESSAGE_ATTACHMENT_EDIT_HISTORY_INDEX + ] ?? [] + ); + + if (message.editHistory) { + message.editHistory = message.editHistory.map((editHistory, idx) => { + return hydrateMessageRootOrRevisionWithAttachments( + editHistory, + attachmentsByEditHistoryIndex[idx] ?? [] + ); + }); + } + + return message; + }); +} + +function hydrateMessageRootOrRevisionWithAttachments< + T extends Pick< + MessageAttributesType, + | 'attachments' + | 'bodyAttachment' + | 'contact' + | 'preview' + | 'quote' + | 'sticker' + >, +>(message: T, messageAttachments: Array): T { + const attachmentsByType = groupBy( + messageAttachments, + 'attachmentType' + ) as Record< + MessageAttachmentDBType['attachmentType'], + Array + >; + + const standardAttachments = attachmentsByType.attachment ?? []; + const bodyAttachments = attachmentsByType['long-message'] ?? []; + const quoteAttachments = attachmentsByType.quote ?? []; + const previewAttachments = attachmentsByType.preview ?? []; + const contactAttachments = attachmentsByType.contact ?? []; + const stickerAttachment = (attachmentsByType.sticker ?? []).find( + sticker => sticker.orderInMessage === 0 + ); + + const hydratedMessage = structuredClone(message); + + if (standardAttachments.length) { + hydratedMessage.attachments = standardAttachments + .sort((a, b) => a.orderInMessage - b.orderInMessage) + .map(convertAttachmentDBFieldsToAttachmentType); + } + + if (bodyAttachments[0]) { + hydratedMessage.bodyAttachment = convertAttachmentDBFieldsToAttachmentType( + bodyAttachments[0] + ); + } + + hydratedMessage.quote?.attachments.forEach((quoteAttachment, idx) => { + const quoteThumbnail = quoteAttachments.find( + attachment => attachment.orderInMessage === idx + ); + if (quoteThumbnail) { + // eslint-disable-next-line no-param-reassign + quoteAttachment.thumbnail = + convertAttachmentDBFieldsToAttachmentType(quoteThumbnail); + } + }); + + hydratedMessage.preview?.forEach((preview, idx) => { + const previewAttachment = previewAttachments.find( + attachment => attachment.orderInMessage === idx + ); + + if (previewAttachment) { + // eslint-disable-next-line no-param-reassign + preview.image = + convertAttachmentDBFieldsToAttachmentType(previewAttachment); + } + }); + + hydratedMessage.contact?.forEach((contact, idx) => { + const contactAttachment = contactAttachments.find( + attachment => attachment.orderInMessage === idx + ); + if (contactAttachment && contact.avatar) { + // eslint-disable-next-line no-param-reassign + contact.avatar.avatar = + convertAttachmentDBFieldsToAttachmentType(contactAttachment); + } + }); + + if (hydratedMessage.sticker && stickerAttachment) { + hydratedMessage.sticker.data = + convertAttachmentDBFieldsToAttachmentType(stickerAttachment); + } + + return hydratedMessage; +} + +function convertAttachmentDBFieldsToAttachmentType( + dbFields: MessageAttachmentDBType +): AttachmentType { + const messageAttachment = shallowDropNull(dbFields); + strictAssert(messageAttachment != null, 'must exist'); + + const { + clientUuid, + size, + contentType, + plaintextHash, + path, + localKey, + caption, + blurHash, + height, + width, + digest, + iv, + key, + downloadPath, + flags, + fileName, + version, + incrementalMac, + incrementalMacChunkSize: chunkSize, + transitCdnKey: cdnKey, + transitCdnNumber: cdnNumber, + transitCdnUploadTimestamp: uploadTimestamp, + error, + pending, + wasTooBig, + isCorrupted, + backfillError, + storyTextAttachmentJson, + copiedFromQuotedAttachment, + isReencryptableToSameDigest, + localBackupPath, + } = messageAttachment; + + const result: AttachmentType = { + clientUuid, + size, + contentType: stringToMIMEType(contentType), + plaintextHash, + path, + localKey, + caption, + blurHash, + height, + width, + digest, + iv, + key, + downloadPath, + localBackupPath, + flags, + fileName, + version, + incrementalMac, + chunkSize, + cdnKey, + cdnNumber, + uploadTimestamp, + pending: convertOptionalIntegerToBoolean(pending), + error: convertOptionalIntegerToBoolean(error), + wasTooBig: convertOptionalIntegerToBoolean(wasTooBig), + copied: convertOptionalIntegerToBoolean(copiedFromQuotedAttachment), + isCorrupted: convertOptionalIntegerToBoolean(isCorrupted), + backfillError: convertOptionalIntegerToBoolean(backfillError), + isReencryptableToSameDigest: convertOptionalIntegerToBoolean( + isReencryptableToSameDigest + ), + textAttachment: storyTextAttachmentJson + ? jsonToObject(storyTextAttachmentJson) + : undefined, + ...(messageAttachment.backupMediaName + ? { + backupLocator: { + mediaName: messageAttachment.backupMediaName, + cdnNumber: messageAttachment.backupCdnNumber, + }, + } + : {}), + ...(messageAttachment.thumbnailPath + ? { + thumbnail: { + path: messageAttachment.thumbnailPath, + size: messageAttachment.thumbnailSize ?? 0, + contentType: messageAttachment.thumbnailContentType + ? stringToMIMEType(messageAttachment.thumbnailContentType) + : IMAGE_JPEG, + localKey: messageAttachment.thumbnailLocalKey, + version: messageAttachment.thumbnailVersion, + }, + } + : {}), + ...(messageAttachment.screenshotPath + ? { + screenshot: { + path: messageAttachment.screenshotPath, + size: messageAttachment.screenshotSize ?? 0, + contentType: messageAttachment.screenshotContentType + ? stringToMIMEType(messageAttachment.screenshotContentType) + : IMAGE_JPEG, + localKey: messageAttachment.screenshotLocalKey, + version: messageAttachment.screenshotVersion, + }, + } + : {}), + ...(messageAttachment.backupThumbnailPath + ? { + thumbnailFromBackup: { + path: messageAttachment.backupThumbnailPath, + size: messageAttachment.backupThumbnailSize ?? 0, + contentType: messageAttachment.backupThumbnailContentType + ? stringToMIMEType(messageAttachment.backupThumbnailContentType) + : IMAGE_JPEG, + localKey: messageAttachment.backupThumbnailLocalKey, + version: messageAttachment.backupThumbnailVersion, + }, + } + : {}), + }; + + if (result.isReencryptableToSameDigest === false) { + if ( + !messageAttachment.reencryptionIv || + !messageAttachment.reencryptionKey || + !messageAttachment.reencryptionDigest + ) { + sqlLogger.warn( + 'Attachment missing reencryption info despite not being reencryptable' + ); + return result; + } + result.reencryptionInfo = { + iv: messageAttachment.reencryptionIv, + key: messageAttachment.reencryptionKey, + digest: messageAttachment.reencryptionDigest, + }; + } + return result; +} diff --git a/ts/sql/mainWorker.ts b/ts/sql/mainWorker.ts index 63f8e9ade1..5c3ba8d3e4 100644 --- a/ts/sql/mainWorker.ts +++ b/ts/sql/mainWorker.ts @@ -3,15 +3,11 @@ import { parentPort } from 'worker_threads'; -import type { LoggerType } from '../types/Logging'; -import type { - WrappedWorkerRequest, - WrappedWorkerResponse, - WrappedWorkerLogEntry, -} from './main'; +import type { WrappedWorkerRequest, WrappedWorkerResponse } from './main'; import type { WritableDB } from './Interface'; import { initialize, DataReader, DataWriter, removeDB } from './Server'; import { SqliteErrorKind, parseSqliteError } from './errors'; +import { sqlLogger as logger } from './sqlLogger'; if (!parentPort) { throw new Error('Must run as a worker thread'); @@ -31,39 +27,6 @@ function respond(seq: number, response?: any) { port.postMessage(wrappedResponse); } -const log = ( - level: WrappedWorkerLogEntry['level'], - args: Array -): void => { - const wrappedResponse: WrappedWorkerResponse = { - type: 'log', - level, - args, - }; - port.postMessage(wrappedResponse); -}; - -const logger: LoggerType = { - fatal(...args: Array) { - log('fatal', args); - }, - error(...args: Array) { - log('error', args); - }, - warn(...args: Array) { - log('warn', args); - }, - info(...args: Array) { - log('info', args); - }, - debug(...args: Array) { - log('debug', args); - }, - trace(...args: Array) { - log('trace', args); - }, -}; - let db: WritableDB | undefined; let isPrimary = false; let isRemoved = false; @@ -79,7 +42,6 @@ const onMessage = ( db = initialize({ ...request.options, isPrimary, - logger, }); respond(seq, undefined); diff --git a/ts/sql/migrations/1360-attachments.ts b/ts/sql/migrations/1360-attachments.ts new file mode 100644 index 0000000000..24f7ba2f95 --- /dev/null +++ b/ts/sql/migrations/1360-attachments.ts @@ -0,0 +1,111 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import type { LoggerType } from '../../types/Logging'; +import type { WritableDB } from '../Interface'; + +export const version = 1360; + +export function updateToSchemaVersion1360( + currentVersion: number, + db: WritableDB, + logger: LoggerType +): void { + if (currentVersion >= 1360) { + return; + } + + db.transaction(() => { + db.exec(` + DROP TABLE IF EXISTS message_attachments; + `); + + db.exec(` + CREATE TABLE message_attachments ( + messageId TEXT NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + -- For editHistoryIndex to be part of the primary key, it cannot be NULL in strict tables. + -- For that reason, we use a value of -1 to indicate that it is the root message (not in editHistory) + editHistoryIndex INTEGER NOT NULL, + attachmentType TEXT NOT NULL, -- 'long-message' | 'quote' | 'attachment' | 'preview' | 'contact' | 'sticker' + orderInMessage INTEGER NOT NULL, + conversationId TEXT NOT NULL, + sentAt INTEGER NOT NULL, + clientUuid TEXT, + size INTEGER NOT NULL, + contentType TEXT NOT NULL, + path TEXT, + plaintextHash TEXT, + localKey TEXT, + caption TEXT, + fileName TEXT, + blurHash TEXT, + height INTEGER, + width INTEGER, + digest TEXT, + key TEXT, + iv TEXT, + downloadPath TEXT, + version INTEGER, + incrementalMac TEXT, + incrementalMacChunkSize INTEGER, + transitCdnKey TEXT, + transitCdnNumber INTEGER, + transitCdnUploadTimestamp INTEGER, + backupMediaName TEXT, + backupCdnNumber INTEGER, + isReencryptableToSameDigest INTEGER, + reencryptionIv TEXT, + reencryptionKey TEXT, + reencryptionDigest TEXT, + thumbnailPath TEXT, + thumbnailSize INTEGER, + thumbnailContentType TEXT, + thumbnailLocalKey TEXT, + thumbnailVersion INTEGER, + screenshotPath TEXT, + screenshotSize INTEGER, + screenshotContentType TEXT, + screenshotLocalKey TEXT, + screenshotVersion INTEGER, + backupThumbnailPath TEXT, + backupThumbnailSize INTEGER, + backupThumbnailContentType TEXT, + backupThumbnailLocalKey TEXT, + backupThumbnailVersion INTEGER, + storyTextAttachmentJson TEXT, + localBackupPath TEXT, + flags INTEGER, + error INTEGER, + wasTooBig INTEGER, + isCorrupted INTEGER, + copiedFromQuotedAttachment INTEGER, + pending INTEGER, + backfillError INTEGER, + PRIMARY KEY (messageId, editHistoryIndex, attachmentType, orderInMessage) + ) STRICT; + `); + + db.exec( + 'CREATE INDEX message_attachments_messageId ON message_attachments (messageId);' + ); + db.exec( + 'CREATE INDEX message_attachments_plaintextHash ON message_attachments (plaintextHash);' + ); + db.exec( + 'CREATE INDEX message_attachments_path ON message_attachments (path);' + ); + db.exec( + 'CREATE INDEX message_attachments_all_thumbnailPath ON message_attachments (thumbnailPath);' + ); + db.exec( + 'CREATE INDEX message_attachments_all_screenshotPath ON message_attachments (screenshotPath);' + ); + db.exec( + 'CREATE INDEX message_attachments_all_backupThumbnailPath ON message_attachments (backupThumbnailPath);' + ); + + db.pragma('user_version = 1360'); + })(); + + logger.info('updateToSchemaVersion1360: success!'); +} diff --git a/ts/sql/migrations/index.ts b/ts/sql/migrations/index.ts index 0ea10b41e2..6c055ce52c 100644 --- a/ts/sql/migrations/index.ts +++ b/ts/sql/migrations/index.ts @@ -110,10 +110,11 @@ import { updateToSchemaVersion1310 } from './1310-muted-fixup'; import { updateToSchemaVersion1320 } from './1320-unprocessed-received-at-date'; import { updateToSchemaVersion1330 } from './1330-sync-tasks-type-index'; import { updateToSchemaVersion1340 } from './1340-recent-gifs'; +import { updateToSchemaVersion1350 } from './1350-notification-profiles'; import { - updateToSchemaVersion1350, + updateToSchemaVersion1360, version as MAX_VERSION, -} from './1350-notification-profiles'; +} from './1360-attachments'; import { DataWriter } from '../Server'; @@ -2102,6 +2103,7 @@ export const SCHEMA_VERSIONS = [ updateToSchemaVersion1330, updateToSchemaVersion1340, updateToSchemaVersion1350, + updateToSchemaVersion1360, ]; export class DBVersionFromFutureError extends Error { diff --git a/ts/sql/sqlLogger.ts b/ts/sql/sqlLogger.ts new file mode 100644 index 0000000000..9021fead7a --- /dev/null +++ b/ts/sql/sqlLogger.ts @@ -0,0 +1,45 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only +import { parentPort } from 'worker_threads'; +import type { LoggerType } from '../types/Logging'; +import type { WrappedWorkerLogEntry, WrappedWorkerResponse } from './main'; +import { consoleLogger } from '../util/consoleLogger'; +import { strictAssert } from '../util/assert'; + +const log = ( + level: WrappedWorkerLogEntry['level'], + args: Array +): void => { + if (parentPort) { + const wrappedResponse: WrappedWorkerResponse = { + type: 'log', + level, + args, + }; + parentPort.postMessage(wrappedResponse); + } else { + strictAssert(process.env.NODE_ENV === 'test', 'must be test environment'); + consoleLogger[level](...args); + } +}; + +export const sqlLogger: LoggerType = { + fatal(...args: Array) { + log('fatal', args); + }, + error(...args: Array) { + log('error', args); + }, + warn(...args: Array) { + log('warn', args); + }, + info(...args: Array) { + log('info', args); + }, + debug(...args: Array) { + log('debug', args); + }, + trace(...args: Array) { + log('trace', args); + }, +}; diff --git a/ts/sql/util.ts b/ts/sql/util.ts index 39f1483931..c9e41dbde1 100644 --- a/ts/sql/util.ts +++ b/ts/sql/util.ts @@ -418,3 +418,27 @@ export class TableIterator { } } } + +export function convertOptionalIntegerToBoolean( + optionalInteger?: number +): boolean | undefined { + if (optionalInteger === 1) { + return true; + } + if (optionalInteger === 0) { + return false; + } + return undefined; +} + +export function convertOptionalBooleanToNullableInteger( + optionalBoolean?: boolean +): 1 | 0 | null { + if (optionalBoolean === true) { + return 1; + } + if (optionalBoolean === false) { + return 0; + } + return null; +} diff --git a/ts/state/ducks/lightbox.ts b/ts/state/ducks/lightbox.ts index 5836a6bead..dfbe32f9ec 100644 --- a/ts/state/ducks/lightbox.ts +++ b/ts/state/ducks/lightbox.ts @@ -328,10 +328,9 @@ function showLightbox(opts: { sentAt, }, attachment: item, - thumbnailObjectUrl: - item.thumbnail?.objectUrl || item.thumbnail?.path - ? getLocalAttachmentUrl(item.thumbnail) - : undefined, + thumbnailObjectUrl: item.thumbnail?.path + ? getLocalAttachmentUrl(item.thumbnail) + : undefined, size: item.size, totalDownloaded: item.totalDownloaded, })) diff --git a/ts/state/selectors/message.ts b/ts/state/selectors/message.ts index 689943a86c..e75a74c85c 100644 --- a/ts/state/selectors/message.ts +++ b/ts/state/selectors/message.ts @@ -1887,18 +1887,16 @@ export function getPropsForAttachment( function processQuoteAttachment(attachment: QuotedAttachmentType) { const { thumbnail } = attachment; - const path = thumbnail && thumbnail.path && getLocalAttachmentUrl(thumbnail); - const objectUrl = thumbnail && thumbnail.objectUrl; - - const thumbnailWithObjectUrl = - (!path && !objectUrl) || !thumbnail - ? undefined - : { ...thumbnail, objectUrl: path || objectUrl }; return { ...attachment, isVoiceMessage: isVoiceMessage(attachment), - thumbnail: thumbnailWithObjectUrl, + thumbnail: thumbnail?.path + ? { + ...thumbnail, + url: getLocalAttachmentUrl(thumbnail), + } + : undefined, }; } diff --git a/ts/test-electron/backup/attachments_test.ts b/ts/test-electron/backup/attachments_test.ts index 440c1ecd33..9d784e47d2 100644 --- a/ts/test-electron/backup/attachments_test.ts +++ b/ts/test-electron/backup/attachments_test.ts @@ -55,6 +55,7 @@ describe('backup/attachments', () => { beforeEach(async () => { await DataWriter.removeAll(); window.storage.reset(); + window.ConversationController.reset(); await setupBasics(); @@ -166,8 +167,6 @@ describe('backup/attachments', () => { // path & iv will not be roundtripped [ composeMessage(1, { - hasAttachments: true, - hasVisualMediaAttachments: true, attachments: [ omit(longMessageAttachment, NON_ROUNDTRIPPED_FIELDS), omit(normalAttachment, NON_ROUNDTRIPPED_FIELDS), @@ -284,8 +283,6 @@ describe('backup/attachments', () => { // path & iv will not be roundtripped [ composeMessage(1, { - hasAttachments: true, - hasVisualMediaAttachments: true, attachments: [ omit(attachment1, NON_ROUNDTRIPPED_FIELDS), omit(attachment2, NON_ROUNDTRIPPED_FIELDS), @@ -307,9 +304,6 @@ describe('backup/attachments', () => { ], [ composeMessage(1, { - hasAttachments: true, - hasVisualMediaAttachments: true, - // path, iv, and uploadTimestamp will not be roundtripped, // but there will be a backupLocator attachments: [ @@ -341,7 +335,6 @@ describe('backup/attachments', () => { ], [ composeMessage(1, { - hasAttachments: true, attachments: [ { ...omit(attachment, NON_ROUNDTRIPPED_BACKUP_LOCATOR_FIELDS), @@ -373,7 +366,6 @@ describe('backup/attachments', () => { [ composeMessage(1, { body: 'hello', - hasAttachments: true, attachments: [ { ...omit(attachment, NON_ROUNDTRIPPED_BACKUP_LOCATOR_FIELDS), @@ -637,8 +629,6 @@ describe('backup/attachments', () => { [ { ...existingMessage, - hasAttachments: true, - hasVisualMediaAttachments: true, attachments: [ { ...omit( diff --git a/ts/test-electron/backup/helpers.ts b/ts/test-electron/backup/helpers.ts index 56afdf9c3a..e702f36657 100644 --- a/ts/test-electron/backup/helpers.ts +++ b/ts/test-electron/backup/helpers.ts @@ -110,9 +110,6 @@ function sortAndNormalize( return JSON.parse( JSON.stringify({ // Defaults - hasAttachments: false, - hasFileAttachments: false, - hasVisualMediaAttachments: false, isErased: false, isViewOnce: false, mentionsMe: false, diff --git a/ts/test-electron/cleanupOrphanedAttachments_test.ts b/ts/test-electron/cleanupOrphanedAttachments_test.ts new file mode 100644 index 0000000000..a961290cd5 --- /dev/null +++ b/ts/test-electron/cleanupOrphanedAttachments_test.ts @@ -0,0 +1,259 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { assert } from 'chai'; +import { emptyDir, ensureFile } from 'fs-extra'; +import { v4 as generateUuid } from 'uuid'; +import { readdirSync } from 'fs'; +import { dirname } from 'path'; + +import { DataWriter } from '../sql/Client'; +import { missingCaseError } from '../util/missingCaseError'; +import { + getDownloadsPath, + getDraftPath, + getPath, +} from '../windows/attachments'; + +import { generateAci } from '../types/ServiceId'; +import { IMAGE_JPEG, LONG_MESSAGE } from '../types/MIME'; + +function getAbsolutePath( + path: string, + type: 'attachment' | 'download' | 'draft' +) { + switch (type) { + case 'attachment': + return window.Signal.Migrations.getAbsoluteAttachmentPath(path); + case 'download': + return window.Signal.Migrations.getAbsoluteDownloadsPath(path); + case 'draft': + return window.Signal.Migrations.getAbsoluteDraftPath(path); + default: + throw missingCaseError(type); + } +} + +async function writeFile( + path: string, + type: 'attachment' | 'download' | 'draft' +) { + await ensureFile(getAbsolutePath(path, type)); +} + +async function writeFiles( + num: number, + type: 'attachment' | 'download' | 'draft' +) { + for (let i = 0; i < num; i += 1) { + // eslint-disable-next-line no-await-in-loop + await writeFile(`file${i}`, type); + } +} + +function listFiles(type: 'attachment' | 'download' | 'draft'): Array { + return readdirSync(dirname(getAbsolutePath('fakename', type))); +} + +describe('cleanupOrphanedAttachments', () => { + // TODO (DESKTOP-8613): stickers & badges + beforeEach(async () => { + await DataWriter.removeAll(); + await emptyDir(getPath(window.SignalContext.config.userDataPath)); + await emptyDir(getDownloadsPath(window.SignalContext.config.userDataPath)); + await emptyDir(getDraftPath(window.SignalContext.config.userDataPath)); + }); + + afterEach(async () => { + await emptyDir(getPath(window.SignalContext.config.userDataPath)); + await emptyDir(getDownloadsPath(window.SignalContext.config.userDataPath)); + await emptyDir(getDraftPath(window.SignalContext.config.userDataPath)); + }); + + it('deletes paths if not referenced', async () => { + await writeFiles(2, 'attachment'); + await writeFiles(2, 'draft'); + await writeFiles(2, 'download'); + + assert.sameDeepMembers(listFiles('attachment'), ['file0', 'file1']); + assert.sameDeepMembers(listFiles('draft'), ['file0', 'file1']); + assert.sameDeepMembers(listFiles('download'), ['file0', 'file1']); + + await DataWriter.cleanupOrphanedAttachments({ _block: true }); + + assert.sameDeepMembers(listFiles('attachment'), []); + assert.sameDeepMembers(listFiles('draft'), []); + assert.sameDeepMembers(listFiles('download'), []); + }); + + it('does not delete conversation avatar and profileAvatar paths', async () => { + await writeFiles(6, 'attachment'); + + await DataWriter.saveConversation({ + id: generateUuid(), + type: 'private', + version: 2, + expireTimerVersion: 2, + avatar: { + path: 'file0', + }, + profileAvatar: { + path: 'file1', + }, + }); + + await DataWriter.cleanupOrphanedAttachments({ _block: true }); + + assert.sameDeepMembers(listFiles('attachment'), ['file0', 'file1']); + }); + + it('does not delete message attachments (including thumbnails, previews, avatars, etc.)', async () => { + await writeFiles(20, 'attachment'); + await writeFiles(6, 'download'); + + // Save with legacy (un-normalized) sattachment format (attachments in JSON) + await DataWriter.saveMessage( + { + id: generateUuid(), + type: 'outgoing', + sent_at: Date.now(), + timestamp: Date.now(), + received_at: Date.now(), + conversationId: generateUuid(), + + attachments: [ + { + contentType: IMAGE_JPEG, + size: 128, + path: 'file0', + downloadPath: 'file0', + thumbnail: { + contentType: IMAGE_JPEG, + size: 128, + path: 'file1', + }, + screenshot: { + contentType: IMAGE_JPEG, + size: 128, + path: 'file2', + }, + thumbnailFromBackup: { + contentType: IMAGE_JPEG, + size: 128, + path: 'file3', + }, + }, + ], + }, + { + ourAci: generateAci(), + forceSave: true, + _testOnlyAvoidNormalizingAttachments: true, + postSaveUpdates: () => Promise.resolve(), + } + ); + + // Save one with attachments normalized + await DataWriter.saveMessage( + { + id: generateUuid(), + type: 'outgoing', + sent_at: Date.now(), + timestamp: Date.now(), + received_at: Date.now(), + conversationId: generateUuid(), + bodyAttachment: { + contentType: IMAGE_JPEG, + size: 128, + path: 'file4', + }, + contact: [ + { + avatar: { + isProfile: false, + avatar: { + contentType: IMAGE_JPEG, + size: 128, + path: 'file5', + }, + }, + }, + ], + preview: [ + { + url: 'url', + image: { + contentType: IMAGE_JPEG, + size: 128, + path: 'file6', + }, + }, + ], + editHistory: [ + { + timestamp: Date.now(), + received_at: Date.now(), + bodyAttachment: { + contentType: LONG_MESSAGE, + size: 128, + path: 'file7', + }, + }, + ], + quote: { + id: Date.now(), + isViewOnce: false, + referencedMessageNotFound: false, + attachments: [ + { + contentType: IMAGE_JPEG, + + thumbnail: { + contentType: IMAGE_JPEG, + size: 128, + path: 'file8', + }, + }, + ], + }, + sticker: { + packId: 'packId', + stickerId: 42, + packKey: 'packKey', + data: { + contentType: IMAGE_JPEG, + size: 128, + path: 'file9', + thumbnail: { + contentType: IMAGE_JPEG, + size: 128, + path: 'file10', + }, + }, + }, + }, + { + ourAci: generateAci(), + forceSave: true, + postSaveUpdates: () => Promise.resolve(), + } + ); + + await DataWriter.cleanupOrphanedAttachments({ _block: true }); + + assert.sameDeepMembers(listFiles('attachment'), [ + 'file0', + 'file1', + 'file2', + 'file3', + 'file4', + 'file5', + 'file6', + 'file7', + 'file8', + 'file9', + 'file10', + ]); + assert.sameDeepMembers(listFiles('download'), ['file0']); + }); +}); diff --git a/ts/test-electron/models/conversations_test.ts b/ts/test-electron/models/conversations_test.ts index 8c61eef014..80b6850c26 100644 --- a/ts/test-electron/models/conversations_test.ts +++ b/ts/test-electron/models/conversations_test.ts @@ -64,9 +64,6 @@ describe('Conversations', () => { body: 'bananas', conversationId: conversation.id, expirationStartTimestamp: now, - hasAttachments: false, - hasFileAttachments: false, - hasVisualMediaAttachments: false, id: generateUuid(), received_at: now, sent_at: now, diff --git a/ts/test-electron/normalizedAttachments_test.ts b/ts/test-electron/normalizedAttachments_test.ts new file mode 100644 index 0000000000..d060d8f56d --- /dev/null +++ b/ts/test-electron/normalizedAttachments_test.ts @@ -0,0 +1,608 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { assert } from 'chai'; +import { v4 as generateGuid } from 'uuid'; + +import * as Bytes from '../Bytes'; +import type { + EphemeralAttachmentFields, + ScreenshotType, + AttachmentType, + ThumbnailType, + BackupThumbnailType, +} from '../types/Attachment'; +import { IMAGE_JPEG, IMAGE_PNG, LONG_MESSAGE } from '../types/MIME'; +import type { MessageAttributesType } from '../model-types'; +import { generateAci } from '../types/ServiceId'; +import { ReadStatus } from '../messages/MessageReadStatus'; +import { SeenStatus } from '../MessageSeenStatus'; +import { DataWriter, DataReader } from '../sql/Client'; +import { strictAssert } from '../util/assert'; +import { HOUR, MINUTE } from '../util/durations'; + +const CONTACT_A = generateAci(); +const contactAConversationId = generateGuid(); +function getBase64(str: string): string { + return Bytes.toBase64(Bytes.fromString(str)); +} + +function composeThumbnail( + index: number, + overrides?: Partial +): ThumbnailType { + return { + size: 1024, + contentType: IMAGE_PNG, + path: `path/to/thumbnail${index}`, + localKey: `thumbnailLocalKey${index}`, + version: 2, + ...overrides, + }; +} +function composeBackupThumbnail( + index: number, + overrides?: Partial +): BackupThumbnailType { + return { + size: 1024, + contentType: IMAGE_JPEG, + path: `path/to/backupThumbnail${index}`, + localKey: 'backupThumbnailLocalKey', + version: 2, + ...overrides, + }; +} + +function composeScreenshot( + index: number, + overrides?: Partial +): ScreenshotType { + return { + size: 1024, + contentType: IMAGE_PNG, + path: `path/to/screenshot${index}`, + localKey: `screenshotLocalKey${index}`, + version: 2, + ...overrides, + }; +} + +let index = 0; +function composeAttachment( + key?: string, + overrides?: Partial + // NB: Required to ensure we are roundtripping every property in + // AttachmentType! If you are here you probably just added a field to AttachmentType; + // Make sure you add a column to the `message_attachments` table and update + // MESSAGE_ATTACHMENT_COLUMNS. +): Required> { + const label = `${key ?? 'attachment'}${index}`; + const attachment = { + cdnKey: `cdnKey${label}`, + cdnNumber: 3, + key: getBase64(`key${label}`), + digest: getBase64(`digest${label}`), + iv: getBase64(`iv${label}`), + size: 100, + downloadPath: 'downloadPath', + contentType: IMAGE_JPEG, + path: `path/to/file${label}`, + pending: false, + localKey: 'localKey', + plaintextHash: `plaintextHash${label}`, + uploadTimestamp: index, + clientUuid: generateGuid(), + width: 100, + height: 120, + blurHash: 'blurHash', + caption: 'caption', + fileName: 'filename', + flags: 8, + incrementalMac: 'incrementalMac', + chunkSize: 128, + isReencryptableToSameDigest: true, + version: 2, + backupLocator: { + mediaName: `medianame${label}`, + cdnNumber: index, + }, + localBackupPath: `localBackupPath/${label}`, + // This would only exist on a story message with contentType TEXT_ATTACHMENT, + // but inluding it here to ensure we are roundtripping all fields + textAttachment: { + text: 'text', + textStyle: 3, + }, + // defaulting all of these booleans to true to ensure that we are actually + // roundtripping them to/from the DB + wasTooBig: true, + error: true, + isCorrupted: true, + backfillError: true, + copied: true, + thumbnail: composeThumbnail(index), + screenshot: composeScreenshot(index), + thumbnailFromBackup: composeBackupThumbnail(index), + + ...overrides, + } as const; + + index += 1; + return attachment; +} + +function composeMessage( + timestamp: number, + overrides?: Partial +): MessageAttributesType { + return { + schemaVersion: 12, + conversationId: contactAConversationId, + id: generateGuid(), + type: 'incoming', + body: undefined, + received_at: timestamp, + received_at_ms: timestamp, + sourceServiceId: CONTACT_A, + sourceDevice: 1, + sent_at: timestamp, + timestamp, + readStatus: ReadStatus.Read, + seenStatus: SeenStatus.Seen, + isErased: false, + mentionsMe: false, + isViewOnce: false, + unidentifiedDeliveryReceived: false, + serverGuid: undefined, + serverTimestamp: undefined, + source: undefined, + storyId: undefined, + expirationStartTimestamp: undefined, + expireTimer: undefined, + ...overrides, + }; +} + +describe('normalizes attachment references', () => { + beforeEach(async () => { + await DataWriter.removeAll(); + }); + + it('saves message with undownloaded attachments', async () => { + const attachment1: AttachmentType = { + ...composeAttachment(), + path: undefined, + localKey: undefined, + plaintextHash: undefined, + version: undefined, + }; + const attachment2: AttachmentType = { + ...composeAttachment(), + path: undefined, + localKey: undefined, + plaintextHash: undefined, + version: undefined, + }; + + delete attachment1.thumbnail; + delete attachment1.screenshot; + delete attachment1.thumbnailFromBackup; + + delete attachment2.thumbnail; + delete attachment2.screenshot; + delete attachment2.thumbnailFromBackup; + + const attachments = [attachment1, attachment2]; + const message = composeMessage(Date.now(), { + attachments, + }); + + await DataWriter.saveMessage(message, { + forceSave: true, + ourAci: generateAci(), + postSaveUpdates: () => Promise.resolve(), + }); + + const references = await DataReader.getAttachmentReferencesForMessages([ + message.id, + ]); + + assert.equal(references.length, attachments.length); + + const messageFromDB = await DataReader.getMessageById(message.id); + assert(messageFromDB, 'message was saved'); + assert.deepEqual(messageFromDB, message); + }); + + it('saves message with downloaded attachments, and hydrates on get', async () => { + const attachments = [ + composeAttachment('first'), + composeAttachment('second'), + ]; + const message = composeMessage(Date.now(), { + attachments, + }); + + await DataWriter.saveMessage(message, { + forceSave: true, + ourAci: generateAci(), + postSaveUpdates: () => Promise.resolve(), + }); + + const messageFromDB = await DataReader.getMessageById(message.id); + assert(messageFromDB, 'message was saved'); + assert.deepEqual(messageFromDB, message); + }); + + it('saves and re-hydrates messages with normal, body, preview, quote, contact, and sticker attachments', async () => { + const attachment1 = composeAttachment('first'); + const attachment2 = composeAttachment('second'); + const previewAttachment1 = composeAttachment('preview1'); + const previewAttachment2 = composeAttachment('preview2'); + const quoteAttachment1 = composeAttachment('quote1'); + const quoteAttachment2 = composeAttachment('quote2'); + const contactAttachment1 = composeAttachment('contact1'); + const contactAttachment2 = composeAttachment('contact2'); + const stickerAttachment = composeAttachment('sticker'); + const bodyAttachment = composeAttachment('body', { + contentType: LONG_MESSAGE, + }); + + const message = composeMessage(Date.now(), { + attachments: [attachment1, attachment2], + bodyAttachment, + preview: [ + { + title: 'preview', + description: 'description', + domain: 'domain', + url: 'https://signal.org', + isStickerPack: false, + isCallLink: false, + image: previewAttachment1, + date: Date.now(), + }, + { + title: 'preview2', + description: 'description2', + domain: 'domain2', + url: 'https://signal2.org', + isStickerPack: true, + isCallLink: false, + image: previewAttachment2, + date: Date.now(), + }, + ], + quote: { + id: Date.now(), + referencedMessageNotFound: true, + isViewOnce: false, + messageId: 'quotedMessageId', + attachments: [ + { + contentType: IMAGE_JPEG, + thumbnail: quoteAttachment1, + }, + { + contentType: IMAGE_PNG, + thumbnail: quoteAttachment2, + }, + ], + }, + contact: [ + { + name: { + givenName: 'Alice', + familyName: 'User', + }, + avatar: { + isProfile: true, + avatar: contactAttachment1, + }, + }, + { + name: { + givenName: 'Bob', + familyName: 'User', + }, + avatar: { + isProfile: false, + avatar: contactAttachment2, + }, + }, + ], + sticker: { + packId: 'stickerPackId', + stickerId: 123, + packKey: 'abcdefg', + data: stickerAttachment, + }, + }); + + await DataWriter.saveMessage(message, { + forceSave: true, + ourAci: generateAci(), + postSaveUpdates: () => Promise.resolve(), + }); + + const messageFromDB = await DataReader.getMessageById(message.id); + assert(messageFromDB, 'message was saved'); + assert.deepEqual(messageFromDB, message); + }); + + it('handles quote attachments with copied thumbnail', async () => { + const referencedAttachment = composeAttachment('quotedattachment', { + thumbnail: composeThumbnail(0), + }); + strictAssert(referencedAttachment.plaintextHash, 'exists'); + const referencedMessage = composeMessage(1, { + attachments: [referencedAttachment], + }); + const quoteMessage = composeMessage(2, { + quote: { + id: Date.now(), + referencedMessageNotFound: false, + isViewOnce: false, + messageId: 'quotedMessageId', + attachments: [ + { + fileName: 'filename', + contentType: IMAGE_PNG, + thumbnail: { ...composeAttachment(), copied: true }, + }, + ], + }, + }); + + await DataWriter.saveMessage(referencedMessage, { + forceSave: true, + ourAci: generateAci(), + postSaveUpdates: () => Promise.resolve(), + }); + await DataWriter.saveMessage(quoteMessage, { + forceSave: true, + ourAci: generateAci(), + postSaveUpdates: () => Promise.resolve(), + }); + + const messageFromDB = await DataReader.getMessageById(quoteMessage.id); + assert(messageFromDB, 'message was saved'); + assert.deepEqual(messageFromDB, quoteMessage); + }); + + it('deletes and re-orders attachments as necessary', async () => { + await DataWriter.removeAll(); + const attachment1 = composeAttachment(); + const attachment2 = composeAttachment(); + const attachment3 = composeAttachment(); + + const attachments = [attachment1, attachment2, attachment3]; + const message = composeMessage(Date.now(), { + attachments, + }); + + await DataWriter.saveMessage(message, { + forceSave: true, + ourAci: generateAci(), + postSaveUpdates: () => Promise.resolve(), + }); + + const messageFromDB = await DataReader.getMessageById(message.id); + assert(messageFromDB, 'message was saved'); + assert.deepEqual(messageFromDB, message); + + /** Re-order the attachments */ + const messageWithReorderedAttachments = { + ...message, + attachments: [attachment3, attachment2, attachment1], + }; + await DataWriter.saveMessage(messageWithReorderedAttachments, { + ourAci: generateAci(), + postSaveUpdates: () => Promise.resolve(), + }); + const messageWithReorderedAttachmentsFromDB = + await DataReader.getMessageById(message.id); + + assert(messageWithReorderedAttachmentsFromDB, 'message was saved'); + assert.deepEqual( + messageWithReorderedAttachmentsFromDB, + messageWithReorderedAttachments + ); + + /** Drop the last attachment */ + const messageWithDeletedAttachment = { + ...message, + attachments: [attachment1, attachment2], + }; + await DataWriter.saveMessage(messageWithDeletedAttachment, { + ourAci: generateAci(), + postSaveUpdates: () => Promise.resolve(), + }); + const messageWithDeletedAttachmentFromDB = await DataReader.getMessageById( + message.id + ); + + assert(messageWithDeletedAttachmentFromDB, 'message was saved'); + assert.deepEqual( + messageWithDeletedAttachmentFromDB, + messageWithDeletedAttachment + ); + }); + + it('deletes attachment references when message is deleted', async () => { + const attachment1 = composeAttachment(); + const attachment2 = composeAttachment(); + + const attachments = [attachment1, attachment2]; + const message = composeMessage(Date.now(), { + attachments, + }); + + const message2 = composeMessage(Date.now(), { + attachments: [composeAttachment()], + }); + + await DataWriter.saveMessages([message, message2], { + forceSave: true, + ourAci: generateAci(), + postSaveUpdates: () => Promise.resolve(), + }); + + assert.equal( + (await DataReader.getAttachmentReferencesForMessages([message.id])) + .length, + 2 + ); + assert.equal( + (await DataReader.getAttachmentReferencesForMessages([message2.id])) + .length, + 1 + ); + + // Deleting message should delete all references + await DataWriter._removeMessage(message.id); + + assert.deepEqual( + await DataReader.getAttachmentReferencesForMessages([message.id]), + [] + ); + assert.equal( + (await DataReader.getAttachmentReferencesForMessages([message2.id])) + .length, + 1 + ); + }); + it('roundtrips edithistory attachments with normal, body, preview, and quote attachments', async () => { + const mainMessageFields = { + attachments: [composeAttachment('main1'), composeAttachment('main2')], + bodyAttachment: composeAttachment('body1', { + contentType: LONG_MESSAGE, + }), + preview: [ + { + title: 'preview', + description: 'description', + domain: 'domain', + url: 'https://signal.org', + isStickerPack: false, + isCallLink: false, + image: composeAttachment('preview1'), + date: Date.now(), + }, + ], + quote: { + id: Date.now(), + referencedMessageNotFound: true, + isViewOnce: false, + messageId: 'quotedMessageId', + attachments: [ + { + contentType: IMAGE_JPEG, + thumbnail: composeAttachment('quote3'), + }, + ], + }, + }; + + const now = Date.now(); + + const message = composeMessage(now, { + ...mainMessageFields, + editMessageReceivedAt: now + HOUR + 42, + editMessageTimestamp: now + HOUR, + editHistory: [ + { + timestamp: now + HOUR, + received_at: now + HOUR + 42, + attachments: [ + composeAttachment('main.edit1.1'), + composeAttachment('main.edit1.2'), + ], + bodyAttachment: composeAttachment('body.edit1', { + contentType: LONG_MESSAGE, + }), + preview: [ + { + title: 'preview', + description: 'description', + domain: 'domain', + url: 'https://signal.org', + isStickerPack: false, + isCallLink: true, + image: composeAttachment('preview.edit1'), + date: Date.now(), + }, + ], + quote: { + id: Date.now(), + referencedMessageNotFound: true, + isViewOnce: false, + messageId: 'quotedMessageId', + attachments: [ + { + contentType: IMAGE_JPEG, + thumbnail: composeAttachment('quote.edit1'), + }, + ], + }, + }, + { + timestamp: now + MINUTE, + received_at: now + MINUTE + 42, + attachments: [ + composeAttachment('main.edit2.1'), + composeAttachment('main.edit2.2'), + ], + bodyAttachment: composeAttachment('body.edit2', { + contentType: LONG_MESSAGE, + }), + preview: [ + { + title: 'preview', + description: 'description', + domain: 'domain', + url: 'https://signal.org', + isStickerPack: false, + isCallLink: true, + image: composeAttachment('preview.edit2'), + date: Date.now(), + }, + ], + quote: { + id: Date.now(), + referencedMessageNotFound: true, + isViewOnce: false, + messageId: 'quotedMessageId', + attachments: [ + { + contentType: IMAGE_JPEG, + thumbnail: composeAttachment('quote.edit2'), + }, + ], + }, + }, + { + timestamp: now, + received_at: now, + ...mainMessageFields, + }, + ], + }); + + await DataWriter.saveMessage(message, { + forceSave: true, + ourAci: generateAci(), + postSaveUpdates: () => Promise.resolve(), + }); + + const messageAttachments = + await DataReader.getAttachmentReferencesForMessages([message.id]); + // 5 attachments, plus 3 versions in editHistory = 20 attachments total + assert.deepEqual(messageAttachments.length, 20); + + const messageFromDB = await DataReader.getMessageById(message.id); + assert(messageFromDB, 'message was saved'); + assert.deepEqual(messageFromDB, message); + }); +}); diff --git a/ts/test-electron/services/AttachmentDownloadManager_test.ts b/ts/test-electron/services/AttachmentDownloadManager_test.ts index 04d3421d44..21d6571d53 100644 --- a/ts/test-electron/services/AttachmentDownloadManager_test.ts +++ b/ts/test-electron/services/AttachmentDownloadManager_test.ts @@ -22,6 +22,7 @@ import { DataReader, DataWriter } from '../../sql/Client'; import { MINUTE } from '../../util/durations'; import { type AttachmentType, AttachmentVariant } from '../../types/Attachment'; import { strictAssert } from '../../util/assert'; +import type { downloadAttachment as downloadAttachmentUtil } from '../../util/downloadAttachment'; import { AttachmentDownloadSource } from '../../sql/Interface'; import { getAttachmentCiphertextLength } from '../../AttachmentCrypto'; import { MEBIBYTE } from '../../types/AttachmentSize'; @@ -507,14 +508,23 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => { let processNewAttachment: sinon.SinonStub; const abortController = new AbortController(); + const downloadedAttachment: Awaited< + ReturnType + > = { + path: '/path/to/file', + iv: 'iv', + plaintextHash: 'plaintextHash', + isReencryptableToSameDigest: true, + localKey: 'localKey', + version: 2, + size: 128, + }; + beforeEach(async () => { sandbox = sinon.createSandbox(); - downloadAttachment = sandbox.stub().returns({ - path: '/path/to/file', - iv: Buffer.alloc(16), - plaintextHash: 'plaintextHash', - isReencryptableToSameDigest: true, - }); + downloadAttachment = sandbox + .stub() + .returns(Promise.resolve(downloadedAttachment)); processNewAttachment = sandbox.stub().callsFake(attachment => attachment); }); @@ -611,6 +621,8 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => { }, thumbnailFromBackup: { path: '/path/to/thumbnail', + size: 128, + contentType: MIME.IMAGE_JPEG, }, }, }); @@ -724,11 +736,7 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => { if (options.variant === AttachmentVariant.Default) { throw new Error('error while downloading'); } - return { - path: '/path/to/thumbnail', - iv: Buffer.alloc(16), - plaintextHash: 'plaintextHash', - }; + return downloadedAttachment; }); const job = composeJob({ diff --git a/ts/test-node/sql/migration_1360_test.ts b/ts/test-node/sql/migration_1360_test.ts new file mode 100644 index 0000000000..47b075d33e --- /dev/null +++ b/ts/test-node/sql/migration_1360_test.ts @@ -0,0 +1,89 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { assert } from 'chai'; + +import { sql, sqlJoin } from '../../sql/util'; +import { createDB, explain, updateToVersion } from './helpers'; +import type { WritableDB } from '../../sql/Interface'; +import { DataWriter } from '../../sql/Server'; + +describe('SQL/updateToSchemaVersion1360', () => { + let db: WritableDB; + + beforeEach(async () => { + db = createDB(); + updateToVersion(db, 1360); + await DataWriter.removeAll(db); + }); + + afterEach(() => { + db.close(); + }); + + describe('message attachments', () => { + it('uses covering index to delete based on messageId', async () => { + const details = explain( + db, + sql`DELETE from message_attachments WHERE messageId = ${'messageId'}` + ); + assert.strictEqual( + details, + 'SEARCH message_attachments USING COVERING INDEX message_attachments_messageId (messageId=?)' + ); + }); + + it('uses index to select based on messageId', async () => { + const details = explain( + db, + sql`SELECT * from message_attachments WHERE messageId IN (${sqlJoin(['id1', 'id2'])});` + ); + assert.strictEqual( + details, + 'SEARCH message_attachments USING INDEX message_attachments_messageId (messageId=?)' + ); + }); + + it('uses index find path with existing plaintextHash', async () => { + const details = explain( + db, + sql` + SELECT path, localKey + FROM message_attachments + WHERE plaintextHash = ${'plaintextHash'} + LIMIT 1; + ` + ); + assert.strictEqual( + details, + 'SEARCH message_attachments USING INDEX message_attachments_plaintextHash (plaintextHash=?)' + ); + }); + + it('uses all path indices to find if path is being referenced', async () => { + const path = 'path'; + const details = explain( + db, + sql` + SELECT 1 FROM message_attachments + WHERE + path = ${path} OR + thumbnailPath = ${path} OR + screenshotPath = ${path} OR + backupThumbnailPath = ${path}; + ` + ); + assert.deepStrictEqual(details.split('\n'), [ + 'MULTI-INDEX OR', + 'INDEX 1', + 'SEARCH message_attachments USING INDEX message_attachments_path (path=?)', + 'INDEX 2', + 'SEARCH message_attachments USING INDEX message_attachments_all_thumbnailPath (thumbnailPath=?)', + 'INDEX 3', + 'SEARCH message_attachments USING INDEX message_attachments_all_screenshotPath (screenshotPath=?)', + 'INDEX 4', + 'SEARCH message_attachments USING INDEX message_attachments_all_backupThumbnailPath (backupThumbnailPath=?)', + ]); + }); + }); +}); diff --git a/ts/test-node/types/Message2_test.ts b/ts/test-node/types/Message2_test.ts index a4e11b4285..d23457df58 100644 --- a/ts/test-node/types/Message2_test.ts +++ b/ts/test-node/types/Message2_test.ts @@ -197,9 +197,6 @@ describe('Message', () => { fileName: 'test\uFFFDfig.exe', }, ], - hasAttachments: 1, - hasVisualMediaAttachments: undefined, - hasFileAttachments: undefined, schemaVersion: Message.CURRENT_SCHEMA_VERSION, }); @@ -848,6 +845,7 @@ describe('Message', () => { const result = await Message.upgradeSchema(message, { ...getDefaultContext(), doesAttachmentExist: async () => false, + maxVersion: 14, }); assert.deepEqual({ ...message, schemaVersion: 14 }, result); diff --git a/ts/test-node/types/message/initializeAttachmentMetadata_test.ts b/ts/test-node/types/message/initializeAttachmentMetadata_test.ts deleted file mode 100644 index 01020eb50a..0000000000 --- a/ts/test-node/types/message/initializeAttachmentMetadata_test.ts +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2018 Signal Messenger, LLC -// SPDX-License-Identifier: AGPL-3.0-only - -import { assert } from 'chai'; - -import * as Message from '../../../types/message/initializeAttachmentMetadata'; -import { SignalService } from '../../../protobuf'; -import * as MIME from '../../../types/MIME'; -import * as Bytes from '../../../Bytes'; -import type { MessageAttributesType } from '../../../model-types.d'; - -function getDefaultMessage( - props?: Partial -): MessageAttributesType { - return { - id: 'some-id', - type: 'incoming', - sent_at: 45, - received_at: 45, - timestamp: 45, - conversationId: 'some-conversation-id', - ...props, - }; -} - -describe('Message', () => { - describe('initializeAttachmentMetadata', () => { - it('should classify visual media attachments', async () => { - const input = getDefaultMessage({ - type: 'incoming', - conversationId: 'foo', - id: '11111111-1111-1111-1111-111111111111', - timestamp: 1523317140899, - received_at: 1523317140899, - sent_at: 1523317140800, - attachments: [ - { - contentType: MIME.IMAGE_JPEG, - data: Bytes.fromString('foo'), - fileName: 'foo.jpg', - size: 1111, - }, - ], - }); - const expected = getDefaultMessage({ - type: 'incoming', - conversationId: 'foo', - id: '11111111-1111-1111-1111-111111111111', - timestamp: 1523317140899, - received_at: 1523317140899, - sent_at: 1523317140800, - attachments: [ - { - contentType: MIME.IMAGE_JPEG, - data: Bytes.fromString('foo'), - fileName: 'foo.jpg', - size: 1111, - }, - ], - hasAttachments: 1, - hasVisualMediaAttachments: 1, - hasFileAttachments: undefined, - }); - - const actual = await Message.initializeAttachmentMetadata(input); - assert.deepEqual(actual, expected); - }); - - it('should classify file attachments', async () => { - const input = getDefaultMessage({ - type: 'incoming', - conversationId: 'foo', - id: '11111111-1111-1111-1111-111111111111', - timestamp: 1523317140899, - received_at: 1523317140899, - sent_at: 1523317140800, - attachments: [ - { - contentType: MIME.APPLICATION_OCTET_STREAM, - data: Bytes.fromString('foo'), - fileName: 'foo.bin', - size: 1111, - }, - ], - }); - const expected = getDefaultMessage({ - type: 'incoming', - conversationId: 'foo', - id: '11111111-1111-1111-1111-111111111111', - timestamp: 1523317140899, - received_at: 1523317140899, - sent_at: 1523317140800, - attachments: [ - { - contentType: MIME.APPLICATION_OCTET_STREAM, - data: Bytes.fromString('foo'), - fileName: 'foo.bin', - size: 1111, - }, - ], - hasAttachments: 1, - hasVisualMediaAttachments: undefined, - hasFileAttachments: 1, - }); - - const actual = await Message.initializeAttachmentMetadata(input); - assert.deepEqual(actual, expected); - }); - - it('should classify voice message attachments', async () => { - const input = getDefaultMessage({ - type: 'incoming', - conversationId: 'foo', - id: '11111111-1111-1111-1111-111111111111', - timestamp: 1523317140899, - received_at: 1523317140899, - sent_at: 1523317140800, - attachments: [ - { - contentType: MIME.AUDIO_AAC, - flags: SignalService.AttachmentPointer.Flags.VOICE_MESSAGE, - data: Bytes.fromString('foo'), - fileName: 'Voice Message.aac', - size: 1111, - }, - ], - }); - const expected = getDefaultMessage({ - type: 'incoming', - conversationId: 'foo', - id: '11111111-1111-1111-1111-111111111111', - timestamp: 1523317140899, - received_at: 1523317140899, - sent_at: 1523317140800, - attachments: [ - { - contentType: MIME.AUDIO_AAC, - flags: SignalService.AttachmentPointer.Flags.VOICE_MESSAGE, - data: Bytes.fromString('foo'), - fileName: 'Voice Message.aac', - size: 1111, - }, - ], - hasAttachments: 1, - hasVisualMediaAttachments: undefined, - hasFileAttachments: undefined, - }); - - const actual = await Message.initializeAttachmentMetadata(input); - assert.deepEqual(actual, expected); - }); - - it('does not include long message attachments', async () => { - const input = getDefaultMessage({ - type: 'incoming', - conversationId: 'foo', - id: '11111111-1111-1111-1111-111111111111', - timestamp: 1523317140899, - received_at: 1523317140899, - sent_at: 1523317140800, - attachments: [ - { - contentType: MIME.LONG_MESSAGE, - data: Bytes.fromString('foo'), - fileName: 'message.txt', - size: 1111, - }, - ], - }); - const expected = getDefaultMessage({ - type: 'incoming', - conversationId: 'foo', - id: '11111111-1111-1111-1111-111111111111', - timestamp: 1523317140899, - received_at: 1523317140899, - sent_at: 1523317140800, - attachments: [ - { - contentType: MIME.LONG_MESSAGE, - data: Bytes.fromString('foo'), - fileName: 'message.txt', - size: 1111, - }, - ], - hasAttachments: 0, - hasVisualMediaAttachments: undefined, - hasFileAttachments: undefined, - }); - - const actual = await Message.initializeAttachmentMetadata(input); - assert.deepEqual(actual, expected); - }); - - it('handles not attachments', async () => { - const input = getDefaultMessage({ - type: 'incoming', - conversationId: 'foo', - id: '11111111-1111-1111-1111-111111111111', - timestamp: 1523317140899, - received_at: 1523317140899, - sent_at: 1523317140800, - attachments: [], - }); - const expected = getDefaultMessage({ - type: 'incoming', - conversationId: 'foo', - id: '11111111-1111-1111-1111-111111111111', - timestamp: 1523317140899, - received_at: 1523317140899, - sent_at: 1523317140800, - attachments: [], - hasAttachments: 0, - hasVisualMediaAttachments: undefined, - hasFileAttachments: undefined, - }); - - const actual = await Message.initializeAttachmentMetadata(input); - assert.deepEqual(actual, expected); - }); - }); -}); diff --git a/ts/textsecure/downloadAttachment.ts b/ts/textsecure/downloadAttachment.ts index a23cbc9b00..427903235a 100644 --- a/ts/textsecure/downloadAttachment.ts +++ b/ts/textsecure/downloadAttachment.ts @@ -114,7 +114,7 @@ export async function downloadAttachment( variant: AttachmentVariant; abortSignal: AbortSignal; } -): Promise { +): Promise { const logId = `downloadAttachment/${options.logPrefix ?? ''}`; const { digest, incrementalMac, chunkSize, key, size } = attachment; @@ -272,20 +272,17 @@ export async function downloadAttachment( // backup thumbnails don't get trimmed, so we just calculate the size as the // ciphertextSize, less IV and MAC const calculatedSize = downloadSize - IV_LENGTH - MAC_LENGTH; - return { - ...(await decryptAndReencryptLocally({ - type: 'backupThumbnail', - ciphertextPath: cipherTextAbsolutePath, - idForLogging: logId, - size: calculatedSize, - ...thumbnailEncryptionKeys, - outerEncryption: - getBackupThumbnailOuterEncryptionKeyMaterial(attachment), - getAbsoluteAttachmentPath: - window.Signal.Migrations.getAbsoluteAttachmentPath, - })), + return decryptAndReencryptLocally({ + type: 'backupThumbnail', + ciphertextPath: cipherTextAbsolutePath, + idForLogging: logId, size: calculatedSize, - }; + ...thumbnailEncryptionKeys, + outerEncryption: + getBackupThumbnailOuterEncryptionKeyMaterial(attachment), + getAbsoluteAttachmentPath: + window.Signal.Migrations.getAbsoluteAttachmentPath, + }); } default: { throw missingCaseError(options.variant); diff --git a/ts/types/Attachment.ts b/ts/types/Attachment.ts index fcf675a58e..1aa9681841 100644 --- a/ts/types/Attachment.ts +++ b/ts/types/Attachment.ts @@ -24,7 +24,11 @@ import { isImageTypeSupported, isVideoTypeSupported, } from '../util/GoogleChrome'; -import type { LocalizerType, WithRequiredProperties } from './Util'; +import type { + LocalizerType, + WithOptionalProperties, + WithRequiredProperties, +} from './Util'; import { ThemeType } from './Util'; import * as GoogleChrome from '../util/GoogleChrome'; import { ReadStatus } from '../messages/MessageReadStatus'; @@ -56,14 +60,47 @@ export class AttachmentPermanentlyUndownloadableError extends Error { } } -type ScreenshotType = Omit & { - height: number; - width: number; - path: string; - size?: number; +export type ThumbnailType = EphemeralAttachmentFields & { + size: number; + contentType: MIME.MIMEType; + path?: string; + plaintextHash?: string; + width?: number; + height?: number; + version?: 1 | 2; + localKey?: string; // AES + MAC }; -export type AttachmentType = { +export type ScreenshotType = WithOptionalProperties; +export type BackupThumbnailType = WithOptionalProperties; + +// These fields do not get saved to the DB. +export type EphemeralAttachmentFields = { + totalDownloaded?: number; + data?: Uint8Array; + /** Not included in protobuf, needs to be pulled from flags */ + isVoiceMessage?: boolean; + /** For messages not already on disk, this will be a data url */ + url?: string; + screenshotData?: Uint8Array; + /** @deprecated Legacy field */ + screenshotPath?: string; + + /** @deprecated Legacy field. Used only for downloading old attachment */ + id?: number; + /** @deprecated Legacy field, used long ago for migrating attachments to disk. */ + schemaVersion?: number; + /** @deprecated Legacy field, replaced by cdnKey */ + cdnId?: string; +}; + +/** + * Adding a field to AttachmentType requires: + * 1) adding a column to message_attachments + * 2) updating MessageAttachmentDBReferenceType and MESSAGE_ATTACHMENT_COLUMNS + * 3) saving data to the proper column + */ +export type AttachmentType = EphemeralAttachmentFields & { error?: boolean; blurHash?: string; caption?: string; @@ -73,36 +110,27 @@ export type AttachmentType = { fileName?: string; plaintextHash?: string; uploadTimestamp?: number; - /** Not included in protobuf, needs to be pulled from flags */ - isVoiceMessage?: boolean; - /** For messages not already on disk, this will be a data url */ - url?: string; size: number; pending?: boolean; width?: number; height?: number; path?: string; screenshot?: ScreenshotType; - screenshotData?: Uint8Array; - // Legacy Draft - screenshotPath?: string; flags?: number; thumbnail?: ThumbnailType; isCorrupted?: boolean; cdnNumber?: number; - cdnId?: string; cdnKey?: string; downloadPath?: string; key?: string; iv?: string; - data?: Uint8Array; + textAttachment?: TextAttachmentType; wasTooBig?: boolean; // If `true` backfill is unavailable backfillError?: boolean; - totalDownloaded?: number; incrementalMac?: string; chunkSize?: number; @@ -115,25 +143,19 @@ export type AttachmentType = { // See app/attachment_channel.ts version?: 1 | 2; localKey?: string; // AES + MAC - thumbnailFromBackup?: Pick< - AttachmentType, - 'path' | 'version' | 'plaintextHash' - >; + thumbnailFromBackup?: BackupThumbnailType; - /** Legacy field. Used only for downloading old attachments */ - id?: number; - - /** Legacy field, used long ago for migrating attachments to disk. */ - schemaVersion?: number; + /** For quote attachments, if copied from the referenced attachment */ + copied?: boolean; } & ( - | { - isReencryptableToSameDigest?: true; - } - | { - isReencryptableToSameDigest: false; - reencryptionInfo?: ReencryptionInfo; - } -); + | { + isReencryptableToSameDigest?: true; + } + | { + isReencryptableToSameDigest: false; + reencryptionInfo?: ReencryptionInfo; + } + ); export type LocalAttachmentV2Type = Readonly<{ version: 2; @@ -259,13 +281,6 @@ export type AttachmentDraftType = size: number; }; -export type ThumbnailType = AttachmentType & { - // Only used when quote needed to make an in-memory thumbnail - objectUrl?: string; - // Whether the thumbnail has been copied from the original (quoted) message - copied?: boolean; -}; - export enum AttachmentVariant { Default = 'Default', ThumbnailFromBackup = 'thumbnailFromBackup', @@ -1008,6 +1023,10 @@ export const isFile = (attachment: AttachmentType): boolean => { return false; } + if (MIME.isLongMessage(contentType)) { + return false; + } + return true; }; diff --git a/ts/types/Message.ts b/ts/types/Message.ts index 2114ca8eb6..695494ec71 100644 --- a/ts/types/Message.ts +++ b/ts/types/Message.ts @@ -4,7 +4,6 @@ import type { DurationInSeconds } from '../util/durations'; import type { AttachmentType } from './Attachment'; import type { EmbeddedContactType } from './EmbeddedContact'; -import type { IndexableBoolean, IndexablePresence } from './IndexedDB'; export function getMentionsRegex(): RegExp { return /\uFFFC/g; @@ -34,7 +33,6 @@ export type IncomingMessage = Readonly< source?: string; sourceDevice?: number; } & SharedMessageProperties & - MessageSchemaVersion5 & MessageSchemaVersion6 & ExpirationTimerUpdate >; @@ -56,7 +54,6 @@ export type OutgoingMessage = Readonly< isViewOnce?: number; synced: boolean; } & SharedMessageProperties & - MessageSchemaVersion5 & ExpirationTimerUpdate >; @@ -64,7 +61,6 @@ export type VerifiedChangeMessage = Readonly< { type: 'verified-change'; } & SharedMessageProperties & - MessageSchemaVersion5 & ExpirationTimerUpdate >; @@ -72,7 +68,6 @@ export type ProfileChangeNotificationMessage = Readonly< { type: 'profile-change'; } & SharedMessageProperties & - MessageSchemaVersion5 & ExpirationTimerUpdate >; @@ -92,14 +87,6 @@ export type ExpirationTimerUpdate = Partial< }> >; -export type MessageSchemaVersion5 = Partial< - Readonly<{ - hasAttachments: IndexableBoolean; - hasVisualMediaAttachments: IndexablePresence; - hasFileAttachments: IndexablePresence; - }> ->; - export type MessageSchemaVersion6 = Partial< Readonly<{ contact: Array; diff --git a/ts/types/Message2.ts b/ts/types/Message2.ts index 92bb7a1c93..c8c3b49f73 100644 --- a/ts/types/Message2.ts +++ b/ts/types/Message2.ts @@ -23,7 +23,6 @@ import { } from './Attachment'; import * as Errors from './errors'; import * as SchemaVersion from './SchemaVersion'; -import { initializeAttachmentMetadata } from './message/initializeAttachmentMetadata'; import { LONG_MESSAGE } from './MIME'; import type * as MIME from './MIME'; @@ -140,6 +139,8 @@ export type ContextType = { // - Attachments: write bodyAttachment to disk // Version 14 // - All attachments: ensure they are reencryptable to a known digest +// Version 15 +// - A noop migration to cause attachments to be normalized when the message is saved const INITIAL_SCHEMA_VERSION = 0; @@ -488,12 +489,10 @@ const toVersion6 = _withSchemaVersion({ schemaVersion: 6, upgrade: _mapContact(Contact.parseAndWriteAvatar(migrateDataToFileSystem)), }); -// IMPORTANT: We’ve updated our definition of `initializeAttachmentMetadata`, so -// we need to run it again on existing items that have previously been incorrectly -// classified: +// NOOP: hasFileAttachments, etc. is now computed at message save time const toVersion7 = _withSchemaVersion({ schemaVersion: 7, - upgrade: initializeAttachmentMetadata, + upgrade: noopUpgrade, }); const toVersion8 = _withSchemaVersion({ @@ -655,6 +654,7 @@ const toVersion12 = _withSchemaVersion({ return result; }, }); + const toVersion13 = _withSchemaVersion({ schemaVersion: 13, upgrade: migrateBodyAttachmentToDisk, diff --git a/ts/types/Stickers.ts b/ts/types/Stickers.ts index d5fb43585a..629898305c 100644 --- a/ts/types/Stickers.ts +++ b/ts/types/Stickers.ts @@ -31,6 +31,7 @@ import { drop } from '../util/drop'; import { isNotNil } from '../util/isNotNil'; import { encryptLegacyAttachment } from '../util/encryptLegacyAttachment'; import { AttachmentDisposition } from '../util/getLocalAttachmentUrl'; +import { getPlaintextHashForInMemoryAttachment } from '../AttachmentCrypto'; export type ActionSourceType = | 'startup' @@ -1094,7 +1095,6 @@ export async function copyStickerToAttachments( // Fall-back contentType: IMAGE_WEBP, }; - const data = await window.Signal.Migrations.readAttachmentData(newSticker); const sniffedMimeType = sniffImageMimeType(data); @@ -1106,6 +1106,8 @@ export async function copyStickerToAttachments( ); } + newSticker.plaintextHash = getPlaintextHashForInMemoryAttachment(data); + return newSticker; } diff --git a/ts/types/Util.ts b/ts/types/Util.ts index 25f5703070..e1470ebd83 100644 --- a/ts/types/Util.ts +++ b/ts/types/Util.ts @@ -113,6 +113,9 @@ export type JSONWithUnknownFields = export type WithRequiredProperties = Omit & Required>; +export type WithOptionalProperties = Omit & + Partial>; + export function getTypingIndicatorSetting(): boolean { return window.storage.get('typingIndicators', false); } diff --git a/ts/types/message/initializeAttachmentMetadata.ts b/ts/types/message/initializeAttachmentMetadata.ts deleted file mode 100644 index fa4ab7187b..0000000000 --- a/ts/types/message/initializeAttachmentMetadata.ts +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2018 Signal Messenger, LLC -// SPDX-License-Identifier: AGPL-3.0-only - -import * as Attachment from '../Attachment'; -import * as IndexedDB from '../IndexedDB'; - -import type { MessageAttributesType } from '../../model-types.d'; - -const hasAttachment = - (predicate: (value: Attachment.AttachmentType) => boolean) => - (message: MessageAttributesType): IndexedDB.IndexablePresence => - IndexedDB.toIndexablePresence((message.attachments || []).some(predicate)); - -const hasFileAttachment = hasAttachment(Attachment.isFile); -const hasVisualMediaAttachment = hasAttachment(Attachment.isVisualMedia); - -export const initializeAttachmentMetadata = async ( - message: MessageAttributesType -): Promise => { - if (message.type === 'verified-change') { - return message; - } - if (message.type === 'profile-change') { - return message; - } - if (message.messageTimer || message.isViewOnce) { - return message; - } - - const attachments = (message.attachments || []).filter( - (attachment: Attachment.AttachmentType) => - attachment.contentType !== 'text/x-signal-plain' - ); - const hasAttachments = IndexedDB.toIndexableBoolean(attachments.length > 0); - - const hasFileAttachments = hasFileAttachment({ ...message, attachments }); - const hasVisualMediaAttachments = hasVisualMediaAttachment({ - ...message, - attachments, - }); - - return { - ...message, - hasAttachments, - hasFileAttachments, - hasVisualMediaAttachments, - }; -}; diff --git a/ts/util/dropNull.ts b/ts/util/dropNull.ts index 0b35699f51..349c7a62e6 100644 --- a/ts/util/dropNull.ts +++ b/ts/util/dropNull.ts @@ -4,6 +4,9 @@ export type NullToUndefined = Extract extends never ? T : Exclude | undefined; +export type UndefinedToNull = + Extract extends never ? T : Exclude | null; + export function dropNull( value: NonNullable | null | undefined ): T | undefined { @@ -35,3 +38,23 @@ export function shallowDropNull( return result; } + +export function convertUndefinedToNull(value: T | undefined): T | null { + if (value === undefined) { + return null; + } + return value; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function shallowConvertUndefinedToNull( + obj: T +): { [P in keyof T]: UndefinedToNull } { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const result: any = {}; + + for (const [key, propertyValue] of Object.entries(obj)) { + result[key] = convertUndefinedToNull(propertyValue); + } + return result; +} diff --git a/ts/util/makeQuote.ts b/ts/util/makeQuote.ts index 3fc21667a3..c16d3a64c9 100644 --- a/ts/util/makeQuote.ts +++ b/ts/util/makeQuote.ts @@ -84,7 +84,7 @@ export async function getQuoteAttachment( thumbnail && thumbnail.path ? { ...(await loadAttachmentData(thumbnail)), - objectUrl: getLocalAttachmentUrl(thumbnail), + url: getLocalAttachmentUrl(thumbnail), } : undefined, }; @@ -123,7 +123,7 @@ export async function getQuoteAttachment( thumbnail: path ? { ...(await loadAttachmentData(sticker.data)), - objectUrl: getLocalAttachmentUrl(sticker.data), + url: getLocalAttachmentUrl(sticker.data), } : undefined, },