diff --git a/ts/test-mock/helpers.node.ts b/ts/test-mock/helpers.node.ts index 808406b50c..9b8f492e63 100644 --- a/ts/test-mock/helpers.node.ts +++ b/ts/test-mock/helpers.node.ts @@ -1,6 +1,6 @@ // Copyright 2023 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only - +import pTimeout from 'p-timeout'; import createDebug from 'debug'; import { type Device, @@ -15,6 +15,7 @@ import type { Locator, Page } from 'playwright'; import { expect } from 'playwright/test'; import type { SignalService } from '../protobuf/index.std.js'; import { strictAssert } from '../util/assert.std.js'; +import { SECOND } from '../util/durations/constants.std.js'; const debug = createDebug('mock:test:helpers'); @@ -351,7 +352,7 @@ export function getTimelineMessageWithText(page: Page, text: string): Locator { return getTimeline(page).locator('.module-message').filter({ hasText: text }); } -export async function composerAttachImages( +export async function composerAttachFiles( page: Page, filePaths: ReadonlyArray ): Promise { @@ -363,6 +364,12 @@ export async function composerAttachImages( ); debug('setting input files'); + await page + .getByRole('button', { + name: 'Add attachment or poll', + }) + .click(); + await page.getByRole('menuitem', { name: 'File' }).click(); await AttachmentInput.setInputFiles(filePaths); debug(`waiting for ${filePaths.length} items`); @@ -383,9 +390,11 @@ export async function sendMessageWithAttachments( receiver: PrimaryDevice, text: string, filePaths: Array -): Promise> { - await composerAttachImages(page, filePaths); - +): Promise<{ + attachments: Array; + timestamp: number; +}> { + await composerAttachFiles(page, filePaths); debug('sending message'); const input = await waitForEnabledComposer(page); await typeIntoInput(input, text, ''); @@ -406,14 +415,32 @@ export async function sendMessageWithAttachments( ); debug('get received message data'); - const receivedMessage = await receiver.waitForMessage(); - const attachments = receivedMessage.dataMessage.attachments ?? []; - strictAssert( - attachments.length === filePaths.length, - 'attachments must exist' - ); - return attachments; + return pTimeout( + (async () => { + // eslint-disable-next-line no-constant-condition + while (true) { + // eslint-disable-next-line no-await-in-loop + const receivedMessage = await receiver.waitForMessage(); + const attachments = receivedMessage.dataMessage.attachments ?? []; + if ( + attachments.length === filePaths.length && + receivedMessage.body === text + ) { + strictAssert( + receivedMessage.dataMessage.timestamp, + 'timestamp exists' + ); + return { + attachments, + timestamp: receivedMessage.dataMessage.timestamp.toNumber(), + }; + } + } + })(), + 10 * SECOND, + 'Timed out waiting to detect message send with attached files' + ); } export async function waitForEnabledComposer(page: Page): Promise { diff --git a/ts/test-mock/messaging/attachments_test.node.ts b/ts/test-mock/messaging/attachments_test.node.ts index dfd2c1ad6e..962d9caa72 100644 --- a/ts/test-mock/messaging/attachments_test.node.ts +++ b/ts/test-mock/messaging/attachments_test.node.ts @@ -5,8 +5,8 @@ import createDebug from 'debug'; import { assert } from 'chai'; import { expect } from 'playwright/test'; import { type PrimaryDevice, StorageState } from '@signalapp/mock-server'; -import * as path from 'node:path'; -import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { access, readFile } from 'node:fs/promises'; import type { App } from '../playwright.node.js'; import { Bootstrap } from '../bootstrap.node.js'; @@ -20,10 +20,11 @@ import * as durations from '../../util/durations/index.std.js'; import { strictAssert } from '../../util/assert.std.js'; import { VIDEO_MP4 } from '../../types/MIME.std.js'; import { toBase64 } from '../../Bytes.std.js'; +import type { AttachmentType } from '../../types/Attachment.std.js'; export const debug = createDebug('mock:test:attachments'); -const CAT_PATH = path.join( +const CAT_PATH = join( __dirname, '..', '..', @@ -31,7 +32,7 @@ const CAT_PATH = path.join( 'fixtures', 'cat-screenshot.png' ); -const VIDEO_PATH = path.join( +const VIDEO_PATH = join( __dirname, '..', '..', @@ -83,12 +84,11 @@ describe('attachments', function (this: Mocha.Suite) { await page.getByTestId(pinned.device.aci).click(); - const [attachmentCat] = await sendMessageWithAttachments( - page, - pinned, - 'This is my cat', - [CAT_PATH] - ); + const { + attachments: [attachmentCat], + } = await sendMessageWithAttachments(page, pinned, 'This is my cat', [ + CAT_PATH, + ]); const Message = getTimelineMessageWithText(page, 'This is my cat'); const MessageSent = Message.locator( @@ -212,4 +212,189 @@ describe('attachments', function (this: Mocha.Suite) { assert.strictEqual(attachmentInDB?.chunkSize, undefined); } }); + + it('reuses attachment file when receiving and sending same video and deletes only when all references removed', async () => { + const page = await app.getWindow(); + + const { desktop, phone } = bootstrap; + await page.getByTestId(pinned.device.aci).click(); + + const plaintextVideo = await readFile(VIDEO_PATH); + const ciphertextVideo = await bootstrap.encryptAndStoreAttachmentOnCDN( + plaintextVideo, + VIDEO_MP4 + ); + + const ciphertextVideo2 = await bootstrap.encryptAndStoreAttachmentOnCDN( + plaintextVideo, + VIDEO_MP4 + ); + + const phoneTimestamp = bootstrap.getTimestamp(); + const friendTimestamp = bootstrap.getTimestamp(); + + debug('receiving attachment from primary'); + await page.getByTestId(pinned.device.aci).click(); + await sendTextMessage({ + from: phone, + to: pinned, + text: 'from phone', + desktop, + attachments: [{ ...ciphertextVideo, width: 568, height: 320 }], + timestamp: phoneTimestamp, + }); + await getMessageInTimelineByTimestamp(page, phoneTimestamp) + .locator('.module-image--loaded') + .waitFor(); + + debug('receiving same attachment from contact'); + await page.getByTestId(pinned.device.aci).click(); + await sendTextMessage({ + from: pinned, + to: desktop, + text: 'from friend', + desktop, + attachments: [{ ...ciphertextVideo2, width: 568, height: 320 }], + timestamp: friendTimestamp, + }); + + await getMessageInTimelineByTimestamp(page, friendTimestamp) + .locator('.module-image--loaded') + .waitFor(); + + debug('sending same attachment from desktop'); + const { timestamp: sentTimestamp } = await sendMessageWithAttachments( + page, + pinned, + 'from desktop', + [VIDEO_PATH] + ); + + strictAssert(sentTimestamp, 'outgoing timestamp must exist'); + + const phoneDBMessage = (await app.getMessagesBySentAt(phoneTimestamp))[0]; + const phoneDBAttachment: AttachmentType = phoneDBMessage.attachments?.[0]; + + const friendDBMessage = (await app.getMessagesBySentAt(friendTimestamp))[0]; + const friendDBAttachment: AttachmentType = friendDBMessage.attachments?.[0]; + + const sentDBMessage = (await app.getMessagesBySentAt(sentTimestamp))[0]; + const sentDBAttachment: AttachmentType = sentDBMessage.attachments?.[0]; + + strictAssert(phoneDBAttachment, 'outgoing sync message exists in DB'); + strictAssert(friendDBAttachment, 'incoming message exists in DB'); + strictAssert(sentDBAttachment, 'sent message exists in DB'); + + const { path } = phoneDBAttachment; + const thumbnailPath = phoneDBAttachment.thumbnail?.path; + + strictAssert(path, 'path exists'); + strictAssert(thumbnailPath, 'thumbnail path exists'); + + debug( + 'checking that incoming and outgoing messages deduplicated data on disk' + ); + assert.strictEqual(phoneDBAttachment.path, path); + assert.strictEqual(friendDBAttachment.path, path); + assert.strictEqual(sentDBAttachment.path, path); + + assert.strictEqual(phoneDBAttachment.thumbnail?.path, thumbnailPath); + assert.strictEqual(friendDBAttachment.thumbnail?.path, thumbnailPath); + assert.strictEqual(sentDBAttachment.thumbnail?.path, thumbnailPath); + + debug('deleting two of the messages'); + const sentMessage = getMessageInTimelineByTimestamp(page, sentTimestamp); + await sentMessage.click({ button: 'right' }); + + await page.getByRole('menuitem', { name: 'Delete' }).click(); + await page.getByRole('button', { name: 'Delete for me' }).click(); + + await expect(sentMessage).not.toBeVisible(); + + const friendMessage = getMessageInTimelineByTimestamp( + page, + friendTimestamp + ); + await friendMessage.click({ button: 'right' }); + + await page.getByRole('menuitem', { name: 'Delete' }).click(); + await page.getByRole('button', { name: 'Delete for me' }).click(); + await expect(friendMessage).not.toBeVisible(); + + await app.waitForMessageToBeCleanedUp(sentDBMessage.id); + await app.waitForMessageToBeCleanedUp(friendDBMessage.id); + + assert.strictEqual( + (await app.getMessagesBySentAt(friendTimestamp)).length, + 0 + ); + assert.strictEqual( + (await app.getMessagesBySentAt(sentTimestamp)).length, + 0 + ); + // Path still exists! + await access(bootstrap.getAbsoluteAttachmentPath(path)); + + debug('delete last of messages'); + const phoneMessage = getMessageInTimelineByTimestamp(page, phoneTimestamp); + await phoneMessage.click({ button: 'right' }); + + await page.getByRole('menuitem', { name: 'Delete' }).click(); + + await page.getByRole('button', { name: 'Delete for me' }).click(); + await expect(phoneMessage).not.toBeVisible(); + + await app.waitForMessageToBeCleanedUp(phoneDBMessage.id); + assert.strictEqual( + (await app.getMessagesBySentAt(phoneTimestamp)).length, + 0 + ); + + await expect( + access(bootstrap.getAbsoluteAttachmentPath(path)) + ).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + + it('reencodes images to same data and reuses same file when sending', async () => { + const page = await app.getWindow(); + + await page.getByTestId(pinned.device.aci).click(); + + const { timestamp: firstTimestamp } = await sendMessageWithAttachments( + page, + pinned, + 'first send', + [CAT_PATH] + ); + + const { timestamp: secondTimestamp } = await sendMessageWithAttachments( + page, + pinned, + 'second send', + [CAT_PATH] + ); + + const firstAttachment: AttachmentType = ( + await app.getMessagesBySentAt(firstTimestamp) + )[0].attachments?.[0]; + + const secondAttachment: AttachmentType = ( + await app.getMessagesBySentAt(secondTimestamp) + )[0].attachments?.[0]; + + strictAssert(firstAttachment, 'firstAttachment exists in DB'); + strictAssert(secondAttachment, 'secondAttachment exists in DB'); + + debug( + 'checking that incoming and outgoing messages deduplicated data on disk' + ); + assert.strictEqual(firstAttachment.path, secondAttachment.path); + assert.strictEqual(firstAttachment.localKey, secondAttachment.localKey); + assert.strictEqual( + firstAttachment.thumbnail?.path, + secondAttachment.thumbnail?.path + ); + }); }); diff --git a/ts/test-mock/messaging/lightbox_test.node.ts b/ts/test-mock/messaging/lightbox_test.node.ts index 41e29f82b1..3379433752 100644 --- a/ts/test-mock/messaging/lightbox_test.node.ts +++ b/ts/test-mock/messaging/lightbox_test.node.ts @@ -100,17 +100,15 @@ describe('lightbox', function (this: Mocha.Suite) { 'koushik-chowdavarapu-105425-unsplash.jpg' ); - const [attachmentCat] = await sendMessageWithAttachments( - page, - pinned, - 'Message1', - [imageCat] - ); - const [attachmentSnow, attachmentWaterfall] = - await sendMessageWithAttachments(page, pinned, 'Message2', [ - imageSnow, - imageWaterfall, - ]); + const { + attachments: [attachmentCat], + } = await sendMessageWithAttachments(page, pinned, 'Message1', [imageCat]); + const { + attachments: [attachmentSnow, attachmentWaterfall], + } = await sendMessageWithAttachments(page, pinned, 'Message2', [ + imageSnow, + imageWaterfall, + ]); await sendAttachmentsBack('Message3', [attachmentCat]); await sendAttachmentsBack('Message4', [ diff --git a/ts/test-mock/playwright.node.ts b/ts/test-mock/playwright.node.ts index 84c549ddc6..6428b83d52 100644 --- a/ts/test-mock/playwright.node.ts +++ b/ts/test-mock/playwright.node.ts @@ -125,6 +125,11 @@ export class App extends EventEmitter { public async waitForMessageSend(): Promise { return this.#waitForEvent('message:send-complete'); } + public async waitForMessageToBeCleanedUp( + messageId: string + ): Promise> { + return this.#waitForEvent(`message:cleaned-up:${messageId}`); + } public async waitForConversationOpen(): Promise { return this.#waitForEvent('conversation:open'); diff --git a/ts/test-node/types/Attachment_test.std.ts b/ts/test-node/types/Attachment_test.std.ts index 0cb02069e4..f19be65fad 100644 --- a/ts/test-node/types/Attachment_test.std.ts +++ b/ts/test-node/types/Attachment_test.std.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: AGPL-3.0-only import { assert } from 'chai'; +import * as sinon from 'sinon'; import * as Attachment from '../../util/Attachment.std.js'; import type { @@ -28,6 +29,13 @@ const FAKE_LOCAL_ATTACHMENT: LocalAttachmentV2Type = { }; describe('Attachment', () => { + let sandbox: sinon.SinonSandbox; + beforeEach(() => { + sandbox = sinon.createSandbox(); + }); + afterEach(() => { + sandbox.restore(); + }); describe('getFileExtension', () => { it('should return file extension from content type', () => { const input: AttachmentType = fakeAttachment({ @@ -443,10 +451,16 @@ describe('Attachment', () => { return FAKE_LOCAL_ATTACHMENT; }; - const actual = await migrateDataToFileSystem(input, { - writeNewAttachmentData, - logger, - }); + const actual = await migrateDataToFileSystem( + input, + { + writeNewAttachmentData, + getExistingAttachmentDataForReuse: async () => null, + getPlaintextHashForInMemoryAttachment: () => 'fakeplaintextHash', + logger, + }, + { id: 'messageId' } + ); assert.deepEqual(actual, expected); }); @@ -465,10 +479,16 @@ describe('Attachment', () => { const writeNewAttachmentData = async () => FAKE_LOCAL_ATTACHMENT; - const actual = await migrateDataToFileSystem(input, { - writeNewAttachmentData, - logger, - }); + const actual = await migrateDataToFileSystem( + input, + { + writeNewAttachmentData, + getExistingAttachmentDataForReuse: async () => null, + getPlaintextHashForInMemoryAttachment: () => 'fakeplaintextHash', + logger, + }, + { id: 'messageId' } + ); assert.deepEqual(actual, expected); }); @@ -483,12 +503,60 @@ describe('Attachment', () => { const writeNewAttachmentData = async () => FAKE_LOCAL_ATTACHMENT; - const actual = await migrateDataToFileSystem(input, { - writeNewAttachmentData, - logger, - }); + const actual = await migrateDataToFileSystem( + input, + { + writeNewAttachmentData, + getExistingAttachmentDataForReuse: async () => null, + getPlaintextHashForInMemoryAttachment: () => 'fakeplaintextHash', + logger, + }, + { id: 'messageId' } + ); assert.isUndefined(actual.data); }); + it('should reuse existing data if exists', async () => { + const input = { + contentType: MIME.IMAGE_JPEG, + data: Bytes.fromString('Above us only sky'), + fileName: 'foo.jpg', + size: 1111, + }; + + const writeNewAttachmentData = sandbox.stub(); + + const actual = await migrateDataToFileSystem( + input, + { + writeNewAttachmentData, + getExistingAttachmentDataForReuse: async ({ + plaintextHash, + contentType, + }) => { + assert.strictEqual(plaintextHash, 'somePlaintextHash'); + assert.strictEqual(contentType, MIME.IMAGE_JPEG); + return { + path: 'new-path', + version: 2, + localKey: 'new-local-key', + }; + }, + getPlaintextHashForInMemoryAttachment: () => 'somePlaintextHash', + logger, + }, + { id: 'messageId' } + ); + assert.strictEqual(writeNewAttachmentData.callCount, 0); + assert.deepEqual(actual, { + version: 2, + size: 1111, + plaintextHash: 'somePlaintextHash', + contentType: MIME.IMAGE_JPEG, + path: 'new-path', + localKey: 'new-local-key', + fileName: 'foo.jpg', + }); + }); }); }); diff --git a/ts/test-node/types/EmbeddedContact_test.std.ts b/ts/test-node/types/EmbeddedContact_test.std.ts index 3e178086f7..3835b86158 100644 --- a/ts/test-node/types/EmbeddedContact_test.std.ts +++ b/ts/test-node/types/EmbeddedContact_test.std.ts @@ -24,10 +24,6 @@ const logger = createLogger('EmbeddedContact_test'); describe('Contact', () => { const NUMBER = '+12025550099'; - const writeNewAttachmentData = sinon - .stub() - .throws(new Error("Shouldn't be called")); - const getDefaultMessageAttrs = (): Pick< MessageAttributesType, | 'id' @@ -244,6 +240,16 @@ describe('Contact', () => { }); describe('parseAndWriteAvatar', () => { + const defaultContext = { + logger, + getRegionCode: () => '1', + writeNewAttachmentData: sinon + .stub() + .throws(new Error("Shouldn't be called")), + getPlaintextHashForInMemoryAttachment: sinon.stub(), + getExistingAttachmentDataForReuse: sinon.stub(), + }; + it('handles message with no avatar in contact', async () => { const upgradeAttachment = sinon .stub() @@ -268,11 +274,7 @@ describe('Contact', () => { }; const result = await upgradeVersion( message.contact[0], - { - logger, - getRegionCode: () => '1', - writeNewAttachmentData, - }, + defaultContext, message ); assert.deepEqual(result, message.contact[0]); @@ -314,9 +316,8 @@ describe('Contact', () => { const result = await upgradeVersion( message.contact[0], { + ...defaultContext, getRegionCode: () => 'US', - logger, - writeNewAttachmentData, }, message ); @@ -361,11 +362,7 @@ describe('Contact', () => { }; const result = await upgradeVersion( message.contact[0], - { - getRegionCode: () => '1', - writeNewAttachmentData, - logger, - }, + defaultContext, message ); assert.deepEqual(result, expected); @@ -449,11 +446,7 @@ describe('Contact', () => { const result = await upgradeVersion( message.contact[0], - { - getRegionCode: () => '1', - writeNewAttachmentData, - logger, - }, + defaultContext, message ); assert.deepEqual(result, expected); @@ -499,11 +492,7 @@ describe('Contact', () => { }; const result = await upgradeVersion( message.contact[0], - { - getRegionCode: () => '1', - writeNewAttachmentData, - logger, - }, + defaultContext, message ); assert.deepEqual(result, expected); @@ -549,11 +538,7 @@ describe('Contact', () => { }; const result = await upgradeVersion( message.contact[0], - { - getRegionCode: () => '1', - writeNewAttachmentData, - logger, - }, + defaultContext, message ); assert.deepEqual(result, expected); @@ -595,11 +580,7 @@ describe('Contact', () => { }; const result = await upgradeVersion( message.contact[0], - { - getRegionCode: () => '1', - writeNewAttachmentData, - logger, - }, + defaultContext, message ); assert.deepEqual(result, expected); @@ -627,11 +608,7 @@ describe('Contact', () => { }; const result = await upgradeVersion( message.contact[0], - { - getRegionCode: () => '1', - writeNewAttachmentData, - logger, - }, + defaultContext, message ); assert.deepEqual(result, message.contact[0]); diff --git a/ts/test-node/types/Message2_test.preload.ts b/ts/test-node/types/Message2_test.preload.ts index 2afeeabab9..9b608d18b2 100644 --- a/ts/test-node/types/Message2_test.preload.ts +++ b/ts/test-node/types/Message2_test.preload.ts @@ -17,6 +17,7 @@ import type { LocalAttachmentV2Type, } from '../../types/Attachment.std.js'; import type { LoggerType } from '../../types/Logging.std.js'; +import { testPlaintextHash } from '../../test-helpers/attachments.node.js'; const FAKE_LOCAL_ATTACHMENT: LocalAttachmentV2Type = { version: 2, @@ -55,6 +56,7 @@ describe('Message', () => { props?: Partial ): Message.ContextType { return { + getExistingAttachmentDataForReuse: () => Promise.resolve(null), getImageDimensions: async (_params: { objectUrl: string; logger: LoggerType; @@ -62,6 +64,7 @@ describe('Message', () => { width: 10, height: 20, }), + getPlaintextHashForInMemoryAttachment: () => testPlaintextHash(), doesAttachmentExist: async () => true, getRegionCode: () => 'region-code', logger, diff --git a/ts/types/EmbeddedContact.std.ts b/ts/types/EmbeddedContact.std.ts index 65a6a73909..310c9c1485 100644 --- a/ts/types/EmbeddedContact.std.ts +++ b/ts/types/EmbeddedContact.std.ts @@ -16,14 +16,13 @@ import type { AttachmentType, AttachmentForUIType, AttachmentWithHydratedData, - LocalAttachmentV2Type, UploadedAttachmentType, } from './Attachment.std.js'; import { toLogFormat } from './errors.std.js'; -import type { LoggerType } from './Logging.std.js'; import type { ServiceIdString } from './ServiceId.std.js'; import type { migrateDataToFileSystem } from '../util/attachments/migrateDataToFilesystem.std.js'; import { getLocalAttachmentUrl } from '../util/getLocalAttachmentUrl.std.js'; +import type { ContextType } from './Message2.preload.js'; const { omit } = lodash; @@ -231,13 +230,14 @@ export function parseAndWriteAvatar( ) { return async ( contact: EmbeddedContactType, - context: { - getRegionCode: () => string | undefined; - logger: LoggerType; - writeNewAttachmentData: ( - data: Uint8Array - ) => Promise; - }, + context: Pick< + ContextType, + | 'getRegionCode' + | 'logger' + | 'writeNewAttachmentData' + | 'getExistingAttachmentDataForReuse' + | 'getPlaintextHashForInMemoryAttachment' + >, message: ReadonlyMessageAttributesType ): Promise => { const { getRegionCode, logger } = context; @@ -249,7 +249,7 @@ export function parseAndWriteAvatar( ...contact, avatar: { ...avatar, - avatar: await upgradeAttachment(avatar.avatar, context), + avatar: await upgradeAttachment(avatar.avatar, context, message), }, } : omit(contact, ['avatar']); diff --git a/ts/types/Message2.preload.ts b/ts/types/Message2.preload.ts index 6508add436..65a4f862b8 100644 --- a/ts/types/Message2.preload.ts +++ b/ts/types/Message2.preload.ts @@ -64,6 +64,8 @@ import { deleteDownloadFile, maybeDeleteAttachmentFile, } from '../util/migrations.preload.js'; +import type { getExistingAttachmentDataForReuse } from '../util/attachments/deduplicateAttachment.preload.js'; +import type { getPlaintextHashForInMemoryAttachment } from '../AttachmentCrypto.node.js'; const { isFunction, isObject, identity } = lodash; @@ -79,6 +81,7 @@ export type ContextType = { width: number; height: number; }>; + getPlaintextHashForInMemoryAttachment: typeof getPlaintextHashForInMemoryAttachment; getRegionCode: () => string | undefined; logger: LoggerType; makeImageThumbnail: (params: { @@ -104,6 +107,7 @@ export type ContextType = { writeNewAttachmentData: (data: Uint8Array) => Promise; writeNewStickerData: (data: Uint8Array) => Promise; maybeDeleteAttachmentFile: (path: string) => Promise<{ wasDeleted: boolean }>; + getExistingAttachmentDataForReuse: typeof getExistingAttachmentDataForReuse; }; // Schema version history @@ -540,7 +544,11 @@ const toVersion10 = _withSchemaVersion({ ...stickerMessage, sticker: { ...sticker, - data: await migrateDataToFileSystem(sticker.data, stickerContext), + data: await migrateDataToFileSystem( + sticker.data, + stickerContext, + stickerMessage + ), }, }; } diff --git a/ts/util/attachments/migrateDataToFilesystem.std.ts b/ts/util/attachments/migrateDataToFilesystem.std.ts index bbcdebfd38..fdd12c1366 100644 --- a/ts/util/attachments/migrateDataToFilesystem.std.ts +++ b/ts/util/attachments/migrateDataToFilesystem.std.ts @@ -2,11 +2,9 @@ // SPDX-License-Identifier: AGPL-3.0-only import lodash from 'lodash'; -import type { - AttachmentType, - LocalAttachmentV2Type, -} from '../../types/Attachment.std.js'; -import type { LoggerType } from '../../types/Logging.std.js'; +import type { AttachmentType } from '../../types/Attachment.std.js'; +import type { ContextType } from '../../types/Message2.preload.js'; +import type { MessageAttributesType } from '../../model-types.js'; const { isFunction, isTypedArray, isUndefined, omit } = lodash; @@ -14,13 +12,17 @@ export async function migrateDataToFileSystem( attachment: AttachmentType, { writeNewAttachmentData, + getExistingAttachmentDataForReuse, + getPlaintextHashForInMemoryAttachment, logger, - }: { - writeNewAttachmentData: ( - data: Uint8Array - ) => Promise; - logger: LoggerType; - } + }: Pick< + ContextType, + | 'writeNewAttachmentData' + | 'getExistingAttachmentDataForReuse' + | 'getPlaintextHashForInMemoryAttachment' + | 'logger' + >, + message: Pick ): Promise { if (!isFunction(writeNewAttachmentData)) { throw new TypeError("'writeNewAttachmentData' must be a function"); @@ -42,8 +44,24 @@ export async function migrateDataToFileSystem( return omit({ ...attachment }, ['data']); } - const local = await writeNewAttachmentData(data); + const plaintextHash = getPlaintextHashForInMemoryAttachment(data); - const attachmentWithoutData = omit({ ...attachment, ...local }, ['data']); - return attachmentWithoutData; + const existingData = await getExistingAttachmentDataForReuse({ + plaintextHash, + messageId: message.id, + contentType: attachment.contentType, + logId: 'migrateDataToFileSystem', + }); + const attachmentWithoutData = omit(attachment, ['data']); + + if (existingData) { + return { + ...attachmentWithoutData, + plaintextHash, + ...existingData, + }; + } + + const newAttachmentData = await writeNewAttachmentData(data); + return { ...attachmentWithoutData, ...newAttachmentData }; } diff --git a/ts/util/cleanup.preload.ts b/ts/util/cleanup.preload.ts index 662f1e9632..cd1d7ad0e0 100644 --- a/ts/util/cleanup.preload.ts +++ b/ts/util/cleanup.preload.ts @@ -129,6 +129,12 @@ export async function cleanupMessages( DataReader.getBackupAttachmentDownloadProgress ) ); + + if (window.SignalCI) { + messages.forEach(msg => { + window.SignalCI?.handleEvent(`message:cleaned-up:${msg.id}`, null); + }); + } } /** Removes a message from redux caches & MessageCache, but does NOT delete files on disk, diff --git a/ts/util/migrations.preload.ts b/ts/util/migrations.preload.ts index c7fbef84bd..ce38d07d2c 100644 --- a/ts/util/migrations.preload.ts +++ b/ts/util/migrations.preload.ts @@ -51,6 +51,8 @@ import { MEGAPHONES_PATH, } from './basePaths.preload.js'; import { DataReader } from '../sql/Client.preload.js'; +import { getExistingAttachmentDataForReuse } from './attachments/deduplicateAttachment.preload.js'; +import { getPlaintextHashForInMemoryAttachment } from '../AttachmentCrypto.node.js'; const logger = createLogger('migrations'); @@ -215,7 +217,9 @@ export const upgradeMessageSchema = ( return upgradeSchema(message, { maybeDeleteAttachmentFile, doesAttachmentExist, + getExistingAttachmentDataForReuse, getImageDimensions, + getPlaintextHashForInMemoryAttachment, getRegionCode: () => itemStorage.get('regionCode'), makeImageThumbnail, makeObjectUrl,