Reuse files on disk for outgoing messages

ts/test-mock/helpers.node.ts

@@ -1,6 +1,6 @@
 // Copyright 2023 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
-
+import pTimeout from 'p-timeout';
 import createDebug from 'debug';
 import {
   type Device,
@@ -15,6 +15,7 @@ import type { Locator, Page } from 'playwright';
 import { expect } from 'playwright/test';
 import type { SignalService } from '../protobuf/index.std.js';
 import { strictAssert } from '../util/assert.std.js';
+import { SECOND } from '../util/durations/constants.std.js';
 
 const debug = createDebug('mock:test:helpers');
 
@@ -351,7 +352,7 @@ export function getTimelineMessageWithText(page: Page, text: string): Locator {
   return getTimeline(page).locator('.module-message').filter({ hasText: text });
 }
 
-export async function composerAttachImages(
+export async function composerAttachFiles(
   page: Page,
   filePaths: ReadonlyArray<string>
 ): Promise<void> {
@@ -363,6 +364,12 @@ export async function composerAttachImages(
   );
 
   debug('setting input files');
+  await page
+    .getByRole('button', {
+      name: 'Add attachment or poll',
+    })
+    .click();
+  await page.getByRole('menuitem', { name: 'File' }).click();
   await AttachmentInput.setInputFiles(filePaths);
 
   debug(`waiting for ${filePaths.length} items`);
@@ -383,9 +390,11 @@ export async function sendMessageWithAttachments(
   receiver: PrimaryDevice,
   text: string,
   filePaths: Array<string>
-): Promise<Array<SignalService.IAttachmentPointer>> {
-  await composerAttachImages(page, filePaths);
-
+): Promise<{
+  attachments: Array<SignalService.IAttachmentPointer>;
+  timestamp: number;
+}> {
+  await composerAttachFiles(page, filePaths);
   debug('sending message');
   const input = await waitForEnabledComposer(page);
   await typeIntoInput(input, text, '');
@@ -406,14 +415,32 @@ export async function sendMessageWithAttachments(
   );
 
   debug('get received message data');
-  const receivedMessage = await receiver.waitForMessage();
-  const attachments = receivedMessage.dataMessage.attachments ?? [];
-  strictAssert(
-    attachments.length === filePaths.length,
-    'attachments must exist'
-  );
 
-  return attachments;
+  return pTimeout(
+    (async () => {
+      // eslint-disable-next-line no-constant-condition
+      while (true) {
+        // eslint-disable-next-line no-await-in-loop
+        const receivedMessage = await receiver.waitForMessage();
+        const attachments = receivedMessage.dataMessage.attachments ?? [];
+        if (
+          attachments.length === filePaths.length &&
+          receivedMessage.body === text
+        ) {
+          strictAssert(
+            receivedMessage.dataMessage.timestamp,
+            'timestamp exists'
+          );
+          return {
+            attachments,
+            timestamp: receivedMessage.dataMessage.timestamp.toNumber(),
+          };
+        }
+      }
+    })(),
+    10 * SECOND,
+    'Timed out waiting to detect message send with attached files'
+  );
 }
 
 export async function waitForEnabledComposer(page: Page): Promise<Locator> {

ts/test-mock/messaging/attachments_test.node.ts

@@ -5,8 +5,8 @@ import createDebug from 'debug';
 import { assert } from 'chai';
 import { expect } from 'playwright/test';
 import { type PrimaryDevice, StorageState } from '@signalapp/mock-server';
-import * as path from 'node:path';
-import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { access, readFile } from 'node:fs/promises';
 
 import type { App } from '../playwright.node.js';
 import { Bootstrap } from '../bootstrap.node.js';
@@ -20,10 +20,11 @@ import * as durations from '../../util/durations/index.std.js';
 import { strictAssert } from '../../util/assert.std.js';
 import { VIDEO_MP4 } from '../../types/MIME.std.js';
 import { toBase64 } from '../../Bytes.std.js';
+import type { AttachmentType } from '../../types/Attachment.std.js';
 
 export const debug = createDebug('mock:test:attachments');
 
-const CAT_PATH = path.join(
+const CAT_PATH = join(
   __dirname,
   '..',
   '..',
@@ -31,7 +32,7 @@ const CAT_PATH = path.join(
   'fixtures',
   'cat-screenshot.png'
 );
-const VIDEO_PATH = path.join(
+const VIDEO_PATH = join(
   __dirname,
   '..',
   '..',
@@ -83,12 +84,11 @@ describe('attachments', function (this: Mocha.Suite) {
 
     await page.getByTestId(pinned.device.aci).click();
 
-    const [attachmentCat] = await sendMessageWithAttachments(
-      page,
-      pinned,
-      'This is my cat',
-      [CAT_PATH]
-    );
+    const {
+      attachments: [attachmentCat],
+    } = await sendMessageWithAttachments(page, pinned, 'This is my cat', [
+      CAT_PATH,
+    ]);
 
     const Message = getTimelineMessageWithText(page, 'This is my cat');
     const MessageSent = Message.locator(
@@ -212,4 +212,189 @@ describe('attachments', function (this: Mocha.Suite) {
       assert.strictEqual(attachmentInDB?.chunkSize, undefined);
     }
   });
+
+  it('reuses attachment file when receiving and sending same video and deletes only when all references removed', async () => {
+    const page = await app.getWindow();
+
+    const { desktop, phone } = bootstrap;
+    await page.getByTestId(pinned.device.aci).click();
+
+    const plaintextVideo = await readFile(VIDEO_PATH);
+    const ciphertextVideo = await bootstrap.encryptAndStoreAttachmentOnCDN(
+      plaintextVideo,
+      VIDEO_MP4
+    );
+
+    const ciphertextVideo2 = await bootstrap.encryptAndStoreAttachmentOnCDN(
+      plaintextVideo,
+      VIDEO_MP4
+    );
+
+    const phoneTimestamp = bootstrap.getTimestamp();
+    const friendTimestamp = bootstrap.getTimestamp();
+
+    debug('receiving attachment from primary');
+    await page.getByTestId(pinned.device.aci).click();
+    await sendTextMessage({
+      from: phone,
+      to: pinned,
+      text: 'from phone',
+      desktop,
+      attachments: [{ ...ciphertextVideo, width: 568, height: 320 }],
+      timestamp: phoneTimestamp,
+    });
+    await getMessageInTimelineByTimestamp(page, phoneTimestamp)
+      .locator('.module-image--loaded')
+      .waitFor();
+
+    debug('receiving same attachment from contact');
+    await page.getByTestId(pinned.device.aci).click();
+    await sendTextMessage({
+      from: pinned,
+      to: desktop,
+      text: 'from friend',
+      desktop,
+      attachments: [{ ...ciphertextVideo2, width: 568, height: 320 }],
+      timestamp: friendTimestamp,
+    });
+
+    await getMessageInTimelineByTimestamp(page, friendTimestamp)
+      .locator('.module-image--loaded')
+      .waitFor();
+
+    debug('sending same attachment from desktop');
+    const { timestamp: sentTimestamp } = await sendMessageWithAttachments(
+      page,
+      pinned,
+      'from desktop',
+      [VIDEO_PATH]
+    );
+
+    strictAssert(sentTimestamp, 'outgoing timestamp must exist');
+
+    const phoneDBMessage = (await app.getMessagesBySentAt(phoneTimestamp))[0];
+    const phoneDBAttachment: AttachmentType = phoneDBMessage.attachments?.[0];
+
+    const friendDBMessage = (await app.getMessagesBySentAt(friendTimestamp))[0];
+    const friendDBAttachment: AttachmentType = friendDBMessage.attachments?.[0];
+
+    const sentDBMessage = (await app.getMessagesBySentAt(sentTimestamp))[0];
+    const sentDBAttachment: AttachmentType = sentDBMessage.attachments?.[0];
+
+    strictAssert(phoneDBAttachment, 'outgoing sync message exists in DB');
+    strictAssert(friendDBAttachment, 'incoming message exists in DB');
+    strictAssert(sentDBAttachment, 'sent message exists in DB');
+
+    const { path } = phoneDBAttachment;
+    const thumbnailPath = phoneDBAttachment.thumbnail?.path;
+
+    strictAssert(path, 'path exists');
+    strictAssert(thumbnailPath, 'thumbnail path exists');
+
+    debug(
+      'checking that incoming and outgoing messages deduplicated data on disk'
+    );
+    assert.strictEqual(phoneDBAttachment.path, path);
+    assert.strictEqual(friendDBAttachment.path, path);
+    assert.strictEqual(sentDBAttachment.path, path);
+
+    assert.strictEqual(phoneDBAttachment.thumbnail?.path, thumbnailPath);
+    assert.strictEqual(friendDBAttachment.thumbnail?.path, thumbnailPath);
+    assert.strictEqual(sentDBAttachment.thumbnail?.path, thumbnailPath);
+
+    debug('deleting two of the messages');
+    const sentMessage = getMessageInTimelineByTimestamp(page, sentTimestamp);
+    await sentMessage.click({ button: 'right' });
+
+    await page.getByRole('menuitem', { name: 'Delete' }).click();
+    await page.getByRole('button', { name: 'Delete for me' }).click();
+
+    await expect(sentMessage).not.toBeVisible();
+
+    const friendMessage = getMessageInTimelineByTimestamp(
+      page,
+      friendTimestamp
+    );
+    await friendMessage.click({ button: 'right' });
+
+    await page.getByRole('menuitem', { name: 'Delete' }).click();
+    await page.getByRole('button', { name: 'Delete for me' }).click();
+    await expect(friendMessage).not.toBeVisible();
+
+    await app.waitForMessageToBeCleanedUp(sentDBMessage.id);
+    await app.waitForMessageToBeCleanedUp(friendDBMessage.id);
+
+    assert.strictEqual(
+      (await app.getMessagesBySentAt(friendTimestamp)).length,
+      0
+    );
+    assert.strictEqual(
+      (await app.getMessagesBySentAt(sentTimestamp)).length,
+      0
+    );
+    // Path still exists!
+    await access(bootstrap.getAbsoluteAttachmentPath(path));
+
+    debug('delete last of messages');
+    const phoneMessage = getMessageInTimelineByTimestamp(page, phoneTimestamp);
+    await phoneMessage.click({ button: 'right' });
+
+    await page.getByRole('menuitem', { name: 'Delete' }).click();
+
+    await page.getByRole('button', { name: 'Delete for me' }).click();
+    await expect(phoneMessage).not.toBeVisible();
+
+    await app.waitForMessageToBeCleanedUp(phoneDBMessage.id);
+    assert.strictEqual(
+      (await app.getMessagesBySentAt(phoneTimestamp)).length,
+      0
+    );
+
+    await expect(
+      access(bootstrap.getAbsoluteAttachmentPath(path))
+    ).rejects.toMatchObject({
+      code: 'ENOENT',
+    });
+  });
+
+  it('reencodes images to same data and reuses same file when sending', async () => {
+    const page = await app.getWindow();
+
+    await page.getByTestId(pinned.device.aci).click();
+
+    const { timestamp: firstTimestamp } = await sendMessageWithAttachments(
+      page,
+      pinned,
+      'first send',
+      [CAT_PATH]
+    );
+
+    const { timestamp: secondTimestamp } = await sendMessageWithAttachments(
+      page,
+      pinned,
+      'second send',
+      [CAT_PATH]
+    );
+
+    const firstAttachment: AttachmentType = (
+      await app.getMessagesBySentAt(firstTimestamp)
+    )[0].attachments?.[0];
+
+    const secondAttachment: AttachmentType = (
+      await app.getMessagesBySentAt(secondTimestamp)
+    )[0].attachments?.[0];
+
+    strictAssert(firstAttachment, 'firstAttachment exists in DB');
+    strictAssert(secondAttachment, 'secondAttachment exists in DB');
+
+    debug(
+      'checking that incoming and outgoing messages deduplicated data on disk'
+    );
+    assert.strictEqual(firstAttachment.path, secondAttachment.path);
+    assert.strictEqual(firstAttachment.localKey, secondAttachment.localKey);
+    assert.strictEqual(
+      firstAttachment.thumbnail?.path,
+      secondAttachment.thumbnail?.path
+    );
+  });
 });

ts/test-mock/messaging/lightbox_test.node.ts

@@ -100,17 +100,15 @@ describe('lightbox', function (this: Mocha.Suite) {
       'koushik-chowdavarapu-105425-unsplash.jpg'
     );
 
-    const [attachmentCat] = await sendMessageWithAttachments(
-      page,
-      pinned,
-      'Message1',
-      [imageCat]
-    );
-    const [attachmentSnow, attachmentWaterfall] =
-      await sendMessageWithAttachments(page, pinned, 'Message2', [
-        imageSnow,
-        imageWaterfall,
-      ]);
+    const {
+      attachments: [attachmentCat],
+    } = await sendMessageWithAttachments(page, pinned, 'Message1', [imageCat]);
+    const {
+      attachments: [attachmentSnow, attachmentWaterfall],
+    } = await sendMessageWithAttachments(page, pinned, 'Message2', [
+      imageSnow,
+      imageWaterfall,
+    ]);
 
     await sendAttachmentsBack('Message3', [attachmentCat]);
     await sendAttachmentsBack('Message4', [

ts/test-mock/playwright.node.ts

@@ -125,6 +125,11 @@ export class App extends EventEmitter {
   public async waitForMessageSend(): Promise<MessageSendInfoType> {
     return this.#waitForEvent('message:send-complete');
   }
+  public async waitForMessageToBeCleanedUp(
+    messageId: string
+  ): Promise<Array<string>> {
+    return this.#waitForEvent(`message:cleaned-up:${messageId}`);
+  }
 
   public async waitForConversationOpen(): Promise<ConversationOpenInfoType> {
     return this.#waitForEvent('conversation:open');

ts/test-node/types/Attachment_test.std.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import { assert } from 'chai';
+import * as sinon from 'sinon';
 
 import * as Attachment from '../../util/Attachment.std.js';
 import type {
@@ -28,6 +29,13 @@ const FAKE_LOCAL_ATTACHMENT: LocalAttachmentV2Type = {
 };
 
 describe('Attachment', () => {
+  let sandbox: sinon.SinonSandbox;
+  beforeEach(() => {
+    sandbox = sinon.createSandbox();
+  });
+  afterEach(() => {
+    sandbox.restore();
+  });
   describe('getFileExtension', () => {
     it('should return file extension from content type', () => {
       const input: AttachmentType = fakeAttachment({
@@ -443,10 +451,16 @@ describe('Attachment', () => {
         return FAKE_LOCAL_ATTACHMENT;
       };
 
-      const actual = await migrateDataToFileSystem(input, {
-        writeNewAttachmentData,
-        logger,
-      });
+      const actual = await migrateDataToFileSystem(
+        input,
+        {
+          writeNewAttachmentData,
+          getExistingAttachmentDataForReuse: async () => null,
+          getPlaintextHashForInMemoryAttachment: () => 'fakeplaintextHash',
+          logger,
+        },
+        { id: 'messageId' }
+      );
       assert.deepEqual(actual, expected);
     });
 
@@ -465,10 +479,16 @@ describe('Attachment', () => {
 
       const writeNewAttachmentData = async () => FAKE_LOCAL_ATTACHMENT;
 
-      const actual = await migrateDataToFileSystem(input, {
-        writeNewAttachmentData,
-        logger,
-      });
+      const actual = await migrateDataToFileSystem(
+        input,
+        {
+          writeNewAttachmentData,
+          getExistingAttachmentDataForReuse: async () => null,
+          getPlaintextHashForInMemoryAttachment: () => 'fakeplaintextHash',
+          logger,
+        },
+        { id: 'messageId' }
+      );
       assert.deepEqual(actual, expected);
     });
 
@@ -483,12 +503,60 @@ describe('Attachment', () => {
 
       const writeNewAttachmentData = async () => FAKE_LOCAL_ATTACHMENT;
 
-      const actual = await migrateDataToFileSystem(input, {
-        writeNewAttachmentData,
-        logger,
-      });
+      const actual = await migrateDataToFileSystem(
+        input,
+        {
+          writeNewAttachmentData,
+          getExistingAttachmentDataForReuse: async () => null,
+          getPlaintextHashForInMemoryAttachment: () => 'fakeplaintextHash',
+          logger,
+        },
+        { id: 'messageId' }
+      );
 
       assert.isUndefined(actual.data);
     });
+    it('should reuse existing data if exists', async () => {
+      const input = {
+        contentType: MIME.IMAGE_JPEG,
+        data: Bytes.fromString('Above us only sky'),
+        fileName: 'foo.jpg',
+        size: 1111,
+      };
+
+      const writeNewAttachmentData = sandbox.stub();
+
+      const actual = await migrateDataToFileSystem(
+        input,
+        {
+          writeNewAttachmentData,
+          getExistingAttachmentDataForReuse: async ({
+            plaintextHash,
+            contentType,
+          }) => {
+            assert.strictEqual(plaintextHash, 'somePlaintextHash');
+            assert.strictEqual(contentType, MIME.IMAGE_JPEG);
+            return {
+              path: 'new-path',
+              version: 2,
+              localKey: 'new-local-key',
+            };
+          },
+          getPlaintextHashForInMemoryAttachment: () => 'somePlaintextHash',
+          logger,
+        },
+        { id: 'messageId' }
+      );
+      assert.strictEqual(writeNewAttachmentData.callCount, 0);
+      assert.deepEqual(actual, {
+        version: 2,
+        size: 1111,
+        plaintextHash: 'somePlaintextHash',
+        contentType: MIME.IMAGE_JPEG,
+        path: 'new-path',
+        localKey: 'new-local-key',
+        fileName: 'foo.jpg',
+      });
+    });
   });
 });

ts/test-node/types/EmbeddedContact_test.std.ts

@@ -24,10 +24,6 @@ const logger = createLogger('EmbeddedContact_test');
 describe('Contact', () => {
   const NUMBER = '+12025550099';
 
-  const writeNewAttachmentData = sinon
-    .stub()
-    .throws(new Error("Shouldn't be called"));
-
   const getDefaultMessageAttrs = (): Pick<
     MessageAttributesType,
     | 'id'
@@ -244,6 +240,16 @@ describe('Contact', () => {
   });
 
   describe('parseAndWriteAvatar', () => {
+    const defaultContext = {
+      logger,
+      getRegionCode: () => '1',
+      writeNewAttachmentData: sinon
+        .stub()
+        .throws(new Error("Shouldn't be called")),
+      getPlaintextHashForInMemoryAttachment: sinon.stub(),
+      getExistingAttachmentDataForReuse: sinon.stub(),
+    };
+
     it('handles message with no avatar in contact', async () => {
       const upgradeAttachment = sinon
         .stub()
@@ -268,11 +274,7 @@ describe('Contact', () => {
       };
       const result = await upgradeVersion(
         message.contact[0],
-        {
-          logger,
-          getRegionCode: () => '1',
-          writeNewAttachmentData,
-        },
+        defaultContext,
         message
       );
       assert.deepEqual(result, message.contact[0]);
@@ -314,9 +316,8 @@ describe('Contact', () => {
       const result = await upgradeVersion(
         message.contact[0],
         {
+          ...defaultContext,
           getRegionCode: () => 'US',
-          logger,
-          writeNewAttachmentData,
         },
         message
       );
@@ -361,11 +362,7 @@ describe('Contact', () => {
       };
       const result = await upgradeVersion(
         message.contact[0],
-        {
-          getRegionCode: () => '1',
-          writeNewAttachmentData,
-          logger,
-        },
+        defaultContext,
         message
       );
       assert.deepEqual(result, expected);
@@ -449,11 +446,7 @@ describe('Contact', () => {
 
       const result = await upgradeVersion(
         message.contact[0],
-        {
-          getRegionCode: () => '1',
-          writeNewAttachmentData,
-          logger,
-        },
+        defaultContext,
         message
       );
       assert.deepEqual(result, expected);
@@ -499,11 +492,7 @@ describe('Contact', () => {
       };
       const result = await upgradeVersion(
         message.contact[0],
-        {
-          getRegionCode: () => '1',
-          writeNewAttachmentData,
-          logger,
-        },
+        defaultContext,
         message
       );
       assert.deepEqual(result, expected);
@@ -549,11 +538,7 @@ describe('Contact', () => {
       };
       const result = await upgradeVersion(
         message.contact[0],
-        {
-          getRegionCode: () => '1',
-          writeNewAttachmentData,
-          logger,
-        },
+        defaultContext,
         message
       );
       assert.deepEqual(result, expected);
@@ -595,11 +580,7 @@ describe('Contact', () => {
       };
       const result = await upgradeVersion(
         message.contact[0],
-        {
-          getRegionCode: () => '1',
-          writeNewAttachmentData,
-          logger,
-        },
+        defaultContext,
         message
       );
       assert.deepEqual(result, expected);
@@ -627,11 +608,7 @@ describe('Contact', () => {
       };
       const result = await upgradeVersion(
         message.contact[0],
-        {
-          getRegionCode: () => '1',
-          writeNewAttachmentData,
-          logger,
-        },
+        defaultContext,
         message
       );
       assert.deepEqual(result, message.contact[0]);

ts/test-node/types/Message2_test.preload.ts

@@ -17,6 +17,7 @@ import type {
   LocalAttachmentV2Type,
 } from '../../types/Attachment.std.js';
 import type { LoggerType } from '../../types/Logging.std.js';
+import { testPlaintextHash } from '../../test-helpers/attachments.node.js';
 
 const FAKE_LOCAL_ATTACHMENT: LocalAttachmentV2Type = {
   version: 2,
@@ -55,6 +56,7 @@ describe('Message', () => {
     props?: Partial<Message.ContextType>
   ): Message.ContextType {
     return {
+      getExistingAttachmentDataForReuse: () => Promise.resolve(null),
       getImageDimensions: async (_params: {
         objectUrl: string;
         logger: LoggerType;
@@ -62,6 +64,7 @@ describe('Message', () => {
         width: 10,
         height: 20,
       }),
+      getPlaintextHashForInMemoryAttachment: () => testPlaintextHash(),
       doesAttachmentExist: async () => true,
       getRegionCode: () => 'region-code',
       logger,

ts/types/EmbeddedContact.std.ts

@@ -16,14 +16,13 @@ import type {
   AttachmentType,
   AttachmentForUIType,
   AttachmentWithHydratedData,
-  LocalAttachmentV2Type,
   UploadedAttachmentType,
 } from './Attachment.std.js';
 import { toLogFormat } from './errors.std.js';
-import type { LoggerType } from './Logging.std.js';
 import type { ServiceIdString } from './ServiceId.std.js';
 import type { migrateDataToFileSystem } from '../util/attachments/migrateDataToFilesystem.std.js';
 import { getLocalAttachmentUrl } from '../util/getLocalAttachmentUrl.std.js';
+import type { ContextType } from './Message2.preload.js';
 
 const { omit } = lodash;
 
@@ -231,13 +230,14 @@ export function parseAndWriteAvatar(
 ) {
   return async (
     contact: EmbeddedContactType,
-    context: {
-      getRegionCode: () => string | undefined;
-      logger: LoggerType;
-      writeNewAttachmentData: (
-        data: Uint8Array
-      ) => Promise<LocalAttachmentV2Type>;
-    },
+    context: Pick<
+      ContextType,
+      | 'getRegionCode'
+      | 'logger'
+      | 'writeNewAttachmentData'
+      | 'getExistingAttachmentDataForReuse'
+      | 'getPlaintextHashForInMemoryAttachment'
+    >,
     message: ReadonlyMessageAttributesType
   ): Promise<EmbeddedContactType> => {
     const { getRegionCode, logger } = context;
@@ -249,7 +249,7 @@ export function parseAndWriteAvatar(
             ...contact,
             avatar: {
               ...avatar,
-              avatar: await upgradeAttachment(avatar.avatar, context),
+              avatar: await upgradeAttachment(avatar.avatar, context, message),
             },
           }
         : omit(contact, ['avatar']);

ts/types/Message2.preload.ts

@@ -64,6 +64,8 @@ import {
   deleteDownloadFile,
   maybeDeleteAttachmentFile,
 } from '../util/migrations.preload.js';
+import type { getExistingAttachmentDataForReuse } from '../util/attachments/deduplicateAttachment.preload.js';
+import type { getPlaintextHashForInMemoryAttachment } from '../AttachmentCrypto.node.js';
 
 const { isFunction, isObject, identity } = lodash;
 
@@ -79,6 +81,7 @@ export type ContextType = {
     width: number;
     height: number;
   }>;
+  getPlaintextHashForInMemoryAttachment: typeof getPlaintextHashForInMemoryAttachment;
   getRegionCode: () => string | undefined;
   logger: LoggerType;
   makeImageThumbnail: (params: {
@@ -104,6 +107,7 @@ export type ContextType = {
   writeNewAttachmentData: (data: Uint8Array) => Promise<LocalAttachmentV2Type>;
   writeNewStickerData: (data: Uint8Array) => Promise<LocalAttachmentV2Type>;
   maybeDeleteAttachmentFile: (path: string) => Promise<{ wasDeleted: boolean }>;
+  getExistingAttachmentDataForReuse: typeof getExistingAttachmentDataForReuse;
 };
 
 // Schema version history
@@ -540,7 +544,11 @@ const toVersion10 = _withSchemaVersion({
           ...stickerMessage,
           sticker: {
             ...sticker,
-            data: await migrateDataToFileSystem(sticker.data, stickerContext),
+            data: await migrateDataToFileSystem(
+              sticker.data,
+              stickerContext,
+              stickerMessage
+            ),
           },
         };
       }

ts/util/attachments/migrateDataToFilesystem.std.ts

@@ -2,11 +2,9 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import lodash from 'lodash';
-import type {
-  AttachmentType,
-  LocalAttachmentV2Type,
-} from '../../types/Attachment.std.js';
-import type { LoggerType } from '../../types/Logging.std.js';
+import type { AttachmentType } from '../../types/Attachment.std.js';
+import type { ContextType } from '../../types/Message2.preload.js';
+import type { MessageAttributesType } from '../../model-types.js';
 
 const { isFunction, isTypedArray, isUndefined, omit } = lodash;
 
@@ -14,13 +12,17 @@ export async function migrateDataToFileSystem(
   attachment: AttachmentType,
   {
     writeNewAttachmentData,
+    getExistingAttachmentDataForReuse,
+    getPlaintextHashForInMemoryAttachment,
     logger,
-  }: {
-    writeNewAttachmentData: (
-      data: Uint8Array
-    ) => Promise<LocalAttachmentV2Type>;
-    logger: LoggerType;
-  }
+  }: Pick<
+    ContextType,
+    | 'writeNewAttachmentData'
+    | 'getExistingAttachmentDataForReuse'
+    | 'getPlaintextHashForInMemoryAttachment'
+    | 'logger'
+  >,
+  message: Pick<MessageAttributesType, 'id'>
 ): Promise<AttachmentType> {
   if (!isFunction(writeNewAttachmentData)) {
     throw new TypeError("'writeNewAttachmentData' must be a function");
@@ -42,8 +44,24 @@ export async function migrateDataToFileSystem(
     return omit({ ...attachment }, ['data']);
   }
 
-  const local = await writeNewAttachmentData(data);
+  const plaintextHash = getPlaintextHashForInMemoryAttachment(data);
 
-  const attachmentWithoutData = omit({ ...attachment, ...local }, ['data']);
-  return attachmentWithoutData;
+  const existingData = await getExistingAttachmentDataForReuse({
+    plaintextHash,
+    messageId: message.id,
+    contentType: attachment.contentType,
+    logId: 'migrateDataToFileSystem',
+  });
+  const attachmentWithoutData = omit(attachment, ['data']);
+
+  if (existingData) {
+    return {
+      ...attachmentWithoutData,
+      plaintextHash,
+      ...existingData,
+    };
+  }
+
+  const newAttachmentData = await writeNewAttachmentData(data);
+  return { ...attachmentWithoutData, ...newAttachmentData };
 }

ts/util/cleanup.preload.ts

@@ -129,6 +129,12 @@ export async function cleanupMessages(
       DataReader.getBackupAttachmentDownloadProgress
     )
   );
+
+  if (window.SignalCI) {
+    messages.forEach(msg => {
+      window.SignalCI?.handleEvent(`message:cleaned-up:${msg.id}`, null);
+    });
+  }
 }
 
 /** Removes a message from redux caches & MessageCache, but does NOT delete files on disk,

ts/util/migrations.preload.ts

@@ -51,6 +51,8 @@ import {
   MEGAPHONES_PATH,
 } from './basePaths.preload.js';
 import { DataReader } from '../sql/Client.preload.js';
+import { getExistingAttachmentDataForReuse } from './attachments/deduplicateAttachment.preload.js';
+import { getPlaintextHashForInMemoryAttachment } from '../AttachmentCrypto.node.js';
 
 const logger = createLogger('migrations');
 
@@ -215,7 +217,9 @@ export const upgradeMessageSchema = (
   return upgradeSchema(message, {
     maybeDeleteAttachmentFile,
     doesAttachmentExist,
+    getExistingAttachmentDataForReuse,
     getImageDimensions,
+    getPlaintextHashForInMemoryAttachment,
     getRegionCode: () => itemStorage.get('regionCode'),
     makeImageThumbnail,
     makeObjectUrl,