New attachment storage system

ts/types/Attachment.ts

@@ -30,6 +30,7 @@ import { strictAssert } from '../util/assert';
 import type { SignalService as Proto } from '../protobuf';
 import { isMoreRecentThan } from '../util/timestamp';
 import { DAY } from '../util/durations';
+import { getLocalAttachmentUrl } from '../util/getLocalAttachmentUrl';
 
 const MAX_WIDTH = 300;
 const MAX_HEIGHT = MAX_WIDTH * 1.5;
@@ -40,6 +41,13 @@ const MIN_HEIGHT = 50;
 
 export class AttachmentSizeError extends Error {}
 
+type ScreenshotType = Omit<AttachmentType, 'size'> & {
+  height: number;
+  width: number;
+  path: string;
+  size?: number;
+};
+
 export type AttachmentType = {
   error?: boolean;
   blurHash?: string;
@@ -60,15 +68,9 @@ export type AttachmentType = {
   width?: number;
   height?: number;
   path?: string;
-  screenshot?: {
-    height: number;
-    width: number;
-    url?: string;
-    contentType: MIME.MIMEType;
-    path: string;
-    data?: Uint8Array;
-  };
+  screenshot?: ScreenshotType;
   screenshotData?: Uint8Array;
+  // Legacy Draft
   screenshotPath?: string;
   flags?: number;
   thumbnail?: ThumbnailType;
@@ -90,6 +92,10 @@ export type AttachmentType = {
     cdnNumber?: number;
   };
 
+  // See app/attachment_channel.ts
+  version?: 1 | 2;
+  localKey?: string; // AES + MAC
+
   /** Legacy field. Used only for downloading old attachments */
   id?: number;
 
@@ -97,6 +103,24 @@ export type AttachmentType = {
   schemaVersion?: number;
 };
 
+export type LocalAttachmentV2Type = Readonly<{
+  version: 2;
+  path: string;
+  localKey: string;
+  plaintextHash: string;
+  size: number;
+}>;
+
+export type AddressableAttachmentType = Readonly<{
+  version?: 1 | 2;
+  path: string;
+  localKey?: string;
+  size?: number;
+
+  // In-memory data, for outgoing attachments that are not saved to disk.
+  data?: Uint8Array;
+}>;
+
 export type UploadedAttachmentType = Proto.IAttachmentPointer &
   Readonly<{
     // Required fields
@@ -138,10 +162,6 @@ export type TextAttachmentType = {
   color?: number | null;
 };
 
-export type DownloadedAttachmentType = AttachmentType & {
-  data: Uint8Array;
-};
-
 export type BaseAttachmentDraftType = {
   blurHash?: string;
   contentType: MIME.MIMEType;
@@ -174,6 +194,8 @@ export type InMemoryAttachmentDraftType =
 export type AttachmentDraftType =
   | ({
       url?: string;
+      screenshot?: ScreenshotType;
+      // Legacy field
       screenshotPath?: string;
       pending: false;
       // Old draft attachments may have a caption, though they are no longer editable
@@ -184,6 +206,8 @@ export type AttachmentDraftType =
       width?: number;
       height?: number;
       clientUuid: string;
+      version?: 2;
+      localKey?: string;
     } & BaseAttachmentDraftType)
   | {
       clientUuid: string;
@@ -318,11 +342,13 @@ export function hasData(attachment: AttachmentType): boolean {
 }
 
 export function loadData(
-  readAttachmentData: (path: string) => Promise<Uint8Array>
+  readAttachmentV2Data: (
+    attachment: Partial<AddressableAttachmentType>
+  ) => Promise<Uint8Array>
 ): (
-  attachment: Pick<AttachmentType, 'data' | 'path'>
+  attachment: Partial<AttachmentType>
 ) => Promise<AttachmentWithHydratedData> {
-  if (!isFunction(readAttachmentData)) {
+  if (!isFunction(readAttachmentV2Data)) {
     throw new TypeError("'readAttachmentData' must be a function");
   }
 
@@ -340,7 +366,7 @@ export function loadData(
       throw new TypeError("'attachment.path' is required");
     }
 
-    const data = await readAttachmentData(attachment.path);
+    const data = await readAttachmentV2Data(attachment);
     return { ...attachment, data, size: data.byteLength };
   };
 }
@@ -378,8 +404,9 @@ const THUMBNAIL_CONTENT_TYPE = MIME.IMAGE_PNG;
 export async function captureDimensionsAndScreenshot(
   attachment: AttachmentType,
   params: {
-    writeNewAttachmentData: (data: Uint8Array) => Promise<string>;
-    getAbsoluteAttachmentPath: (path: string) => string;
+    writeNewAttachmentData: (
+      data: Uint8Array
+    ) => Promise<LocalAttachmentV2Type>;
     makeObjectUrl: (
       data: Uint8Array | ArrayBuffer,
       contentType: MIME.MIMEType
@@ -410,7 +437,6 @@ export async function captureDimensionsAndScreenshot(
 
   const {
     writeNewAttachmentData,
-    getAbsoluteAttachmentPath,
     makeObjectUrl,
     revokeObjectUrl,
     getImageDimensions: getImageDimensionsFromURL,
@@ -431,24 +457,24 @@ export async function captureDimensionsAndScreenshot(
     return attachment;
   }
 
-  const absolutePath = getAbsoluteAttachmentPath(attachment.path);
+  const localUrl = getLocalAttachmentUrl(attachment);
 
   if (GoogleChrome.isImageTypeSupported(contentType)) {
     try {
       const { width, height } = await getImageDimensionsFromURL({
-        objectUrl: absolutePath,
+        objectUrl: localUrl,
         logger,
       });
       const thumbnailBuffer = await blobToArrayBuffer(
         await makeImageThumbnail({
           size: THUMBNAIL_SIZE,
-          objectUrl: absolutePath,
+          objectUrl: localUrl,
           contentType: THUMBNAIL_CONTENT_TYPE,
           logger,
         })
       );
 
-      const thumbnailPath = await writeNewAttachmentData(
+      const thumbnail = await writeNewAttachmentData(
         new Uint8Array(thumbnailBuffer)
       );
       return {
@@ -456,11 +482,10 @@ export async function captureDimensionsAndScreenshot(
         width,
         height,
         thumbnail: {
-          path: thumbnailPath,
+          ...thumbnail,
           contentType: THUMBNAIL_CONTENT_TYPE,
           width: THUMBNAIL_SIZE,
           height: THUMBNAIL_SIZE,
-          size: thumbnailBuffer.byteLength,
         },
       };
     } catch (error) {
@@ -477,7 +502,7 @@ export async function captureDimensionsAndScreenshot(
   try {
     const screenshotBuffer = await blobToArrayBuffer(
       await makeVideoScreenshot({
-        objectUrl: absolutePath,
+        objectUrl: localUrl,
         contentType: THUMBNAIL_CONTENT_TYPE,
         logger,
       })
@@ -490,7 +515,7 @@ export async function captureDimensionsAndScreenshot(
       objectUrl: screenshotObjectUrl,
       logger,
     });
-    const screenshotPath = await writeNewAttachmentData(
+    const screenshot = await writeNewAttachmentData(
       new Uint8Array(screenshotBuffer)
     );
 
@@ -503,24 +528,23 @@ export async function captureDimensionsAndScreenshot(
       })
     );
 
-    const thumbnailPath = await writeNewAttachmentData(
+    const thumbnail = await writeNewAttachmentData(
       new Uint8Array(thumbnailBuffer)
     );
 
     return {
       ...attachment,
       screenshot: {
+        ...screenshot,
         contentType: THUMBNAIL_CONTENT_TYPE,
-        path: screenshotPath,
         width,
         height,
       },
       thumbnail: {
-        path: thumbnailPath,
+        ...thumbnail,
         contentType: THUMBNAIL_CONTENT_TYPE,
         width: THUMBNAIL_SIZE,
         height: THUMBNAIL_SIZE,
-        size: thumbnailBuffer.byteLength,
       },
       width,
       height,
@@ -663,14 +687,18 @@ export function hasImage(attachments?: ReadonlyArray<AttachmentType>): boolean {
   );
 }
 
-export function isVideo(attachments?: ReadonlyArray<AttachmentType>): boolean {
+export function isVideo(
+  attachments?: ReadonlyArray<Pick<AttachmentType, 'contentType'>>
+): boolean {
   if (!attachments || attachments.length === 0) {
     return false;
   }
   return isVideoAttachment(attachments[0]);
 }
 
-export function isVideoAttachment(attachment?: AttachmentType): boolean {
+export function isVideoAttachment(
+  attachment?: Pick<AttachmentType, 'contentType'>
+): boolean {
   if (!attachment || !attachment.contentType) {
     return false;
   }
@@ -914,7 +942,9 @@ export const save = async ({
 }: {
   attachment: AttachmentType;
   index?: number;
-  readAttachmentData: (relativePath: string) => Promise<Uint8Array>;
+  readAttachmentData: (
+    attachment: Partial<AddressableAttachmentType>
+  ) => Promise<Uint8Array>;
   saveAttachmentToDisk: (options: {
     data: Uint8Array;
     name: string;
@@ -923,7 +953,7 @@ export const save = async ({
 }): Promise<string | null> => {
   let data: Uint8Array;
   if (attachment.path) {
-    data = await readAttachmentData(attachment.path);
+    data = await readAttachmentData(attachment);
   } else if (attachment.data) {
     data = attachment.data;
   } else {

ts/types/AttachmentBackup.ts

@@ -27,6 +27,9 @@ type StandardAttachmentBackupJobType = {
       uploadTimestamp?: number;
     };
     size: number;
+
+    version?: 2;
+    localKey?: string;
   };
 };
 
@@ -57,6 +60,8 @@ const standardBackupJobDataSchema = z.object({
         uploadTimestamp: z.number().optional(),
       })
       .optional(),
+    version: z.literal(2).optional(),
+    localKey: z.string().optional(),
   }),
 });
 

ts/types/Avatar.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import type { AvatarColorType } from './Colors';
+import type { AddressableAttachmentType } from './Attachment';
 import { strictAssert } from '../util/assert';
 
 export const PersonalAvatarIcons = [
@@ -35,15 +36,15 @@ export const GroupAvatarIcons = [
 ] as const;
 
 export type ContactAvatarType =
-  | {
+  | ({
       // Downloaded avatar
       path: string;
       url?: string;
       hash?: string;
-    }
+    } & Partial<AddressableAttachmentType>)
   | {
       // Not-yet downloaded avatar
-      path?: string;
+      path?: undefined;
       url: string;
       hash?: string;
     };
@@ -59,8 +60,13 @@ export type AvatarDataType = {
   buffer?: Uint8Array;
   color?: AvatarColorType;
   icon?: AvatarIconType;
-  imagePath?: string;
   text?: string;
+  imagePath?: string;
+
+  // LocalAttachmentV2Type compatibility (except for `path` being `imagePath`)
+  version?: 2;
+  localKey?: string;
+  size?: number;
 };
 
 export type DeleteAvatarFromDiskActionType = (

ts/types/Conversation.ts

@@ -3,6 +3,7 @@
 
 import type { ConversationAttributesType } from '../model-types.d';
 import type { ContactAvatarType } from './Avatar';
+import type { LocalAttachmentV2Type } from './Attachment';
 import { computeHash } from '../Crypto';
 
 export type BuildAvatarUpdaterOptions = Readonly<{
@@ -10,7 +11,7 @@ export type BuildAvatarUpdaterOptions = Readonly<{
   newAvatar?: ContactAvatarType;
   deleteAttachmentData: (path: string) => Promise<void>;
   doesAttachmentExist: (path: string) => Promise<boolean>;
-  writeNewAttachmentData: (data: Uint8Array) => Promise<string>;
+  writeNewAttachmentData: (data: Uint8Array) => Promise<LocalAttachmentV2Type>;
 }>;
 
 // This function is ready to handle raw avatar data as well as an avatar which has
@@ -49,7 +50,7 @@ function buildAvatarUpdater({ field }: { field: 'avatar' | 'profileAvatar' }) {
           ...conversation,
           [field]: {
             hash: newHash,
-            path: await writeNewAttachmentData(data),
+            ...(await writeNewAttachmentData(data)),
           },
         };
       }
@@ -91,7 +92,7 @@ function buildAvatarUpdater({ field }: { field: 'avatar' | 'profileAvatar' }) {
         ...conversation,
         [field]: {
           hash: newHash,
-          path: await writeNewAttachmentData(data),
+          ...(await writeNewAttachmentData(data)),
         },
       };
     }

ts/types/Crypto.ts

@@ -13,3 +13,9 @@ export enum CipherType {
 }
 
 export const UUID_BYTE_SIZE = 16;
+
+export const IV_LENGTH = 16;
+
+export const KEY_LENGTH = 32;
+
+export const MAC_LENGTH = 32;

ts/types/EmbeddedContact.ts

@@ -14,12 +14,14 @@ import {
 import type {
   AttachmentType,
   AttachmentWithHydratedData,
+  LocalAttachmentV2Type,
   UploadedAttachmentType,
 } from './Attachment';
 import { toLogFormat } from './errors';
 import type { LoggerType } from './Logging';
 import type { ServiceIdString } from './ServiceId';
 import type { migrateDataToFileSystem } from '../util/attachments/migrateDataToFilesystem';
+import { getLocalAttachmentUrl } from '../util/getLocalAttachmentUrl';
 
 type GenericEmbeddedContactType<AvatarType> = {
   name?: Name;
@@ -150,11 +152,9 @@ export function embeddedContactSelector(
     regionCode?: string;
     firstNumber?: string;
     serviceId?: ServiceIdString;
-    getAbsoluteAttachmentPath: (path: string) => string;
   }
 ): EmbeddedContactType {
-  const { getAbsoluteAttachmentPath, firstNumber, serviceId, regionCode } =
-    options;
+  const { firstNumber, serviceId, regionCode } = options;
 
   let { avatar } = contact;
   if (avatar && avatar.avatar) {
@@ -166,7 +166,7 @@ export function embeddedContactSelector(
         avatar: {
           ...avatar.avatar,
           path: avatar.avatar.path
-            ? getAbsoluteAttachmentPath(avatar.avatar.path)
+            ? getLocalAttachmentUrl(avatar.avatar)
             : undefined,
         },
       };
@@ -209,7 +209,9 @@ export function parseAndWriteAvatar(
       message: MessageAttributesType;
       getRegionCode: () => string | undefined;
       logger: LoggerType;
-      writeNewAttachmentData: (data: Uint8Array) => Promise<string>;
+      writeNewAttachmentData: (
+        data: Uint8Array
+      ) => Promise<LocalAttachmentV2Type>;
     }
   ): Promise<EmbeddedContactType> => {
     const { message, getRegionCode, logger } = context;

ts/types/Message2.ts

@@ -1,13 +1,17 @@
 // Copyright 2018 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
-import { isFunction, isObject, isString, omit } from 'lodash';
+import { isFunction, isObject } from 'lodash';
 
 import * as Contact from './EmbeddedContact';
-import type { AttachmentType, AttachmentWithHydratedData } from './Attachment';
+import type {
+  AddressableAttachmentType,
+  AttachmentType,
+  AttachmentWithHydratedData,
+  LocalAttachmentV2Type,
+} from './Attachment';
 import {
   captureDimensionsAndScreenshot,
-  hasData,
   removeSchemaVersion,
   replaceUnicodeOrderOverrides,
   replaceUnicodeV2,
@@ -33,8 +37,12 @@ import type {
   LinkPreviewWithHydratedData,
 } from './message/LinkPreviews';
 import type { StickerType, StickerWithHydratedData } from './Stickers';
-import { addPlaintextHashToAttachment } from '../AttachmentCrypto';
 import { migrateDataToFileSystem } from '../util/attachments/migrateDataToFilesystem';
+import {
+  getLocalAttachmentUrl,
+  AttachmentDisposition,
+} from '../util/getLocalAttachmentUrl';
+import { encryptLegacyAttachment } from '../util/encryptLegacyAttachment';
 
 export { hasExpiration } from './Message';
 
@@ -42,8 +50,6 @@ export const GROUP = 'group';
 export const PRIVATE = 'private';
 
 export type ContextType = {
-  getAbsoluteAttachmentPath: (path: string) => string;
-  getAbsoluteStickerPath: (path: string) => string;
   getImageDimensions: (params: {
     objectUrl: string;
     logger: LoggerType;
@@ -70,15 +76,14 @@ export type ContextType = {
   }) => Promise<Blob>;
   maxVersion?: number;
   revokeObjectUrl: (objectUrl: string) => void;
-  writeNewAttachmentData: (data: Uint8Array) => Promise<string>;
-  writeNewStickerData: (data: Uint8Array) => Promise<string>;
+  readAttachmentData: (
+    attachment: Partial<AddressableAttachmentType>
+  ) => Promise<Uint8Array>;
+  writeNewAttachmentData: (data: Uint8Array) => Promise<LocalAttachmentV2Type>;
+  writeNewStickerData: (data: Uint8Array) => Promise<LocalAttachmentV2Type>;
   deleteOnDisk: (path: string) => Promise<void>;
 };
 
-type WriteExistingAttachmentDataType = (
-  attachment: Pick<AttachmentType, 'data' | 'path'>
-) => Promise<string>;
-
 export type ContextWithMessageType = ContextType & {
   message: MessageAttributesType;
 };
@@ -449,7 +454,109 @@ const toVersion10 = _withSchemaVersion({
 
 const toVersion11 = _withSchemaVersion({
   schemaVersion: 11,
-  upgrade: _mapAttachments(addPlaintextHashToAttachment),
+  // NOOP: We no longer need to get plaintextHash here because we get it once
+  // we migrate attachments to v2.
+  upgrade: noopUpgrade,
+});
+
+const toVersion12 = _withSchemaVersion({
+  schemaVersion: 12,
+  upgrade: async (message, context) => {
+    const { attachments, quote, contact, preview, sticker } = message;
+
+    const result = { ...message };
+
+    if (attachments?.length) {
+      result.attachments = await Promise.all(
+        attachments.map(async attachment => {
+          const copy = await encryptLegacyAttachment(attachment, context);
+          if (copy.thumbnail) {
+            copy.thumbnail = await encryptLegacyAttachment(
+              copy.thumbnail,
+              context
+            );
+          }
+          if (copy.screenshot) {
+            copy.screenshot = await encryptLegacyAttachment(
+              copy.screenshot,
+              context
+            );
+          }
+          return copy;
+        })
+      );
+    }
+
+    if (quote && quote.attachments?.length) {
+      try {
+        result.quote = {
+          ...quote,
+          attachments: await Promise.all(
+            quote.attachments.map(async quoteAttachment => {
+              return {
+                ...quoteAttachment,
+                thumbnail:
+                  quoteAttachment.thumbnail &&
+                  (await encryptLegacyAttachment(
+                    quoteAttachment.thumbnail,
+                    context
+                  )),
+              };
+            })
+          ),
+        };
+      } catch (error) {
+        context.logger.error(`Failed to migrate quote for ${message.id}`);
+      }
+    }
+
+    if (contact?.length) {
+      result.contact = await Promise.all(
+        contact.map(async c => {
+          if (!c.avatar?.avatar) {
+            return c;
+          }
+
+          return {
+            ...c,
+            avatar: {
+              ...c.avatar,
+              avatar: await encryptLegacyAttachment(c.avatar.avatar, context),
+            },
+          };
+        })
+      );
+    }
+
+    if (preview?.length) {
+      result.preview = await Promise.all(
+        preview.map(async p => {
+          if (!p.image) {
+            return p;
+          }
+
+          return {
+            ...p,
+            image: await encryptLegacyAttachment(p.image, context),
+          };
+        })
+      );
+    }
+
+    if (sticker) {
+      result.sticker = {
+        ...sticker,
+        data: sticker.data && {
+          ...(await encryptLegacyAttachment(sticker.data, context)),
+          thumbnail:
+            sticker.data.thumbnail &&
+            (await encryptLegacyAttachment(sticker.data.thumbnail, context)),
+        },
+      };
+    }
+
+    return result;
+  },
 });
 
 const VERSIONS = [
@@ -465,6 +572,7 @@ const VERSIONS = [
   toVersion9,
   toVersion10,
   toVersion11,
+  toVersion12,
 ];
 export const CURRENT_SCHEMA_VERSION = VERSIONS.length - 1;
 
@@ -475,10 +583,9 @@ export const VERSION_NEEDED_FOR_DISPLAY = 9;
 export const upgradeSchema = async (
   rawMessage: MessageAttributesType,
   {
+    readAttachmentData,
     writeNewAttachmentData,
     getRegionCode,
-    getAbsoluteAttachmentPath,
-    getAbsoluteStickerPath,
     makeObjectUrl,
     revokeObjectUrl,
     getImageDimensions,
@@ -490,15 +597,15 @@ export const upgradeSchema = async (
     maxVersion = CURRENT_SCHEMA_VERSION,
   }: ContextType
 ): Promise<MessageAttributesType> => {
+  if (!isFunction(readAttachmentData)) {
+    throw new TypeError('context.readAttachmentData is required');
+  }
   if (!isFunction(writeNewAttachmentData)) {
     throw new TypeError('context.writeNewAttachmentData is required');
   }
   if (!isFunction(getRegionCode)) {
     throw new TypeError('context.getRegionCode is required');
   }
-  if (!isFunction(getAbsoluteAttachmentPath)) {
-    throw new TypeError('context.getAbsoluteAttachmentPath is required');
-  }
   if (!isFunction(makeObjectUrl)) {
     throw new TypeError('context.makeObjectUrl is required');
   }
@@ -517,9 +624,6 @@ export const upgradeSchema = async (
   if (!isObject(logger)) {
     throw new TypeError('context.logger is required');
   }
-  if (!isFunction(getAbsoluteStickerPath)) {
-    throw new TypeError('context.getAbsoluteStickerPath is required');
-  }
   if (!isFunction(writeNewStickerData)) {
     throw new TypeError('context.writeNewStickerData is required');
   }
@@ -538,15 +642,14 @@ export const upgradeSchema = async (
     //   each step dependent on the previous
     // eslint-disable-next-line no-await-in-loop
     message = await currentVersion(message, {
+      readAttachmentData,
       writeNewAttachmentData,
-      getAbsoluteAttachmentPath,
       makeObjectUrl,
       revokeObjectUrl,
       getImageDimensions,
       makeImageThumbnail,
       makeVideoScreenshot,
       logger,
-      getAbsoluteStickerPath,
       getRegionCode,
       writeNewStickerData,
       deleteOnDisk,
@@ -562,7 +665,6 @@ export const processNewAttachment = async (
   attachment: AttachmentType,
   {
     writeNewAttachmentData,
-    getAbsoluteAttachmentPath,
     makeObjectUrl,
     revokeObjectUrl,
     getImageDimensions,
@@ -572,7 +674,6 @@ export const processNewAttachment = async (
   }: Pick<
     ContextType,
     | 'writeNewAttachmentData'
-    | 'getAbsoluteAttachmentPath'
     | 'makeObjectUrl'
     | 'revokeObjectUrl'
     | 'getImageDimensions'
@@ -585,9 +686,6 @@ export const processNewAttachment = async (
   if (!isFunction(writeNewAttachmentData)) {
     throw new TypeError('context.writeNewAttachmentData is required');
   }
-  if (!isFunction(getAbsoluteAttachmentPath)) {
-    throw new TypeError('context.getAbsoluteAttachmentPath is required');
-  }
   if (!isFunction(makeObjectUrl)) {
     throw new TypeError('context.makeObjectUrl is required');
   }
@@ -609,7 +707,6 @@ export const processNewAttachment = async (
 
   const finalAttachment = await captureDimensionsAndScreenshot(attachment, {
     writeNewAttachmentData,
-    getAbsoluteAttachmentPath,
     makeObjectUrl,
     revokeObjectUrl,
     getImageDimensions,
@@ -623,25 +720,16 @@ export const processNewAttachment = async (
 
 export const processNewSticker = async (
   stickerData: Uint8Array,
+  isEphemeral: boolean,
   {
     writeNewStickerData,
-    getAbsoluteStickerPath,
     getImageDimensions,
     logger,
-  }: Pick<
-    ContextType,
-    | 'writeNewStickerData'
-    | 'getAbsoluteStickerPath'
-    | 'getImageDimensions'
-    | 'logger'
-  >
-): Promise<{ path: string; width: number; height: number }> => {
+  }: Pick<ContextType, 'writeNewStickerData' | 'getImageDimensions' | 'logger'>
+): Promise<LocalAttachmentV2Type & { width: number; height: number }> => {
   if (!isFunction(writeNewStickerData)) {
     throw new TypeError('context.writeNewStickerData is required');
   }
-  if (!isFunction(getAbsoluteStickerPath)) {
-    throw new TypeError('context.getAbsoluteStickerPath is required');
-  }
   if (!isFunction(getImageDimensions)) {
     throw new TypeError('context.getImageDimensions is required');
   }
@@ -649,23 +737,27 @@ export const processNewSticker = async (
     throw new TypeError('context.logger is required');
   }
 
-  const path = await writeNewStickerData(stickerData);
-  const absolutePath = await getAbsoluteStickerPath(path);
+  const local = await writeNewStickerData(stickerData);
+  const url = await getLocalAttachmentUrl(local, {
+    disposition: isEphemeral
+      ? AttachmentDisposition.Temporary
+      : AttachmentDisposition.Sticker,
+  });
 
   const { width, height } = await getImageDimensions({
-    objectUrl: absolutePath,
+    objectUrl: url,
     logger,
   });
 
   return {
-    path,
+    ...local,
     width,
     height,
   };
 };
 
 type LoadAttachmentType = (
-  attachment: Pick<AttachmentType, 'data' | 'path'>
+  attachment: Partial<AttachmentType>
 ) => Promise<AttachmentWithHydratedData>;
 
 export const createAttachmentLoader = (
@@ -932,168 +1024,3 @@ async function deletePreviews(
     })
   );
 }
-
-//      createAttachmentDataWriter :: (RelativePath -> IO Unit)
-//                                    Message ->
-//                                    IO (Promise Message)
-export const createAttachmentDataWriter = ({
-  writeExistingAttachmentData,
-  logger,
-}: {
-  writeExistingAttachmentData: WriteExistingAttachmentDataType;
-  logger: LoggerType;
-}): ((message: MessageAttributesType) => Promise<MessageAttributesType>) => {
-  if (!isFunction(writeExistingAttachmentData)) {
-    throw new TypeError(
-      'createAttachmentDataWriter: writeExistingAttachmentData must be a function'
-    );
-  }
-  if (!isObject(logger)) {
-    throw new TypeError('createAttachmentDataWriter: logger must be an object');
-  }
-
-  return async (
-    rawMessage: MessageAttributesType
-  ): Promise<MessageAttributesType> => {
-    if (!isValid(rawMessage)) {
-      throw new TypeError("'rawMessage' is not valid");
-    }
-
-    const message = initializeSchemaVersion({
-      message: rawMessage,
-      logger,
-    });
-
-    const { attachments, quote, contact, preview } = message;
-    const hasFilesToWrite =
-      (quote && quote.attachments && quote.attachments.length > 0) ||
-      (attachments && attachments.length > 0) ||
-      (contact && contact.length > 0) ||
-      (preview && preview.length > 0);
-
-    if (!hasFilesToWrite) {
-      return message;
-    }
-
-    const lastVersionWithAttachmentDataInMemory = 2;
-    const willAttachmentsGoToFileSystemOnUpgrade =
-      (message.schemaVersion || 0) <= lastVersionWithAttachmentDataInMemory;
-    if (willAttachmentsGoToFileSystemOnUpgrade) {
-      return message;
-    }
-
-    (attachments || []).forEach(attachment => {
-      if (!hasData(attachment)) {
-        throw new TypeError(
-          "'attachment.data' is required during message import"
-        );
-      }
-
-      if (!isString(attachment.path)) {
-        throw new TypeError(
-          "'attachment.path' is required during message import"
-        );
-      }
-    });
-
-    const writeQuoteAttachment = async (attachment: QuotedAttachmentType) => {
-      const { thumbnail } = attachment;
-      if (!thumbnail) {
-        return attachment;
-      }
-
-      const { data, path } = thumbnail;
-
-      // we want to be bulletproof to attachments without data
-      if (!data || !path) {
-        logger.warn(
-          'quote attachment had neither data nor path.',
-          'id:',
-          message.id,
-          'source:',
-          message.source
-        );
-        return attachment;
-      }
-
-      await writeExistingAttachmentData(thumbnail);
-      return {
-        ...attachment,
-        thumbnail: omit(thumbnail, ['data']),
-      };
-    };
-
-    const writeContactAvatar = async (
-      messageContact: EmbeddedContactType
-    ): Promise<EmbeddedContactType> => {
-      const { avatar } = messageContact;
-      if (!avatar) {
-        return messageContact;
-      }
-
-      if (avatar && !avatar.avatar) {
-        return omit(messageContact, ['avatar']);
-      }
-
-      await writeExistingAttachmentData(avatar.avatar);
-
-      return {
-        ...messageContact,
-        avatar: { ...avatar, avatar: omit(avatar.avatar, ['data']) },
-      };
-    };
-
-    const writePreviewImage = async (
-      item: LinkPreviewType
-    ): Promise<LinkPreviewType> => {
-      const { image } = item;
-      if (!image) {
-        return omit(item, ['image']);
-      }
-
-      await writeExistingAttachmentData(image);
-
-      return { ...item, image: omit(image, ['data']) };
-    };
-
-    const messageWithoutAttachmentData = {
-      ...message,
-      ...(quote
-        ? {
-            quote: {
-              ...quote,
-              attachments: await Promise.all(
-                (quote?.attachments || []).map(writeQuoteAttachment)
-              ),
-            },
-          }
-        : undefined),
-      contact: await Promise.all((contact || []).map(writeContactAvatar)),
-      preview: await Promise.all((preview || []).map(writePreviewImage)),
-      attachments: await Promise.all(
-        (attachments || []).map(async attachment => {
-          await writeExistingAttachmentData(attachment);
-
-          if (attachment.screenshot && attachment.screenshot.data) {
-            await writeExistingAttachmentData(attachment.screenshot);
-          }
-          if (attachment.thumbnail && attachment.thumbnail.data) {
-            await writeExistingAttachmentData(attachment.thumbnail);
-          }
-
-          return {
-            ...omit(attachment, ['data']),
-            ...(attachment.thumbnail
-              ? { thumbnail: omit(attachment.thumbnail, ['data']) }
-              : null),
-            ...(attachment.screenshot
-              ? { screenshot: omit(attachment.screenshot, ['data']) }
-              : null),
-          };
-        })
-      ),
-    };
-
-    return messageWithoutAttachmentData;
-  };
-};

ts/types/Stickers.ts

@@ -13,7 +13,6 @@ import * as Bytes from '../Bytes';
 import * as Errors from './errors';
 import { deriveStickerPackKey, decryptAttachmentV1 } from '../Crypto';
 import { IMAGE_WEBP } from './MIME';
-import type { MIMEType } from './MIME';
 import { sniffImageMimeType } from '../util/sniffImageMimeType';
 import type { AttachmentType, AttachmentWithHydratedData } from './Attachment';
 import type {
@@ -27,6 +26,9 @@ import * as log from '../logging/log';
 import type { StickersStateType } from '../state/ducks/stickers';
 import { MINUTE } from '../util/durations';
 import { drop } from '../util/drop';
+import { isNotNil } from '../util/isNotNil';
+import { encryptLegacyAttachment } from '../util/encryptLegacyAttachment';
+import { AttachmentDisposition } from '../util/getLocalAttachmentUrl';
 
 export type StickerType = {
   packId: string;
@@ -37,6 +39,8 @@ export type StickerType = {
   path?: string;
   width?: number;
   height?: number;
+  version?: 2;
+  localKey?: string;
 };
 export type StickerWithHydratedData = StickerType & {
   data: AttachmentWithHydratedData;
@@ -872,25 +876,27 @@ export async function copyStickerToAttachments(
   const { path, size } =
     await window.Signal.Migrations.copyIntoAttachmentsDirectory(absolutePath);
 
-  const data = await window.Signal.Migrations.readAttachmentData(path);
+  const newSticker: AttachmentType = {
+    ...sticker,
+    path,
+    size,
+
+    // Fall-back
+    contentType: IMAGE_WEBP,
+  };
+
+  const data = await window.Signal.Migrations.readAttachmentData(newSticker);
 
-  let contentType: MIMEType;
   const sniffedMimeType = sniffImageMimeType(data);
   if (sniffedMimeType) {
-    contentType = sniffedMimeType;
+    newSticker.contentType = sniffedMimeType;
   } else {
     log.warn(
       'copyStickerToAttachments: Unable to sniff sticker MIME type; falling back to WebP'
     );
-    contentType = IMAGE_WEBP;
   }
 
-  return {
-    ...sticker,
-    contentType,
-    path,
-    size,
-  };
+  return newSticker;
 }
 
 // In the case where a sticker pack is uninstalled, we want to delete it if there are no
@@ -948,3 +954,69 @@ async function deletePack(packId: string): Promise<void> {
     concurrency: 3,
   });
 }
+
+export async function encryptLegacyStickers(): Promise<void> {
+  const CONCURRENCY = 32;
+
+  const all = await Data.getAllStickers();
+
+  log.info(`encryptLegacyStickers: checking ${all.length}`);
+
+  const updated = (
+    await pMap(
+      all,
+      async sticker => {
+        try {
+          return await encryptLegacySticker(sticker);
+        } catch (error) {
+          log.error('encryptLegacyStickers: processing failed', error);
+          return undefined;
+        }
+      },
+      {
+        concurrency: CONCURRENCY,
+      }
+    )
+  ).filter(isNotNil);
+
+  await Data.createOrUpdateStickers(updated.map(({ sticker }) => sticker));
+
+  log.info(`encryptLegacyStickers: updated ${updated.length}`);
+
+  await pMap(
+    updated,
+    async ({ cleanup }) => {
+      try {
+        await cleanup();
+      } catch (error) {
+        log.error('encryptLegacyStickers: cleanup failed', error);
+      }
+    },
+    {
+      concurrency: CONCURRENCY,
+    }
+  );
+
+  log.info(`encryptLegacyStickers: cleaned up ${updated.length}`);
+}
+
+async function encryptLegacySticker(
+  sticker: StickerFromDBType
+): Promise<
+  { sticker: StickerFromDBType; cleanup: () => Promise<void> } | undefined
+> {
+  const { deleteSticker, readStickerData, writeNewStickerData } =
+    window.Signal.Migrations;
+
+  const updated = await encryptLegacyAttachment(sticker, {
+    readAttachmentData: readStickerData,
+    writeNewAttachmentData: writeNewStickerData,
+    disposition: AttachmentDisposition.Sticker,
+  });
+
+  if (updated !== sticker && sticker.path) {
+    return { sticker: updated, cleanup: () => deleteSticker(sticker.path) };
+  }
+
+  return undefined;
+}

ts/types/Storage.d.ts

@@ -169,6 +169,7 @@ export type StorageAccessType = {
     entropy: Uint8Array;
     serverId: Uint8Array;
   };
+  needOrphanedAttachmentCheck: boolean;
 
   // Deprecated
   'challenge:retry-message-ids': never;

ts/types/Stories.ts

@@ -16,7 +16,7 @@ export type ReplyType = {
   author: Pick<
     ConversationType,
     | 'acceptedMessageRequest'
-    | 'avatarPath'
+    | 'avatarUrl'
     | 'badges'
     | 'color'
     | 'id'
@@ -49,7 +49,7 @@ export type ConversationStoryType = {
   group?: Pick<
     ConversationType,
     | 'acceptedMessageRequest'
-    | 'avatarPath'
+    | 'avatarUrl'
     | 'color'
     | 'id'
     | 'name'
@@ -83,7 +83,7 @@ export type StoryViewType = {
   sender: Pick<
     ConversationType,
     | 'acceptedMessageRequest'
-    | 'avatarPath'
+    | 'avatarUrl'
     | 'badges'
     | 'color'
     | 'firstName'