Reuse recent CDN locators

This commit is contained in:
trevor-signal
2026-02-24 12:58:17 -05:00
committed by GitHub
parent d38277e2ce
commit 239b57576f
8 changed files with 398 additions and 120 deletions

View File

@@ -4,7 +4,6 @@
import lodash from 'lodash';
import PQueue from 'p-queue';
import { ContentHint } from '@signalapp/libsignal-client';
import Long from 'long';
import * as Errors from '../../types/errors.std.js';
import { strictAssert } from '../../util/assert.std.js';
@@ -38,7 +37,6 @@ import type {
OutgoingStickerType,
} from '../../textsecure/SendMessage.preload.js';
import type {
AttachmentDownloadableFromTransitTier,
AttachmentType,
UploadedAttachmentType,
} from '../../types/Attachment.std.js';
@@ -82,10 +80,6 @@ import {
import { getMessageIdForLogging } from '../../util/idForLogging.preload.js';
import { send, sendSyncMessageOnly } from '../../messages/send.preload.js';
import type { SignalService } from '../../protobuf/index.std.js';
import { uuidToBytes } from '../../util/uuidToBytes.std.js';
import { fromBase64 } from '../../Bytes.std.js';
import { MIMETypeToString } from '../../types/MIME.std.js';
import { canReuseExistingTransitCdnPointerForEditedMessage } from '../../util/Attachment.std.js';
import { eraseMessageContents } from '../../util/cleanup.preload.js';
const { isNumber } = lodash;
@@ -245,7 +239,6 @@ export async function sendNormalMessage(
log,
message,
targetTimestamp,
isEditedMessageSend: editedMessageTimestamp != null,
});
if (reaction) {
@@ -621,12 +614,10 @@ async function getMessageSendData({
log,
message,
targetTimestamp,
isEditedMessageSend,
}: Readonly<{
log: LoggerType;
message: MessageModel;
targetTimestamp: number;
isEditedMessageSend: boolean;
}>): Promise<{
attachments: Array<SignalService.IAttachmentPointer>;
body: undefined | string;
@@ -694,13 +685,6 @@ async function getMessageSendData({
] = await Promise.all([
uploadQueue.addAll(
preUploadAttachments.map(attachment => async () => {
if (isEditedMessageSend) {
if (canReuseExistingTransitCdnPointerForEditedMessage(attachment)) {
return convertAttachmentToPointer(attachment);
}
log.error('Unable to reuse attachment pointer for edited message');
}
return uploadSingleAttachment({
attachment,
log,
@@ -1306,47 +1290,3 @@ function didSendToEveryone({
}
);
}
function convertAttachmentToPointer(
attachment: AttachmentDownloadableFromTransitTier
): SignalService.IAttachmentPointer {
const {
cdnKey,
cdnNumber,
clientUuid,
key,
size,
digest,
incrementalMac,
chunkSize,
uploadTimestamp,
contentType,
fileName,
flags,
width,
height,
caption,
blurHash,
} = attachment;
return {
cdnKey,
cdnNumber,
clientUuid: clientUuid ? uuidToBytes(clientUuid) : undefined,
key: fromBase64(key),
size,
digest: fromBase64(digest),
incrementalMac: incrementalMac ? fromBase64(incrementalMac) : undefined,
chunkSize,
uploadTimestamp: uploadTimestamp
? Long.fromNumber(uploadTimestamp)
: undefined,
contentType: MIMETypeToString(contentType),
fileName,
flags,
width,
height,
caption,
blurHash,
};
}

View File

@@ -804,6 +804,16 @@ strictAssert(
'attachment_columns must match DB fields type'
);
export type ExistingAttachmentUploadData = {
cdnKey: string;
cdnNumber: number;
digest: string;
key: string;
uploadTimestamp: number;
incrementalMac: string | null;
chunkSize: number | null;
};
export type ExistingAttachmentData = Pick<
MessageAttachmentDBType,
| 'version'
@@ -1085,6 +1095,10 @@ type ReadableInterface = {
version: number
) => Array<MessageAttributesType>;
getMostRecentAttachmentUploadData: (
plaintextHash: string
) => ExistingAttachmentUploadData | undefined;
__dangerouslyRunAbitraryReadOnlySqlQuery: (
readOnlySqlQuery: string
) => ReadonlyArray<RowType<object>>;

View File

@@ -198,6 +198,7 @@ import type {
GetMessagesBetweenOptions,
MaybeStaleCallHistory,
ExistingAttachmentData,
ExistingAttachmentUploadData,
} from './Interface.std.js';
import {
AttachmentDownloadSource,
@@ -559,6 +560,7 @@ export const DataReader: ServerReadableInterface = {
getStatisticsForLogging,
getMostRecentAttachmentUploadData,
getBackupCdnObjectMetadata,
getBackupAttachmentDownloadProgress,
getAttachmentReferencesForMessages,
@@ -3043,6 +3045,35 @@ function isAttachmentSafeToDelete(db: ReadableDB, path: string): boolean {
return db.prepare(query, { pluck: true }).get(params) === 0;
}
function getMostRecentAttachmentUploadData(
db: ReadableDB,
plaintextHash: string
): ExistingAttachmentUploadData | undefined {
const [query, params] = sql`
SELECT
key,
digest,
transitCdnKey AS cdnKey,
transitCdnNumber AS cdnNumber,
transitCdnUploadTimestamp AS uploadTimestamp,
incrementalMac,
incrementalMacChunkSize as chunkSize
FROM message_attachments
INDEXED BY message_attachments_plaintextHash
WHERE
plaintextHash = ${plaintextHash} AND
key IS NOT NULL AND
digest IS NOT NULL AND
transitCdnKey IS NOT NULL AND
transitCdnNumber IS NOT NULL AND
transitCdnUploadTimestamp IS NOT NULL
ORDER BY transitCdnUploadTimestamp DESC
LIMIT 1
`;
return db.prepare(query).get<ExistingAttachmentUploadData>(params);
}
function _testOnlyRemoveMessageAttachments(
db: WritableDB,
timestamp: number

View File

@@ -0,0 +1,144 @@
// Copyright 2026 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { assert } from 'chai';
import { v4 as generateUuid } from 'uuid';
import { DataReader, DataWriter } from '../../sql/Client.preload.js';
import { generateAci } from '../../types/ServiceId.std.js';
import type { MessageAttributesType } from '../../model-types.d.ts';
import { postSaveUpdates } from '../../util/cleanup.preload.js';
import { IMAGE_JPEG } from '../../types/MIME.std.js';
import { testPlaintextHash } from '../../test-helpers/attachments.node.js';
const { getMostRecentAttachmentUploadData } = DataReader;
const { removeAll, saveMessages } = DataWriter;
describe('sql/getMostRecentAttachmentUploadData', () => {
const ourAci = generateAci();
const now = Date.now();
beforeEach(async () => {
await removeAll();
});
function composeMessage(
attachments: MessageAttributesType['attachments']
): MessageAttributesType {
return {
id: generateUuid(),
type: 'incoming',
conversationId: generateUuid(),
sent_at: now,
received_at: now,
timestamp: now,
attachments,
};
}
it('returns undefined when no attachment matches the plaintextHash', async () => {
const result = await getMostRecentAttachmentUploadData('nonexistent-hash');
assert.isUndefined(result);
});
it('returns CDN upload data for a matching attachment', async () => {
const plaintextHash = testPlaintextHash();
await saveMessages(
[
composeMessage([
{
size: 128,
contentType: IMAGE_JPEG,
plaintextHash,
key: 'test-key',
digest: 'test-digest',
cdnKey: 'cdn-key-1',
cdnNumber: 3,
uploadTimestamp: now,
incrementalMac: 'test-incremental-mac',
chunkSize: 256,
},
]),
],
{ forceSave: true, ourAci, postSaveUpdates }
);
const result = await getMostRecentAttachmentUploadData(plaintextHash);
assert.deepStrictEqual(result, {
key: 'test-key',
digest: 'test-digest',
cdnKey: 'cdn-key-1',
cdnNumber: 3,
uploadTimestamp: now,
incrementalMac: 'test-incremental-mac',
chunkSize: 256,
});
});
it('returns undefined when a matching attachment lacks CDN fields', async () => {
const plaintextHash = testPlaintextHash();
await saveMessages(
[
composeMessage([
{
size: 128,
contentType: IMAGE_JPEG,
plaintextHash,
key: 'test-key',
digest: 'test-digest',
// No cdnKey, cdnNumber, or uploadTimestamp
},
]),
],
{ forceSave: true, ourAci, postSaveUpdates }
);
const result = await getMostRecentAttachmentUploadData(plaintextHash);
assert.isUndefined(result);
});
it('returns the most recently uploaded entry when multiple attachments share a plaintextHash', async () => {
const plaintextHash = testPlaintextHash();
await saveMessages(
[
composeMessage([
{
size: 128,
contentType: IMAGE_JPEG,
plaintextHash,
key: 'older-key',
digest: 'older-digest',
cdnKey: 'older-cdn-key',
cdnNumber: 3,
uploadTimestamp: now - 1000,
},
]),
composeMessage([
{
size: 128,
contentType: IMAGE_JPEG,
plaintextHash,
key: 'newer-key',
digest: 'newer-digest',
cdnKey: 'newer-cdn-key',
cdnNumber: 3,
uploadTimestamp: now,
},
]),
],
{ forceSave: true, ourAci, postSaveUpdates }
);
const result = await getMostRecentAttachmentUploadData(plaintextHash);
assert.deepStrictEqual(result, {
key: 'newer-key',
digest: 'newer-digest',
cdnKey: 'newer-cdn-key',
cdnNumber: 3,
uploadTimestamp: now,
incrementalMac: null,
chunkSize: null,
});
});
});

View File

@@ -7,6 +7,8 @@ import { expect } from 'playwright/test';
import { type PrimaryDevice, StorageState } from '@signalapp/mock-server';
import { join } from 'node:path';
import { access, readFile } from 'node:fs/promises';
import Long from 'long';
import { v4 } from 'uuid';
import type { App } from '../playwright.node.js';
import { Bootstrap } from '../bootstrap.node.js';
@@ -18,9 +20,10 @@ import {
} from '../helpers.node.js';
import * as durations from '../../util/durations/index.std.js';
import { strictAssert } from '../../util/assert.std.js';
import { VIDEO_MP4 } from '../../types/MIME.std.js';
import { IMAGE_PNG, VIDEO_MP4 } from '../../types/MIME.std.js';
import { toBase64 } from '../../Bytes.std.js';
import type { AttachmentType } from '../../types/Attachment.std.js';
import type { SignalService } from '../../protobuf/index.std.js';
export const debug = createDebug('mock:test:attachments');
@@ -397,4 +400,89 @@ describe('attachments', function (this: Mocha.Suite) {
secondAttachment.thumbnail?.path
);
});
it('reuses recent CDN upload info', async () => {
const { desktop } = bootstrap;
const page = await app.getWindow();
await page.getByTestId(pinned.device.aci).click();
const plaintextVideo = await readFile(VIDEO_PATH);
const recentPointer: SignalService.IAttachmentPointer = {
...(await bootstrap.encryptAndStoreAttachmentOnCDN(
plaintextVideo,
VIDEO_MP4
)),
clientUuid: Buffer.from(v4(), 'utf8'),
uploadTimestamp: Long.fromNumber(Date.now() - 2 * durations.DAY),
fileName: 'incoming filename',
};
const plaintextCat = await readFile(CAT_PATH);
const stalePointer: SignalService.IAttachmentPointer = {
...(await bootstrap.encryptAndStoreAttachmentOnCDN(
plaintextCat,
IMAGE_PNG
)),
uploadTimestamp: Long.fromNumber(Date.now() - 4 * durations.DAY),
};
const incomingVideoTimestamp = bootstrap.getTimestamp();
await sendTextMessage({
from: pinned,
to: desktop,
desktop,
text: 'incoming video',
attachments: [recentPointer],
timestamp: incomingVideoTimestamp,
});
const incomingCatTimestamp = bootstrap.getTimestamp();
await sendTextMessage({
from: pinned,
to: desktop,
desktop,
text: 'incoming cat',
attachments: [stalePointer],
timestamp: incomingCatTimestamp,
});
await expect(
getMessageInTimelineByTimestamp(page, incomingVideoTimestamp).locator(
'img.module-image__image'
)
).toBeVisible();
await expect(
getMessageInTimelineByTimestamp(page, incomingCatTimestamp).locator(
'img.module-image__image'
)
).toBeVisible();
const { attachments: sentVideoAttachments } =
await sendMessageWithAttachments(page, pinned, 'sending same video', [
VIDEO_PATH,
]);
const { attachments: sentCatAttachments } =
await sendMessageWithAttachments(page, pinned, 'sending same cat', [
CAT_PATH,
]);
const sentVideo = sentVideoAttachments[0];
assert.deepStrictEqual(sentVideo.cdnKey, recentPointer.cdnKey);
assert.deepStrictEqual(sentVideo.cdnNumber, recentPointer.cdnNumber);
assert.deepStrictEqual(sentVideo.key, recentPointer.key);
assert.deepStrictEqual(sentVideo.digest, recentPointer.digest);
assert.deepStrictEqual(
sentVideo.incrementalMac,
recentPointer.incrementalMac
);
assert.deepStrictEqual(sentVideo.chunkSize, recentPointer.chunkSize);
assert.notDeepEqual(sentVideo.clientUuid, recentPointer.clientUuid);
assert.notDeepEqual(sentVideo.fileName, recentPointer.fileName);
const sentCat = sentCatAttachments[0];
assert.notDeepEqual(sentCat.cdnKey, stalePointer.cdnKey);
assert.notDeepEqual(sentCat.key, stalePointer.key);
assert.notDeepEqual(sentCat.digest, stalePointer.digest);
});
});

View File

@@ -39,7 +39,11 @@ import {
APPLICATION_OCTET_STREAM,
stringToMIMEType,
} from '../types/MIME.std.js';
import { SECOND, DurationInSeconds } from '../util/durations/index.std.js';
import {
SECOND,
DurationInSeconds,
HOUR,
} from '../util/durations/index.std.js';
import type { AnyPaymentEvent } from '../types/Payment.std.js';
import { PaymentEventKind } from '../types/Payment.std.js';
import { filterAndClean } from '../util/BodyRange.node.js';
@@ -47,9 +51,12 @@ import { bytesToUuid } from '../util/uuidToBytes.std.js';
import { createName } from '../util/attachmentPath.node.js';
import { partitionBodyAndNormalAttachments } from '../util/Attachment.std.js';
import { isNotNil } from '../util/isNotNil.std.js';
import { createLogger } from '../logging/log.std.js';
const { isNumber } = lodash;
const log = createLogger('processDataMessage');
const FLAGS = Proto.DataMessage.Flags;
export const ATTACHMENT_MAX = 32;
@@ -85,7 +92,6 @@ export function processAttachment(
height,
caption,
blurHash,
uploadTimestamp,
} = attachmentWithoutNulls;
const hasCdnId = Long.isLong(cdnId) ? !cdnId.isZero() : Boolean(cdnId);
@@ -94,6 +100,15 @@ export function processAttachment(
throw new Error('Missing size on incoming attachment!');
}
let uploadTimestamp = attachmentWithoutNulls.uploadTimestamp?.toNumber();
// Make sure uploadTimestamp is not set to an obviously wrong future value (we use
// uploadTimestamp to determine whether to re-use CDN pointers)
if (uploadTimestamp && uploadTimestamp > Date.now() + 12 * HOUR) {
log.warn('uploadTimestamp is in the future, dropping');
uploadTimestamp = undefined;
}
return {
cdnKey,
cdnNumber,
@@ -104,7 +119,7 @@ export function processAttachment(
height,
caption,
blurHash,
uploadTimestamp: uploadTimestamp?.toNumber(),
uploadTimestamp,
cdnId: hasCdnId ? String(cdnId) : undefined,
clientUuid: Bytes.isNotEmpty(clientUuid)
? bytesToUuid(clientUuid)

View File

@@ -24,8 +24,6 @@ import {
} from './GoogleChrome.std.js';
import type { LocalizerType } from '../types/Util.std.js';
import { ThemeType } from '../types/Util.std.js';
import { isMoreRecentThan } from './timestamp.std.js';
import { DAY } from './durations/index.std.js';
import {
isValidAttachmentKey,
isValidDigest,
@@ -54,8 +52,6 @@ const MIN_TIMELINE_IMAGE_HEIGHT = 50;
const MAX_DISPLAYABLE_IMAGE_WIDTH = 8192;
const MAX_DISPLAYABLE_IMAGE_HEIGHT = 8192;
const MAX_DURATION_TO_REUSE_ATTACHMENT_CDN_POINTER = 3 * DAY;
// // Incoming message attachment fields
// {
// id: string
@@ -936,23 +932,6 @@ export function partitionBodyAndNormalAttachments<
};
}
export function canReuseExistingTransitCdnPointerForEditedMessage(
attachment: AttachmentType
): attachment is AttachmentDownloadableFromTransitTier {
// In practice, this should always return true, since the timeframe for editing a
// message is less than MAX_DURATION_TO_REUSE_ATTACHMENT_CDN_POINTER
return (
isValidDigest(attachment.digest) &&
isValidAttachmentKey(attachment.key) &&
attachment.cdnKey != null &&
attachment.cdnNumber != null &&
isMoreRecentThan(
attachment.uploadTimestamp ?? 0,
MAX_DURATION_TO_REUSE_ATTACHMENT_CDN_POINTER
)
);
}
const MESSAGE_ATTACHMENT_TYPES_NEEDING_THUMBNAILS: Set<MessageAttachmentType> =
new Set(['attachment', 'sticker']);

View File

@@ -3,6 +3,7 @@
import Long from 'long';
import { createReadStream } from 'node:fs';
import type {
AttachmentType,
AttachmentWithHydratedData,
UploadedAttachmentType,
} from '../types/Attachment.std.js';
@@ -27,12 +28,20 @@ import {
} from '../AttachmentCrypto.node.js';
import { missingCaseError } from './missingCaseError.std.js';
import { uuidToBytes } from './uuidToBytes.std.js';
import { DAY } from './durations/index.std.js';
import { DAY, HOUR } from './durations/index.std.js';
import { isImageAttachment, isVideoAttachment } from './Attachment.std.js';
import { getAbsoluteAttachmentPath } from './migrations.preload.js';
import { isMoreRecentThan } from './timestamp.std.js';
import { DataReader } from '../sql/Client.preload.js';
import {
isValidAttachmentKey,
isValidDigest,
isValidPlaintextHash,
} from '../types/Crypto.std.js';
import type { ExistingAttachmentUploadData } from '../sql/Interface.std.js';
const CDNS_SUPPORTING_TUS = new Set([3]);
const MAX_DURATION_TO_REUSE_ATTACHMENT_CDN_POINTER = 3 * DAY;
const log = createLogger('uploadAttachment');
@@ -42,45 +51,40 @@ export async function uploadAttachment(
let keys: Uint8Array;
let cdnKey: string;
let cdnNumber: number;
let encrypted: Pick<
EncryptedAttachmentV2,
'digest' | 'plaintextHash' | 'incrementalMac' | 'chunkSize'
>;
let digest: Uint8Array;
let plaintextHash: string;
let incrementalMac: Uint8Array | undefined;
let chunkSize: number | undefined;
let uploadTimestamp: number;
// Recently uploaded attachment
if (
attachment.cdnKey &&
attachment.cdnNumber &&
attachment.key &&
attachment.size != null &&
attachment.digest &&
attachment.contentType != null &&
attachment.plaintextHash != null &&
attachment.uploadTimestamp != null &&
isMoreRecentThan(attachment.uploadTimestamp, 3 * DAY)
) {
log.info('reusing attachment uploaded at', attachment.uploadTimestamp);
const needIncrementalMac = supportsIncrementalMac(attachment.contentType);
const dataForReuse = await getAttachmentUploadDataForReuse(attachment);
({ cdnKey, cdnNumber, uploadTimestamp } = attachment);
if (dataForReuse && isValidPlaintextHash(attachment.plaintextHash)) {
log.info('reusing attachment uploaded at', dataForReuse.uploadTimestamp);
keys = Bytes.fromBase64(attachment.key);
({ cdnKey, cdnNumber, uploadTimestamp } = dataForReuse);
keys = Bytes.fromBase64(dataForReuse.key);
encrypted = {
digest: Bytes.fromBase64(attachment.digest),
plaintextHash: attachment.plaintextHash,
incrementalMac: attachment.incrementalMac
? Bytes.fromBase64(attachment.incrementalMac)
: undefined,
chunkSize: attachment.chunkSize,
};
digest = Bytes.fromBase64(dataForReuse.digest);
plaintextHash = attachment.plaintextHash;
incrementalMac =
needIncrementalMac && dataForReuse.incrementalMac
? Bytes.fromBase64(dataForReuse.incrementalMac)
: undefined;
chunkSize =
needIncrementalMac && dataForReuse.chunkSize
? dataForReuse.chunkSize
: undefined;
} else {
keys = getRandomBytes(64);
uploadTimestamp = Date.now();
const needIncrementalMac = supportsIncrementalMac(attachment.contentType);
({ cdnKey, cdnNumber, encrypted } = await encryptAndUploadAttachment({
({
cdnKey,
cdnNumber,
encrypted: { digest, plaintextHash, incrementalMac, chunkSize },
} = await encryptAndUploadAttachment({
keys,
needIncrementalMac,
plaintext: { data: attachment.data },
@@ -101,10 +105,10 @@ export async function uploadAttachment(
clientUuid: clientUuid ? uuidToBytes(clientUuid) : undefined,
key: keys,
size: attachment.data.byteLength,
digest: encrypted.digest,
plaintextHash: encrypted.plaintextHash,
incrementalMac: encrypted.incrementalMac,
chunkSize: encrypted.chunkSize,
digest,
plaintextHash,
incrementalMac,
chunkSize,
uploadTimestamp: Long.fromNumber(uploadTimestamp),
contentType: MIMETypeToString(attachment.contentType),
@@ -199,3 +203,66 @@ export async function uploadFile({
);
}
}
async function getAttachmentUploadDataForReuse(
attachment: AttachmentType
): Promise<ExistingAttachmentUploadData | null> {
if (!isValidPlaintextHash(attachment.plaintextHash)) {
return null;
}
if (isValidCdnDataForReuse(attachment)) {
return {
cdnKey: attachment.cdnKey,
cdnNumber: attachment.cdnNumber,
key: attachment.key,
digest: attachment.digest,
uploadTimestamp: attachment.uploadTimestamp,
incrementalMac: attachment.incrementalMac ?? null,
chunkSize: attachment.chunkSize ?? null,
};
}
const recentAttachmentUploadData =
await DataReader.getMostRecentAttachmentUploadData(
attachment.plaintextHash
);
if (
recentAttachmentUploadData &&
isValidCdnDataForReuse(recentAttachmentUploadData)
) {
return recentAttachmentUploadData;
}
return null;
}
function isValidCdnDataForReuse(attachment: {
cdnKey?: string;
cdnNumber?: number;
digest?: string;
key?: string;
uploadTimestamp?: number;
}): attachment is {
cdnKey: string;
cdnNumber: number;
digest: string;
key: string;
uploadTimestamp: number;
} {
return Boolean(
isValidDigest(attachment.digest) &&
isValidAttachmentKey(attachment.key) &&
attachment.cdnKey != null &&
attachment.cdnNumber != null &&
attachment.uploadTimestamp &&
isMoreRecentThan(
attachment.uploadTimestamp,
MAX_DURATION_TO_REUSE_ATTACHMENT_CDN_POINTER
) &&
// for extra safety, check to make sure we don't have an uploadTimestamp that's
// incorrectly in the future
attachment.uploadTimestamp < Date.now() + 12 * HOUR
);
}