mirror of
https://github.com/signalapp/Signal-Desktop.git
synced 2026-04-18 23:49:20 +01:00
Reuse recent CDN locators
This commit is contained in:
@@ -4,7 +4,6 @@
|
||||
import lodash from 'lodash';
|
||||
import PQueue from 'p-queue';
|
||||
import { ContentHint } from '@signalapp/libsignal-client';
|
||||
import Long from 'long';
|
||||
|
||||
import * as Errors from '../../types/errors.std.js';
|
||||
import { strictAssert } from '../../util/assert.std.js';
|
||||
@@ -38,7 +37,6 @@ import type {
|
||||
OutgoingStickerType,
|
||||
} from '../../textsecure/SendMessage.preload.js';
|
||||
import type {
|
||||
AttachmentDownloadableFromTransitTier,
|
||||
AttachmentType,
|
||||
UploadedAttachmentType,
|
||||
} from '../../types/Attachment.std.js';
|
||||
@@ -82,10 +80,6 @@ import {
|
||||
import { getMessageIdForLogging } from '../../util/idForLogging.preload.js';
|
||||
import { send, sendSyncMessageOnly } from '../../messages/send.preload.js';
|
||||
import type { SignalService } from '../../protobuf/index.std.js';
|
||||
import { uuidToBytes } from '../../util/uuidToBytes.std.js';
|
||||
import { fromBase64 } from '../../Bytes.std.js';
|
||||
import { MIMETypeToString } from '../../types/MIME.std.js';
|
||||
import { canReuseExistingTransitCdnPointerForEditedMessage } from '../../util/Attachment.std.js';
|
||||
import { eraseMessageContents } from '../../util/cleanup.preload.js';
|
||||
|
||||
const { isNumber } = lodash;
|
||||
@@ -245,7 +239,6 @@ export async function sendNormalMessage(
|
||||
log,
|
||||
message,
|
||||
targetTimestamp,
|
||||
isEditedMessageSend: editedMessageTimestamp != null,
|
||||
});
|
||||
|
||||
if (reaction) {
|
||||
@@ -621,12 +614,10 @@ async function getMessageSendData({
|
||||
log,
|
||||
message,
|
||||
targetTimestamp,
|
||||
isEditedMessageSend,
|
||||
}: Readonly<{
|
||||
log: LoggerType;
|
||||
message: MessageModel;
|
||||
targetTimestamp: number;
|
||||
isEditedMessageSend: boolean;
|
||||
}>): Promise<{
|
||||
attachments: Array<SignalService.IAttachmentPointer>;
|
||||
body: undefined | string;
|
||||
@@ -694,13 +685,6 @@ async function getMessageSendData({
|
||||
] = await Promise.all([
|
||||
uploadQueue.addAll(
|
||||
preUploadAttachments.map(attachment => async () => {
|
||||
if (isEditedMessageSend) {
|
||||
if (canReuseExistingTransitCdnPointerForEditedMessage(attachment)) {
|
||||
return convertAttachmentToPointer(attachment);
|
||||
}
|
||||
log.error('Unable to reuse attachment pointer for edited message');
|
||||
}
|
||||
|
||||
return uploadSingleAttachment({
|
||||
attachment,
|
||||
log,
|
||||
@@ -1306,47 +1290,3 @@ function didSendToEveryone({
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function convertAttachmentToPointer(
|
||||
attachment: AttachmentDownloadableFromTransitTier
|
||||
): SignalService.IAttachmentPointer {
|
||||
const {
|
||||
cdnKey,
|
||||
cdnNumber,
|
||||
clientUuid,
|
||||
key,
|
||||
size,
|
||||
digest,
|
||||
incrementalMac,
|
||||
chunkSize,
|
||||
uploadTimestamp,
|
||||
contentType,
|
||||
fileName,
|
||||
flags,
|
||||
width,
|
||||
height,
|
||||
caption,
|
||||
blurHash,
|
||||
} = attachment;
|
||||
|
||||
return {
|
||||
cdnKey,
|
||||
cdnNumber,
|
||||
clientUuid: clientUuid ? uuidToBytes(clientUuid) : undefined,
|
||||
key: fromBase64(key),
|
||||
size,
|
||||
digest: fromBase64(digest),
|
||||
incrementalMac: incrementalMac ? fromBase64(incrementalMac) : undefined,
|
||||
chunkSize,
|
||||
uploadTimestamp: uploadTimestamp
|
||||
? Long.fromNumber(uploadTimestamp)
|
||||
: undefined,
|
||||
contentType: MIMETypeToString(contentType),
|
||||
fileName,
|
||||
flags,
|
||||
width,
|
||||
height,
|
||||
caption,
|
||||
blurHash,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -804,6 +804,16 @@ strictAssert(
|
||||
'attachment_columns must match DB fields type'
|
||||
);
|
||||
|
||||
export type ExistingAttachmentUploadData = {
|
||||
cdnKey: string;
|
||||
cdnNumber: number;
|
||||
digest: string;
|
||||
key: string;
|
||||
uploadTimestamp: number;
|
||||
incrementalMac: string | null;
|
||||
chunkSize: number | null;
|
||||
};
|
||||
|
||||
export type ExistingAttachmentData = Pick<
|
||||
MessageAttachmentDBType,
|
||||
| 'version'
|
||||
@@ -1085,6 +1095,10 @@ type ReadableInterface = {
|
||||
version: number
|
||||
) => Array<MessageAttributesType>;
|
||||
|
||||
getMostRecentAttachmentUploadData: (
|
||||
plaintextHash: string
|
||||
) => ExistingAttachmentUploadData | undefined;
|
||||
|
||||
__dangerouslyRunAbitraryReadOnlySqlQuery: (
|
||||
readOnlySqlQuery: string
|
||||
) => ReadonlyArray<RowType<object>>;
|
||||
|
||||
@@ -198,6 +198,7 @@ import type {
|
||||
GetMessagesBetweenOptions,
|
||||
MaybeStaleCallHistory,
|
||||
ExistingAttachmentData,
|
||||
ExistingAttachmentUploadData,
|
||||
} from './Interface.std.js';
|
||||
import {
|
||||
AttachmentDownloadSource,
|
||||
@@ -559,6 +560,7 @@ export const DataReader: ServerReadableInterface = {
|
||||
|
||||
getStatisticsForLogging,
|
||||
|
||||
getMostRecentAttachmentUploadData,
|
||||
getBackupCdnObjectMetadata,
|
||||
getBackupAttachmentDownloadProgress,
|
||||
getAttachmentReferencesForMessages,
|
||||
@@ -3043,6 +3045,35 @@ function isAttachmentSafeToDelete(db: ReadableDB, path: string): boolean {
|
||||
return db.prepare(query, { pluck: true }).get(params) === 0;
|
||||
}
|
||||
|
||||
function getMostRecentAttachmentUploadData(
|
||||
db: ReadableDB,
|
||||
plaintextHash: string
|
||||
): ExistingAttachmentUploadData | undefined {
|
||||
const [query, params] = sql`
|
||||
SELECT
|
||||
key,
|
||||
digest,
|
||||
transitCdnKey AS cdnKey,
|
||||
transitCdnNumber AS cdnNumber,
|
||||
transitCdnUploadTimestamp AS uploadTimestamp,
|
||||
incrementalMac,
|
||||
incrementalMacChunkSize as chunkSize
|
||||
FROM message_attachments
|
||||
INDEXED BY message_attachments_plaintextHash
|
||||
WHERE
|
||||
plaintextHash = ${plaintextHash} AND
|
||||
key IS NOT NULL AND
|
||||
digest IS NOT NULL AND
|
||||
transitCdnKey IS NOT NULL AND
|
||||
transitCdnNumber IS NOT NULL AND
|
||||
transitCdnUploadTimestamp IS NOT NULL
|
||||
ORDER BY transitCdnUploadTimestamp DESC
|
||||
LIMIT 1
|
||||
`;
|
||||
|
||||
return db.prepare(query).get<ExistingAttachmentUploadData>(params);
|
||||
}
|
||||
|
||||
function _testOnlyRemoveMessageAttachments(
|
||||
db: WritableDB,
|
||||
timestamp: number
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
// Copyright 2026 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import { assert } from 'chai';
|
||||
import { v4 as generateUuid } from 'uuid';
|
||||
|
||||
import { DataReader, DataWriter } from '../../sql/Client.preload.js';
|
||||
import { generateAci } from '../../types/ServiceId.std.js';
|
||||
import type { MessageAttributesType } from '../../model-types.d.ts';
|
||||
import { postSaveUpdates } from '../../util/cleanup.preload.js';
|
||||
import { IMAGE_JPEG } from '../../types/MIME.std.js';
|
||||
import { testPlaintextHash } from '../../test-helpers/attachments.node.js';
|
||||
|
||||
const { getMostRecentAttachmentUploadData } = DataReader;
|
||||
const { removeAll, saveMessages } = DataWriter;
|
||||
|
||||
describe('sql/getMostRecentAttachmentUploadData', () => {
|
||||
const ourAci = generateAci();
|
||||
const now = Date.now();
|
||||
|
||||
beforeEach(async () => {
|
||||
await removeAll();
|
||||
});
|
||||
|
||||
function composeMessage(
|
||||
attachments: MessageAttributesType['attachments']
|
||||
): MessageAttributesType {
|
||||
return {
|
||||
id: generateUuid(),
|
||||
type: 'incoming',
|
||||
conversationId: generateUuid(),
|
||||
sent_at: now,
|
||||
received_at: now,
|
||||
timestamp: now,
|
||||
attachments,
|
||||
};
|
||||
}
|
||||
|
||||
it('returns undefined when no attachment matches the plaintextHash', async () => {
|
||||
const result = await getMostRecentAttachmentUploadData('nonexistent-hash');
|
||||
assert.isUndefined(result);
|
||||
});
|
||||
|
||||
it('returns CDN upload data for a matching attachment', async () => {
|
||||
const plaintextHash = testPlaintextHash();
|
||||
await saveMessages(
|
||||
[
|
||||
composeMessage([
|
||||
{
|
||||
size: 128,
|
||||
contentType: IMAGE_JPEG,
|
||||
plaintextHash,
|
||||
key: 'test-key',
|
||||
digest: 'test-digest',
|
||||
cdnKey: 'cdn-key-1',
|
||||
cdnNumber: 3,
|
||||
uploadTimestamp: now,
|
||||
incrementalMac: 'test-incremental-mac',
|
||||
chunkSize: 256,
|
||||
},
|
||||
]),
|
||||
],
|
||||
{ forceSave: true, ourAci, postSaveUpdates }
|
||||
);
|
||||
|
||||
const result = await getMostRecentAttachmentUploadData(plaintextHash);
|
||||
assert.deepStrictEqual(result, {
|
||||
key: 'test-key',
|
||||
digest: 'test-digest',
|
||||
cdnKey: 'cdn-key-1',
|
||||
cdnNumber: 3,
|
||||
uploadTimestamp: now,
|
||||
incrementalMac: 'test-incremental-mac',
|
||||
chunkSize: 256,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns undefined when a matching attachment lacks CDN fields', async () => {
|
||||
const plaintextHash = testPlaintextHash();
|
||||
await saveMessages(
|
||||
[
|
||||
composeMessage([
|
||||
{
|
||||
size: 128,
|
||||
contentType: IMAGE_JPEG,
|
||||
plaintextHash,
|
||||
key: 'test-key',
|
||||
digest: 'test-digest',
|
||||
// No cdnKey, cdnNumber, or uploadTimestamp
|
||||
},
|
||||
]),
|
||||
],
|
||||
{ forceSave: true, ourAci, postSaveUpdates }
|
||||
);
|
||||
|
||||
const result = await getMostRecentAttachmentUploadData(plaintextHash);
|
||||
assert.isUndefined(result);
|
||||
});
|
||||
|
||||
it('returns the most recently uploaded entry when multiple attachments share a plaintextHash', async () => {
|
||||
const plaintextHash = testPlaintextHash();
|
||||
|
||||
await saveMessages(
|
||||
[
|
||||
composeMessage([
|
||||
{
|
||||
size: 128,
|
||||
contentType: IMAGE_JPEG,
|
||||
plaintextHash,
|
||||
key: 'older-key',
|
||||
digest: 'older-digest',
|
||||
cdnKey: 'older-cdn-key',
|
||||
cdnNumber: 3,
|
||||
uploadTimestamp: now - 1000,
|
||||
},
|
||||
]),
|
||||
composeMessage([
|
||||
{
|
||||
size: 128,
|
||||
contentType: IMAGE_JPEG,
|
||||
plaintextHash,
|
||||
key: 'newer-key',
|
||||
digest: 'newer-digest',
|
||||
cdnKey: 'newer-cdn-key',
|
||||
cdnNumber: 3,
|
||||
uploadTimestamp: now,
|
||||
},
|
||||
]),
|
||||
],
|
||||
{ forceSave: true, ourAci, postSaveUpdates }
|
||||
);
|
||||
|
||||
const result = await getMostRecentAttachmentUploadData(plaintextHash);
|
||||
assert.deepStrictEqual(result, {
|
||||
key: 'newer-key',
|
||||
digest: 'newer-digest',
|
||||
cdnKey: 'newer-cdn-key',
|
||||
cdnNumber: 3,
|
||||
uploadTimestamp: now,
|
||||
incrementalMac: null,
|
||||
chunkSize: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,8 @@ import { expect } from 'playwright/test';
|
||||
import { type PrimaryDevice, StorageState } from '@signalapp/mock-server';
|
||||
import { join } from 'node:path';
|
||||
import { access, readFile } from 'node:fs/promises';
|
||||
import Long from 'long';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import type { App } from '../playwright.node.js';
|
||||
import { Bootstrap } from '../bootstrap.node.js';
|
||||
@@ -18,9 +20,10 @@ import {
|
||||
} from '../helpers.node.js';
|
||||
import * as durations from '../../util/durations/index.std.js';
|
||||
import { strictAssert } from '../../util/assert.std.js';
|
||||
import { VIDEO_MP4 } from '../../types/MIME.std.js';
|
||||
import { IMAGE_PNG, VIDEO_MP4 } from '../../types/MIME.std.js';
|
||||
import { toBase64 } from '../../Bytes.std.js';
|
||||
import type { AttachmentType } from '../../types/Attachment.std.js';
|
||||
import type { SignalService } from '../../protobuf/index.std.js';
|
||||
|
||||
export const debug = createDebug('mock:test:attachments');
|
||||
|
||||
@@ -397,4 +400,89 @@ describe('attachments', function (this: Mocha.Suite) {
|
||||
secondAttachment.thumbnail?.path
|
||||
);
|
||||
});
|
||||
|
||||
it('reuses recent CDN upload info', async () => {
|
||||
const { desktop } = bootstrap;
|
||||
const page = await app.getWindow();
|
||||
|
||||
await page.getByTestId(pinned.device.aci).click();
|
||||
|
||||
const plaintextVideo = await readFile(VIDEO_PATH);
|
||||
const recentPointer: SignalService.IAttachmentPointer = {
|
||||
...(await bootstrap.encryptAndStoreAttachmentOnCDN(
|
||||
plaintextVideo,
|
||||
VIDEO_MP4
|
||||
)),
|
||||
clientUuid: Buffer.from(v4(), 'utf8'),
|
||||
uploadTimestamp: Long.fromNumber(Date.now() - 2 * durations.DAY),
|
||||
fileName: 'incoming filename',
|
||||
};
|
||||
|
||||
const plaintextCat = await readFile(CAT_PATH);
|
||||
const stalePointer: SignalService.IAttachmentPointer = {
|
||||
...(await bootstrap.encryptAndStoreAttachmentOnCDN(
|
||||
plaintextCat,
|
||||
IMAGE_PNG
|
||||
)),
|
||||
uploadTimestamp: Long.fromNumber(Date.now() - 4 * durations.DAY),
|
||||
};
|
||||
|
||||
const incomingVideoTimestamp = bootstrap.getTimestamp();
|
||||
await sendTextMessage({
|
||||
from: pinned,
|
||||
to: desktop,
|
||||
desktop,
|
||||
text: 'incoming video',
|
||||
attachments: [recentPointer],
|
||||
timestamp: incomingVideoTimestamp,
|
||||
});
|
||||
|
||||
const incomingCatTimestamp = bootstrap.getTimestamp();
|
||||
await sendTextMessage({
|
||||
from: pinned,
|
||||
to: desktop,
|
||||
desktop,
|
||||
text: 'incoming cat',
|
||||
attachments: [stalePointer],
|
||||
timestamp: incomingCatTimestamp,
|
||||
});
|
||||
|
||||
await expect(
|
||||
getMessageInTimelineByTimestamp(page, incomingVideoTimestamp).locator(
|
||||
'img.module-image__image'
|
||||
)
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
getMessageInTimelineByTimestamp(page, incomingCatTimestamp).locator(
|
||||
'img.module-image__image'
|
||||
)
|
||||
).toBeVisible();
|
||||
|
||||
const { attachments: sentVideoAttachments } =
|
||||
await sendMessageWithAttachments(page, pinned, 'sending same video', [
|
||||
VIDEO_PATH,
|
||||
]);
|
||||
const { attachments: sentCatAttachments } =
|
||||
await sendMessageWithAttachments(page, pinned, 'sending same cat', [
|
||||
CAT_PATH,
|
||||
]);
|
||||
|
||||
const sentVideo = sentVideoAttachments[0];
|
||||
assert.deepStrictEqual(sentVideo.cdnKey, recentPointer.cdnKey);
|
||||
assert.deepStrictEqual(sentVideo.cdnNumber, recentPointer.cdnNumber);
|
||||
assert.deepStrictEqual(sentVideo.key, recentPointer.key);
|
||||
assert.deepStrictEqual(sentVideo.digest, recentPointer.digest);
|
||||
assert.deepStrictEqual(
|
||||
sentVideo.incrementalMac,
|
||||
recentPointer.incrementalMac
|
||||
);
|
||||
assert.deepStrictEqual(sentVideo.chunkSize, recentPointer.chunkSize);
|
||||
assert.notDeepEqual(sentVideo.clientUuid, recentPointer.clientUuid);
|
||||
assert.notDeepEqual(sentVideo.fileName, recentPointer.fileName);
|
||||
|
||||
const sentCat = sentCatAttachments[0];
|
||||
assert.notDeepEqual(sentCat.cdnKey, stalePointer.cdnKey);
|
||||
assert.notDeepEqual(sentCat.key, stalePointer.key);
|
||||
assert.notDeepEqual(sentCat.digest, stalePointer.digest);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -39,7 +39,11 @@ import {
|
||||
APPLICATION_OCTET_STREAM,
|
||||
stringToMIMEType,
|
||||
} from '../types/MIME.std.js';
|
||||
import { SECOND, DurationInSeconds } from '../util/durations/index.std.js';
|
||||
import {
|
||||
SECOND,
|
||||
DurationInSeconds,
|
||||
HOUR,
|
||||
} from '../util/durations/index.std.js';
|
||||
import type { AnyPaymentEvent } from '../types/Payment.std.js';
|
||||
import { PaymentEventKind } from '../types/Payment.std.js';
|
||||
import { filterAndClean } from '../util/BodyRange.node.js';
|
||||
@@ -47,9 +51,12 @@ import { bytesToUuid } from '../util/uuidToBytes.std.js';
|
||||
import { createName } from '../util/attachmentPath.node.js';
|
||||
import { partitionBodyAndNormalAttachments } from '../util/Attachment.std.js';
|
||||
import { isNotNil } from '../util/isNotNil.std.js';
|
||||
import { createLogger } from '../logging/log.std.js';
|
||||
|
||||
const { isNumber } = lodash;
|
||||
|
||||
const log = createLogger('processDataMessage');
|
||||
|
||||
const FLAGS = Proto.DataMessage.Flags;
|
||||
export const ATTACHMENT_MAX = 32;
|
||||
|
||||
@@ -85,7 +92,6 @@ export function processAttachment(
|
||||
height,
|
||||
caption,
|
||||
blurHash,
|
||||
uploadTimestamp,
|
||||
} = attachmentWithoutNulls;
|
||||
|
||||
const hasCdnId = Long.isLong(cdnId) ? !cdnId.isZero() : Boolean(cdnId);
|
||||
@@ -94,6 +100,15 @@ export function processAttachment(
|
||||
throw new Error('Missing size on incoming attachment!');
|
||||
}
|
||||
|
||||
let uploadTimestamp = attachmentWithoutNulls.uploadTimestamp?.toNumber();
|
||||
|
||||
// Make sure uploadTimestamp is not set to an obviously wrong future value (we use
|
||||
// uploadTimestamp to determine whether to re-use CDN pointers)
|
||||
if (uploadTimestamp && uploadTimestamp > Date.now() + 12 * HOUR) {
|
||||
log.warn('uploadTimestamp is in the future, dropping');
|
||||
uploadTimestamp = undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
cdnKey,
|
||||
cdnNumber,
|
||||
@@ -104,7 +119,7 @@ export function processAttachment(
|
||||
height,
|
||||
caption,
|
||||
blurHash,
|
||||
uploadTimestamp: uploadTimestamp?.toNumber(),
|
||||
uploadTimestamp,
|
||||
cdnId: hasCdnId ? String(cdnId) : undefined,
|
||||
clientUuid: Bytes.isNotEmpty(clientUuid)
|
||||
? bytesToUuid(clientUuid)
|
||||
|
||||
@@ -24,8 +24,6 @@ import {
|
||||
} from './GoogleChrome.std.js';
|
||||
import type { LocalizerType } from '../types/Util.std.js';
|
||||
import { ThemeType } from '../types/Util.std.js';
|
||||
import { isMoreRecentThan } from './timestamp.std.js';
|
||||
import { DAY } from './durations/index.std.js';
|
||||
import {
|
||||
isValidAttachmentKey,
|
||||
isValidDigest,
|
||||
@@ -54,8 +52,6 @@ const MIN_TIMELINE_IMAGE_HEIGHT = 50;
|
||||
const MAX_DISPLAYABLE_IMAGE_WIDTH = 8192;
|
||||
const MAX_DISPLAYABLE_IMAGE_HEIGHT = 8192;
|
||||
|
||||
const MAX_DURATION_TO_REUSE_ATTACHMENT_CDN_POINTER = 3 * DAY;
|
||||
|
||||
// // Incoming message attachment fields
|
||||
// {
|
||||
// id: string
|
||||
@@ -936,23 +932,6 @@ export function partitionBodyAndNormalAttachments<
|
||||
};
|
||||
}
|
||||
|
||||
export function canReuseExistingTransitCdnPointerForEditedMessage(
|
||||
attachment: AttachmentType
|
||||
): attachment is AttachmentDownloadableFromTransitTier {
|
||||
// In practice, this should always return true, since the timeframe for editing a
|
||||
// message is less than MAX_DURATION_TO_REUSE_ATTACHMENT_CDN_POINTER
|
||||
return (
|
||||
isValidDigest(attachment.digest) &&
|
||||
isValidAttachmentKey(attachment.key) &&
|
||||
attachment.cdnKey != null &&
|
||||
attachment.cdnNumber != null &&
|
||||
isMoreRecentThan(
|
||||
attachment.uploadTimestamp ?? 0,
|
||||
MAX_DURATION_TO_REUSE_ATTACHMENT_CDN_POINTER
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const MESSAGE_ATTACHMENT_TYPES_NEEDING_THUMBNAILS: Set<MessageAttachmentType> =
|
||||
new Set(['attachment', 'sticker']);
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import Long from 'long';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import type {
|
||||
AttachmentType,
|
||||
AttachmentWithHydratedData,
|
||||
UploadedAttachmentType,
|
||||
} from '../types/Attachment.std.js';
|
||||
@@ -27,12 +28,20 @@ import {
|
||||
} from '../AttachmentCrypto.node.js';
|
||||
import { missingCaseError } from './missingCaseError.std.js';
|
||||
import { uuidToBytes } from './uuidToBytes.std.js';
|
||||
import { DAY } from './durations/index.std.js';
|
||||
import { DAY, HOUR } from './durations/index.std.js';
|
||||
import { isImageAttachment, isVideoAttachment } from './Attachment.std.js';
|
||||
import { getAbsoluteAttachmentPath } from './migrations.preload.js';
|
||||
import { isMoreRecentThan } from './timestamp.std.js';
|
||||
import { DataReader } from '../sql/Client.preload.js';
|
||||
import {
|
||||
isValidAttachmentKey,
|
||||
isValidDigest,
|
||||
isValidPlaintextHash,
|
||||
} from '../types/Crypto.std.js';
|
||||
import type { ExistingAttachmentUploadData } from '../sql/Interface.std.js';
|
||||
|
||||
const CDNS_SUPPORTING_TUS = new Set([3]);
|
||||
const MAX_DURATION_TO_REUSE_ATTACHMENT_CDN_POINTER = 3 * DAY;
|
||||
|
||||
const log = createLogger('uploadAttachment');
|
||||
|
||||
@@ -42,45 +51,40 @@ export async function uploadAttachment(
|
||||
let keys: Uint8Array;
|
||||
let cdnKey: string;
|
||||
let cdnNumber: number;
|
||||
let encrypted: Pick<
|
||||
EncryptedAttachmentV2,
|
||||
'digest' | 'plaintextHash' | 'incrementalMac' | 'chunkSize'
|
||||
>;
|
||||
let digest: Uint8Array;
|
||||
let plaintextHash: string;
|
||||
let incrementalMac: Uint8Array | undefined;
|
||||
let chunkSize: number | undefined;
|
||||
let uploadTimestamp: number;
|
||||
|
||||
// Recently uploaded attachment
|
||||
if (
|
||||
attachment.cdnKey &&
|
||||
attachment.cdnNumber &&
|
||||
attachment.key &&
|
||||
attachment.size != null &&
|
||||
attachment.digest &&
|
||||
attachment.contentType != null &&
|
||||
attachment.plaintextHash != null &&
|
||||
attachment.uploadTimestamp != null &&
|
||||
isMoreRecentThan(attachment.uploadTimestamp, 3 * DAY)
|
||||
) {
|
||||
log.info('reusing attachment uploaded at', attachment.uploadTimestamp);
|
||||
const needIncrementalMac = supportsIncrementalMac(attachment.contentType);
|
||||
const dataForReuse = await getAttachmentUploadDataForReuse(attachment);
|
||||
|
||||
({ cdnKey, cdnNumber, uploadTimestamp } = attachment);
|
||||
if (dataForReuse && isValidPlaintextHash(attachment.plaintextHash)) {
|
||||
log.info('reusing attachment uploaded at', dataForReuse.uploadTimestamp);
|
||||
|
||||
keys = Bytes.fromBase64(attachment.key);
|
||||
({ cdnKey, cdnNumber, uploadTimestamp } = dataForReuse);
|
||||
keys = Bytes.fromBase64(dataForReuse.key);
|
||||
|
||||
encrypted = {
|
||||
digest: Bytes.fromBase64(attachment.digest),
|
||||
plaintextHash: attachment.plaintextHash,
|
||||
incrementalMac: attachment.incrementalMac
|
||||
? Bytes.fromBase64(attachment.incrementalMac)
|
||||
: undefined,
|
||||
chunkSize: attachment.chunkSize,
|
||||
};
|
||||
digest = Bytes.fromBase64(dataForReuse.digest);
|
||||
plaintextHash = attachment.plaintextHash;
|
||||
incrementalMac =
|
||||
needIncrementalMac && dataForReuse.incrementalMac
|
||||
? Bytes.fromBase64(dataForReuse.incrementalMac)
|
||||
: undefined;
|
||||
chunkSize =
|
||||
needIncrementalMac && dataForReuse.chunkSize
|
||||
? dataForReuse.chunkSize
|
||||
: undefined;
|
||||
} else {
|
||||
keys = getRandomBytes(64);
|
||||
uploadTimestamp = Date.now();
|
||||
|
||||
const needIncrementalMac = supportsIncrementalMac(attachment.contentType);
|
||||
|
||||
({ cdnKey, cdnNumber, encrypted } = await encryptAndUploadAttachment({
|
||||
({
|
||||
cdnKey,
|
||||
cdnNumber,
|
||||
encrypted: { digest, plaintextHash, incrementalMac, chunkSize },
|
||||
} = await encryptAndUploadAttachment({
|
||||
keys,
|
||||
needIncrementalMac,
|
||||
plaintext: { data: attachment.data },
|
||||
@@ -101,10 +105,10 @@ export async function uploadAttachment(
|
||||
clientUuid: clientUuid ? uuidToBytes(clientUuid) : undefined,
|
||||
key: keys,
|
||||
size: attachment.data.byteLength,
|
||||
digest: encrypted.digest,
|
||||
plaintextHash: encrypted.plaintextHash,
|
||||
incrementalMac: encrypted.incrementalMac,
|
||||
chunkSize: encrypted.chunkSize,
|
||||
digest,
|
||||
plaintextHash,
|
||||
incrementalMac,
|
||||
chunkSize,
|
||||
uploadTimestamp: Long.fromNumber(uploadTimestamp),
|
||||
|
||||
contentType: MIMETypeToString(attachment.contentType),
|
||||
@@ -199,3 +203,66 @@ export async function uploadFile({
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function getAttachmentUploadDataForReuse(
|
||||
attachment: AttachmentType
|
||||
): Promise<ExistingAttachmentUploadData | null> {
|
||||
if (!isValidPlaintextHash(attachment.plaintextHash)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (isValidCdnDataForReuse(attachment)) {
|
||||
return {
|
||||
cdnKey: attachment.cdnKey,
|
||||
cdnNumber: attachment.cdnNumber,
|
||||
key: attachment.key,
|
||||
digest: attachment.digest,
|
||||
uploadTimestamp: attachment.uploadTimestamp,
|
||||
incrementalMac: attachment.incrementalMac ?? null,
|
||||
chunkSize: attachment.chunkSize ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
const recentAttachmentUploadData =
|
||||
await DataReader.getMostRecentAttachmentUploadData(
|
||||
attachment.plaintextHash
|
||||
);
|
||||
|
||||
if (
|
||||
recentAttachmentUploadData &&
|
||||
isValidCdnDataForReuse(recentAttachmentUploadData)
|
||||
) {
|
||||
return recentAttachmentUploadData;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function isValidCdnDataForReuse(attachment: {
|
||||
cdnKey?: string;
|
||||
cdnNumber?: number;
|
||||
digest?: string;
|
||||
key?: string;
|
||||
uploadTimestamp?: number;
|
||||
}): attachment is {
|
||||
cdnKey: string;
|
||||
cdnNumber: number;
|
||||
digest: string;
|
||||
key: string;
|
||||
uploadTimestamp: number;
|
||||
} {
|
||||
return Boolean(
|
||||
isValidDigest(attachment.digest) &&
|
||||
isValidAttachmentKey(attachment.key) &&
|
||||
attachment.cdnKey != null &&
|
||||
attachment.cdnNumber != null &&
|
||||
attachment.uploadTimestamp &&
|
||||
isMoreRecentThan(
|
||||
attachment.uploadTimestamp,
|
||||
MAX_DURATION_TO_REUSE_ATTACHMENT_CDN_POINTER
|
||||
) &&
|
||||
// for extra safety, check to make sure we don't have an uploadTimestamp that's
|
||||
// incorrectly in the future
|
||||
attachment.uploadTimestamp < Date.now() + 12 * HOUR
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user