diff --git a/ts/jobs/AttachmentDownloadManager.preload.ts b/ts/jobs/AttachmentDownloadManager.preload.ts index 6bd7e5f6aa..48aa28ecf4 100644 --- a/ts/jobs/AttachmentDownloadManager.preload.ts +++ b/ts/jobs/AttachmentDownloadManager.preload.ts @@ -38,7 +38,7 @@ import { shouldAttachmentEndUpInRemoteBackup, getUndownloadedAttachmentSignature, isIncremental, - hasRequiredInformationForBackup, + hasRequiredInformationForRemoteBackup, } from '../util/Attachment.std.js'; import type { ReadonlyMessageAttributesType } from '../model-types.d.ts'; import { backupsService } from '../services/backups/index.preload.js'; @@ -310,7 +310,7 @@ export class AttachmentDownloadManager extends JobManager; - backupLevel: BackupLevel; - isLocalBackup: boolean; }>; type NonBubbleOptionsType = Pick< @@ -287,18 +289,11 @@ export class BackupExportStream extends Readable { // array. #customColorIdByUuid = new Map(); - constructor(private readonly backupType: BackupType) { + constructor(private readonly options: Readonly) { super(); } - public run( - backupLevel: BackupLevel, - localBackupSnapshotDir: string | undefined = undefined - ): void { - const localBackupsBaseDir = localBackupSnapshotDir - ? dirname(localBackupSnapshotDir) - : undefined; - const isLocalBackup = localBackupsBaseDir != null; + public run(): void { drop( (async () => { log.info('BackupExportStream: starting...'); @@ -309,37 +304,44 @@ export class BackupExportStream extends Readable { await pauseWriteAccess(); try { - await this.#unsafeRun(backupLevel, isLocalBackup); + await this.#unsafeRun(); } catch (error) { this.emit('error', error); } finally { await resumeWriteAccess(); + // TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction + const { type } = this.options; + switch (type) { + case 'local-encrypted': + { + log.info( + `BackupExportStream: Adding ${this.#attachmentBackupJobs.length} jobs for AttachmentLocalBackupManager` + ); + const backupsBaseDir = dirname( + this.options.localBackupSnapshotDir + ); - if (isLocalBackup) { - log.info( - `BackupExportStream: Adding ${this.#attachmentBackupJobs.length} jobs for AttachmentLocalBackupManager` - ); - AttachmentLocalBackupManager.clearAllJobs(); - await Promise.all( - this.#attachmentBackupJobs.map(job => { - if (job.type !== 'local') { - log.error( - "BackupExportStream: Can't enqueue remote backup jobs during local backup, skipping" - ); - return Promise.resolve(); - } + AttachmentLocalBackupManager.clearAllJobs(); + await Promise.all( + this.#attachmentBackupJobs.map(job => { + if (job.type !== 'local') { + log.error( + "BackupExportStream: Can't enqueue remote backup jobs during local backup, skipping" + ); + return Promise.resolve(); + } - return AttachmentLocalBackupManager.addJob({ - ...job, - backupsBaseDir: localBackupsBaseDir, - }); - }) - ); - drop(AttachmentLocalBackupManager.start()); - } else { - // TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction - await DataWriter.clearAllAttachmentBackupJobs(); - if (this.backupType !== BackupType.TestOnlyPlaintext) { + return AttachmentLocalBackupManager.addJob({ + ...job, + backupsBaseDir, + }); + }) + ); + drop(AttachmentLocalBackupManager.start()); + } + break; + case 'remote': + await DataWriter.clearAllAttachmentBackupJobs(); await Promise.all( this.#attachmentBackupJobs.map(job => { if (job.type === 'local') { @@ -355,9 +357,13 @@ export class BackupExportStream extends Readable { }) ); drop(AttachmentBackupManager.start()); - } + break; + case 'cross-client-integration-test': + break; + default: + // eslint-disable-next-line no-unsafe-finally + throw missingCaseError(type); } - log.info('BackupExportStream: finished'); } })() @@ -372,10 +378,7 @@ export class BackupExportStream extends Readable { return this.#stats; } - async #unsafeRun( - backupLevel: BackupLevel, - isLocalBackup: boolean - ): Promise { + async #unsafeRun(): Promise { this.#ourConversation = window.ConversationController.getOurConversationOrThrow().attributes; this.push( @@ -752,8 +755,6 @@ export class BackupExportStream extends Readable { const chatItem = await this.#toChatItem(message, { aboutMe, callHistoryByCallId, - backupLevel, - isLocalBackup, }); if (chatItem === undefined) { @@ -1214,12 +1215,7 @@ export class BackupExportStream extends Readable { async #toChatItem( message: MessageAttributesType, - { - aboutMe, - callHistoryByCallId, - backupLevel, - isLocalBackup, - }: ToChatItemOptionsType + { aboutMe, callHistoryByCallId }: ToChatItemOptionsType ): Promise { const conversation = window.ConversationController.get( message.conversationId @@ -1378,8 +1374,6 @@ export class BackupExportStream extends Readable { if (isTapToView(message)) { result.viewOnceMessage = await this.#toViewOnceMessage({ message, - backupLevel, - isLocalBackup, }); } else if (message.deletedForEveryone) { result.remoteDeletedMessage = {}; @@ -1440,8 +1434,6 @@ export class BackupExportStream extends Readable { avatar: contactDetails.avatar?.avatar ? await this.#processAttachment({ attachment: contactDetails.avatar.avatar, - backupLevel, - isLocalBackup, messageReceivedAt: message.received_at, }) : undefined, @@ -1462,8 +1454,6 @@ export class BackupExportStream extends Readable { stickerProto.data = sticker.data ? await this.#processAttachment({ attachment: sticker.data, - backupLevel, - isLocalBackup, messageReceivedAt: message.received_at, }) : undefined; @@ -1506,29 +1496,15 @@ export class BackupExportStream extends Readable { } else if (message.storyReplyContext) { result.directStoryReplyMessage = await this.#toDirectStoryReplyMessage({ message, - backupLevel, - isLocalBackup, }); - result.revisions = await this.#toChatItemRevisions( - result, - message, - backupLevel, - isLocalBackup - ); + result.revisions = await this.#toChatItemRevisions(result, message); } else { result.standardMessage = await this.#toStandardMessage({ message, - backupLevel, - isLocalBackup, }); - result.revisions = await this.#toChatItemRevisions( - result, - message, - backupLevel, - isLocalBackup - ); + result.revisions = await this.#toChatItemRevisions(result, message); } if (isOutgoing) { @@ -2446,12 +2422,8 @@ export class BackupExportStream extends Readable { async #toQuote({ message, - backupLevel, - isLocalBackup, }: { message: Pick; - backupLevel: BackupLevel; - isLocalBackup: boolean; }): Promise { const { quote } = message; if (!quote) { @@ -2513,9 +2485,7 @@ export class BackupExportStream extends Readable { thumbnail: attachment.thumbnail ? await this.#processMessageAttachment({ attachment: attachment.thumbnail, - backupLevel, message, - isLocalBackup, }) : undefined, }; @@ -2572,21 +2542,15 @@ export class BackupExportStream extends Readable { async #processMessageAttachment({ attachment, - backupLevel, message, - isLocalBackup, }: { attachment: AttachmentType; - backupLevel: BackupLevel; message: Pick; - isLocalBackup: boolean; }): Promise { const { clientUuid } = attachment; const filePointer = await this.#processAttachment({ attachment, - backupLevel, messageReceivedAt: message.received_at, - isLocalBackup, }); return new Backups.MessageAttachment({ @@ -2599,26 +2563,28 @@ export class BackupExportStream extends Readable { async #processAttachment({ attachment, - backupLevel, - isLocalBackup, messageReceivedAt, }: { attachment: AttachmentType; - backupLevel: BackupLevel; - isLocalBackup: boolean; messageReceivedAt: number; }): Promise { const { filePointer, backupJob } = await getFilePointerForAttachment({ attachment, - isLocalBackup, - backupLevel, + backupOptions: this.options, messageReceivedAt, getBackupCdnInfo, }); - if (hasRequiredInformationForBackup(attachment)) { - const mediaName = getMediaNameForAttachment(attachment); + let mediaName: string | undefined; + if (this.options.type === 'local-encrypted') { + if (hasRequiredInformationForLocalBackup(attachment)) { + mediaName = getLocalBackupFileNameForAttachment(attachment); + } + } else if (hasRequiredInformationForRemoteBackup(attachment)) { + mediaName = getMediaNameForAttachment(attachment); + } + if (mediaName) { // Re-use existing locatorInfo and backup job if we've already seen this file const existingFilePointer = this.#mediaNamesToFilePointers.get(mediaName); @@ -2632,7 +2598,6 @@ export class BackupExportStream extends Readable { if (filePointer.locatorInfo) { this.#mediaNamesToFilePointers.set(mediaName, filePointer); } - if (backupJob) { this.#attachmentBackupJobs.push(backupJob); } @@ -2814,8 +2779,6 @@ export class BackupExportStream extends Readable { async #toStandardMessage({ message, - backupLevel, - isLocalBackup, }: { message: Pick< MessageAttributesType, @@ -2829,8 +2792,6 @@ export class BackupExportStream extends Readable { | 'received_at' | 'timestamp' >; - backupLevel: BackupLevel; - isLocalBackup: boolean; }): Promise { if ( message.body && @@ -2842,17 +2803,13 @@ export class BackupExportStream extends Readable { return { quote: await this.#toQuote({ message, - backupLevel, - isLocalBackup, }), attachments: message.attachments?.length ? await Promise.all( message.attachments.map(attachment => { return this.#processMessageAttachment({ attachment, - backupLevel, message, - isLocalBackup, }); }) ) @@ -2863,8 +2820,6 @@ export class BackupExportStream extends Readable { message.bodyAttachment && !isDownloaded(message.bodyAttachment) ? await this.#processAttachment({ attachment: message.bodyAttachment, - backupLevel, - isLocalBackup, messageReceivedAt: message.received_at, }) : undefined, @@ -2890,8 +2845,6 @@ export class BackupExportStream extends Readable { image: preview.image ? await this.#processAttachment({ attachment: preview.image, - backupLevel, - isLocalBackup, messageReceivedAt: message.received_at, }) : undefined, @@ -2905,8 +2858,6 @@ export class BackupExportStream extends Readable { async #toDirectStoryReplyMessage({ message, - backupLevel, - isLocalBackup, }: { message: Pick< MessageAttributesType, @@ -2917,8 +2868,6 @@ export class BackupExportStream extends Readable { | 'received_at' | 'reactions' >; - backupLevel: BackupLevel; - isLocalBackup: boolean; }): Promise { const result = new Backups.DirectStoryReplyMessage({ reactions: this.#getMessageReactions(message), @@ -2931,8 +2880,6 @@ export class BackupExportStream extends Readable { longText: message.bodyAttachment ? await this.#processAttachment({ attachment: message.bodyAttachment, - backupLevel, - isLocalBackup, messageReceivedAt: message.received_at, }) : undefined, @@ -2952,15 +2899,11 @@ export class BackupExportStream extends Readable { async #toViewOnceMessage({ message, - backupLevel, - isLocalBackup, }: { message: Pick< MessageAttributesType, 'attachments' | 'received_at' | 'reactions' >; - backupLevel: BackupLevel; - isLocalBackup: boolean; }): Promise { const attachment = message.attachments?.at(0); // Integration tests use the 'link-and-sync' version of export, which will include @@ -2972,9 +2915,7 @@ export class BackupExportStream extends Readable { ? null : await this.#processMessageAttachment({ attachment, - backupLevel, message, - isLocalBackup, }), reactions: this.#getMessageReactions(message), }; @@ -2982,9 +2923,7 @@ export class BackupExportStream extends Readable { async #toChatItemRevisions( parent: Backups.IChatItem, - message: MessageAttributesType, - backupLevel: BackupLevel, - isLocalBackup: boolean + message: MessageAttributesType ): Promise | undefined> { const { editHistory } = message; if (editHistory == null) { @@ -3022,14 +2961,10 @@ export class BackupExportStream extends Readable { result.directStoryReplyMessage = await this.#toDirectStoryReplyMessage({ message: history, - backupLevel, - isLocalBackup, }); } else { result.standardMessage = await this.#toStandardMessage({ message: history, - backupLevel, - isLocalBackup, }); } return result; diff --git a/ts/services/backups/import.preload.ts b/ts/services/backups/import.preload.ts index 9027a30201..5e9a4b6567 100644 --- a/ts/services/backups/import.preload.ts +++ b/ts/services/backups/import.preload.ts @@ -92,8 +92,11 @@ import { signalProtocolStore } from '../../SignalProtocolStore.preload.js'; import * as Bytes from '../../Bytes.std.js'; import { BACKUP_VERSION, WALLPAPER_TO_BUBBLE_COLOR } from './constants.std.js'; import { UnsupportedBackupVersion } from './errors.std.js'; -import type { AboutMe, LocalChatStyle } from './types.std.js'; -import { BackupType } from './types.std.js'; +import type { + AboutMe, + BackupImportOptions, + LocalChatStyle, +} from './types.std.js'; import { getBackupMediaRootKey } from './crypto.preload.js'; import type { GroupV2ChangeDetailType } from '../../types/groups.std.js'; import { queueAttachmentDownloads } from '../../util/queueAttachmentDownloads.preload.js'; @@ -274,22 +277,18 @@ export class BackupImportStream extends Writable { #frameErrorCount: number = 0; #backupTier: BackupLevel | undefined; - private constructor( - private readonly backupType: BackupType, - private readonly localBackupSnapshotDir: string | undefined - ) { + private constructor(private readonly options: BackupImportOptions) { super({ objectMode: true }); } public static async create( - backupType = BackupType.Ciphertext, - localBackupSnapshotDir: string | undefined = undefined + options: BackupImportOptions ): Promise { await AttachmentDownloadManager.stop(); await DataWriter.removeAllBackupAttachmentDownloadJobs(); await resetBackupMediaDownloadStats(); - return new BackupImportStream(backupType, localBackupSnapshotDir); + return new BackupImportStream(options); } override async _write( @@ -424,7 +423,7 @@ export class BackupImportStream extends Writable { await pMap( [...this.#pendingGroupAvatars.entries()], async ([conversationId, newAvatarUrl]) => { - if (this.backupType === BackupType.TestOnlyPlaintext) { + if (this.options.type === 'cross-client-integration-test') { return; } await groupAvatarJobQueue.add({ conversationId, newAvatarUrl }); @@ -446,7 +445,7 @@ export class BackupImportStream extends Writable { ); if ( - this.backupType !== BackupType.TestOnlyPlaintext && + this.options.type !== 'cross-client-integration-test' && !isTestEnvironment(getEnvironment()) ) { await startBackupMediaDownload(); @@ -1906,17 +1905,14 @@ export class BackupImportStream extends Writable { bodyRanges: this.#fromBodyRanges(data.text), })), bodyAttachment: data.longText - ? convertFilePointerToAttachment( - data.longText, - this.#getFilePointerOptions() - ) + ? convertFilePointerToAttachment(data.longText, this.options) : undefined, attachments: data.attachments?.length ? data.attachments .map(attachment => convertBackupMessageAttachmentToAttachment( attachment, - this.#getFilePointerOptions() + this.options ) ) .filter(isNotNil) @@ -1962,10 +1958,7 @@ export class BackupImportStream extends Writable { description: dropNull(preview.description), date: getCheckedTimestampOrUndefinedFromLong(preview.date), image: preview.image - ? convertFilePointerToAttachment( - preview.image, - this.#getFilePointerOptions() - ) + ? convertFilePointerToAttachment(preview.image, this.options) : undefined, }; }) @@ -1984,7 +1977,7 @@ export class BackupImportStream extends Writable { ? [ convertBackupMessageAttachmentToAttachment( attachment, - this.#getFilePointerOptions() + this.options ), ].filter(isNotNil) : undefined, @@ -2023,10 +2016,7 @@ export class BackupImportStream extends Writable { result.body = textReply.text?.body ?? undefined; result.bodyRanges = this.#fromBodyRanges(textReply.text); result.bodyAttachment = textReply.longText - ? convertFilePointerToAttachment( - textReply.longText, - this.#getFilePointerOptions() - ) + ? convertFilePointerToAttachment(textReply.longText, this.options) : undefined; } else if (emoji) { result.storyReaction = { @@ -2056,10 +2046,7 @@ export class BackupImportStream extends Writable { body: textReply.text?.body ?? undefined, bodyRanges: this.#fromBodyRanges(textReply.text), bodyAttachment: textReply.longText - ? convertFilePointerToAttachment( - textReply.longText, - this.#getFilePointerOptions() - ) + ? convertFilePointerToAttachment(textReply.longText, this.options) : undefined, }; } @@ -2182,10 +2169,7 @@ export class BackupImportStream extends Writable { ? stringToMIMEType(contentType) : APPLICATION_OCTET_STREAM, thumbnail: thumbnail?.pointer - ? convertFilePointerToAttachment( - thumbnail.pointer, - this.#getFilePointerOptions() - ) + ? convertFilePointerToAttachment(thumbnail.pointer, this.options) : undefined, }; }) ?? [], @@ -2347,7 +2331,7 @@ export class BackupImportStream extends Writable { ? { avatar: convertFilePointerToAttachment( avatar, - this.#getFilePointerOptions() + this.options ), isProfile: false, } @@ -2396,10 +2380,7 @@ export class BackupImportStream extends Writable { packKey: Bytes.toBase64(packKey), stickerId, data: data - ? convertFilePointerToAttachment( - data, - this.#getFilePointerOptions() - ) + ? convertFilePointerToAttachment(data, this.options) : undefined, }, reactions: this.#fromReactions(chatItem.stickerMessage.reactions), @@ -3800,16 +3781,8 @@ export class BackupImportStream extends Writable { }; } - #getFilePointerOptions() { - if (this.localBackupSnapshotDir != null) { - return { localBackupSnapshotDir: this.localBackupSnapshotDir }; - } - - return {}; - } - #isLocalBackup() { - return this.localBackupSnapshotDir != null; + return this.options.type === 'local-encrypted'; } #isMediaEnabledBackup() { diff --git a/ts/services/backups/index.preload.ts b/ts/services/backups/index.preload.ts index ee294c2fcd..509a2ca3fb 100644 --- a/ts/services/backups/index.preload.ts +++ b/ts/services/backups/index.preload.ts @@ -69,7 +69,7 @@ import { validateBackupStream, ValidationType, } from './validator.preload.js'; -import { BackupType } from './types.std.js'; +import type { BackupExportOptions, BackupImportOptions } from './types.std.js'; import { BackupInstallerError, BackupDownloadFailedError, @@ -103,8 +103,6 @@ const { isEqual, noop } = lodash; const log = createLogger('backupsService'); -export { BackupType }; - const IV_LENGTH = 16; const BACKUP_REFRESH_INTERVAL = 24 * HOUR; @@ -128,13 +126,6 @@ type DoDownloadOptionsType = Readonly<{ ) => void; }>; -export type ImportOptionsType = Readonly<{ - backupType?: BackupType; - localBackupSnapshotDir?: string; - ephemeralKey?: Uint8Array; - onProgress?: (currentBytes: number, totalBytes: number) => void; -}>; - export type ExportResultType = Readonly<{ totalBytes: number; duration: number; @@ -318,7 +309,10 @@ export class BackupsService { log.info(`exportBackup: starting, backup level: ${backupLevel}...`); try { - const { totalBytes } = await this.exportToDisk(filePath, backupLevel); + const { totalBytes } = await this.exportToDisk(filePath, { + type: 'remote', + level: backupLevel, + }); await this.api.upload(filePath, totalBytes); } finally { @@ -331,8 +325,7 @@ export class BackupsService { } public async exportLocalBackup( - backupsBaseDir: string | undefined = undefined, - backupLevel: BackupLevel = BackupLevel.Free + backupsBaseDir: string | undefined = undefined ): Promise { strictAssert(isLocalBackupsEnabled(), 'Local backups must be enabled'); @@ -347,12 +340,10 @@ export class BackupsService { log.info('exportLocalBackup: starting'); - const exportResult = await this.exportToDisk( - mainProtoPath, - backupLevel, - BackupType.Ciphertext, - snapshotDir - ); + const exportResult = await this.exportToDisk(mainProtoPath, { + type: 'local-encrypted', + localBackupSnapshotDir: snapshotDir, + }); log.info('exportLocalBackup: writing metadata'); const metadataArgs = { @@ -405,6 +396,7 @@ export class BackupsService { const backupFile = join(this.#localBackupSnapshotDir, 'main'); await this.importFromDisk(backupFile, { + type: 'local-encrypted', localBackupSnapshotDir: this.#localBackupSnapshotDir, }); @@ -421,14 +413,13 @@ export class BackupsService { // Test harness public async exportBackupData( - backupLevel: BackupLevel = BackupLevel.Free, - backupType = BackupType.Ciphertext + options: BackupExportOptions ): Promise<{ data: Uint8Array } & ExportResultType> { const sink = new PassThrough(); const chunks = new Array(); sink.on('data', chunk => chunks.push(chunk)); - const result = await this.#exportBackup(sink, backupLevel, backupType); + const result = await this.#exportBackup(sink, options); return { ...result, @@ -438,18 +429,14 @@ export class BackupsService { public async exportToDisk( path: string, - backupLevel: BackupLevel = BackupLevel.Free, - backupType = BackupType.Ciphertext, - localBackupSnapshotDir: string | undefined = undefined + options: BackupExportOptions ): Promise { const exportResult = await this.#exportBackup( createWriteStream(path), - backupLevel, - backupType, - localBackupSnapshotDir + options ); - if (backupType === BackupType.Ciphertext) { + if (options.type !== 'cross-client-integration-test') { await validateBackup( () => new FileStream(path), exportResult.totalBytes, @@ -462,9 +449,7 @@ export class BackupsService { return exportResult; } - public async _internalExportLocalBackup( - backupLevel: BackupLevel = BackupLevel.Free - ): Promise { + public async _internalExportLocalBackup(): Promise { try { const { canceled, dirPath: backupsBaseDir } = await ipcRenderer.invoke( 'show-open-folder-dialog' @@ -473,7 +458,7 @@ export class BackupsService { return { error: 'Backups directory not selected' }; } - const result = await this.exportLocalBackup(backupsBaseDir, backupLevel); + const result = await this.exportLocalBackup(backupsBaseDir); return { result }; } catch (error) { return { error: Errors.toLogFormat(error) }; @@ -496,16 +481,16 @@ export class BackupsService { } // Test harness - public async _internalValidate( - backupLevel: BackupLevel = BackupLevel.Free, - backupType = BackupType.Ciphertext - ): Promise { + public async _internalValidate(): Promise { try { const start = Date.now(); - const recordStream = new BackupExportStream(backupType); + const recordStream = new BackupExportStream({ + type: 'remote', + level: BackupLevel.Free, + }); - recordStream.run(backupLevel); + recordStream.run(); const totalBytes = await validateBackupStream(recordStream); @@ -521,7 +506,10 @@ export class BackupsService { // Test harness public async exportWithDialog(): Promise { - const { data } = await this.exportBackupData(); + const { data } = await this.exportBackupData({ + type: 'remote', + level: BackupLevel.Free, + }); await saveAttachmentToDisk({ name: 'backup.bin', @@ -531,7 +519,7 @@ export class BackupsService { public async importFromDisk( backupFile: string, - options?: ImportOptionsType + options: BackupImportOptions ): Promise { return this.importBackup(() => createReadStream(backupFile), options); } @@ -562,18 +550,13 @@ export class BackupsService { public async importBackup( createBackupStream: () => Readable, - { - backupType = BackupType.Ciphertext, - ephemeralKey, - onProgress, - localBackupSnapshotDir = undefined, - }: ImportOptionsType = {} + options: BackupImportOptions ): Promise { strictAssert(!this.#isRunning, 'BackupService is already running'); window.IPC.startTrackingQueryStats(); - log.info(`importBackup: starting ${backupType}...`); + log.info(`importBackup: starting ${options.type}...`); this.#isRunning = 'import'; const importStart = Date.now(); @@ -588,13 +571,10 @@ export class BackupsService { window.ConversationController.setReadOnly(true); - const importStream = await BackupImportStream.create( - backupType, - localBackupSnapshotDir - ); - if (backupType === BackupType.Ciphertext) { + const importStream = await BackupImportStream.create(options); + if (options.type === 'remote' || options.type === 'local-encrypted') { const { aesKey, macKey } = getKeyMaterial( - ephemeralKey ? new BackupKey(ephemeralKey) : undefined + options.ephemeralKey ? new BackupKey(options.ephemeralKey) : undefined ); // First pass - don't decrypt, only verify mac @@ -621,7 +601,7 @@ export class BackupsService { throw new BackupImportCanceledError(); } - onProgress?.(0, totalBytes); + options.onProgress?.(0, totalBytes); strictAssert(theirMac != null, 'importBackup: Missing MAC'); strictAssert( @@ -638,7 +618,7 @@ export class BackupsService { let currentBytes = 0; progressReporter.on('data', chunk => { currentBytes += chunk.byteLength; - onProgress?.(currentBytes, totalBytes); + options.onProgress?.(currentBytes, totalBytes); }); await pipeline( @@ -656,13 +636,13 @@ export class BackupsService { constantTimeEqual(hmac.digest(), theirMac), 'importBackup: Bad MAC, second pass' ); - } else if (backupType === BackupType.TestOnlyPlaintext) { + } else if (options.type === 'cross-client-integration-test') { strictAssert( isTestOrMockEnvironment(), 'Plaintext backups can be imported only in test harness' ); strictAssert( - ephemeralKey == null, + options.ephemeralKey == null, 'Plaintext backups cannot have ephemeral key' ); await pipeline( @@ -671,7 +651,7 @@ export class BackupsService { importStream ); } else { - throw missingCaseError(backupType); + throw missingCaseError(options.type); } log.info('importBackup: finished...'); @@ -876,6 +856,7 @@ export class BackupsService { await itemStorage.remove('password'); await this.importFromDisk(downloadPath, { + type: 'remote', ephemeralKey, onProgress: (currentBytes, totalBytes) => { onProgress?.( @@ -913,9 +894,7 @@ export class BackupsService { async #exportBackup( sink: Writable, - backupLevel: BackupLevel = BackupLevel.Free, - backupType = BackupType.Ciphertext, - localBackupSnapshotDir: string | undefined = undefined + options: BackupExportOptions ): Promise { strictAssert(!this.#isRunning, 'BackupService is already running'); @@ -925,7 +904,7 @@ export class BackupsService { const start = Date.now(); try { // TODO (DESKTOP-7168): Update mock-server to support this endpoint - if (window.SignalCI || backupType === BackupType.TestOnlyPlaintext) { + if (window.SignalCI || options.type === 'cross-client-integration-test') { strictAssert( isTestOrMockEnvironment(), 'Plaintext backups can be exported only in test harness' @@ -938,47 +917,52 @@ export class BackupsService { } const { aesKey, macKey } = getKeyMaterial(); - const recordStream = new BackupExportStream(backupType); + const recordStream = new BackupExportStream(options); - recordStream.run(backupLevel, localBackupSnapshotDir); + recordStream.run(); const iv = randomBytes(IV_LENGTH); let totalBytes = 0; - if (backupType === BackupType.Ciphertext) { - await pipeline( - recordStream, - createGzip(), - appendPaddingStream(), - createCipheriv(CipherType.AES256CBC, aesKey, iv), - prependStream(iv), - appendMacStream(macKey), - measureSize({ - onComplete: size => { - totalBytes = size; - }, - }), - sink - ); - } else if (backupType === BackupType.TestOnlyPlaintext) { - strictAssert( - isTestOrMockEnvironment(), - 'Plaintext backups can be exported only in test harness' - ); - await pipeline(recordStream, sink); - } else { - throw missingCaseError(backupType); + const { type } = options; + switch (type) { + case 'remote': + case 'local-encrypted': + await pipeline( + recordStream, + createGzip(), + appendPaddingStream(), + createCipheriv(CipherType.AES256CBC, aesKey, iv), + prependStream(iv), + appendMacStream(macKey), + measureSize({ + onComplete: size => { + totalBytes = size; + }, + }), + sink + ); + break; + case 'cross-client-integration-test': + strictAssert( + isTestOrMockEnvironment(), + 'Plaintext backups can be exported only in test harness' + ); + await pipeline(recordStream, sink); + break; + default: + throw missingCaseError(type); } - if (localBackupSnapshotDir) { + if (type === 'local-encrypted') { log.info('exportBackup: writing local backup files list'); const filesWritten = await writeLocalBackupFilesList({ - snapshotDir: localBackupSnapshotDir, + snapshotDir: options.localBackupSnapshotDir, mediaNamesIterator: recordStream.getMediaNamesIterator(), }); const filesRead = await readLocalBackupFilesList( - localBackupSnapshotDir + options.localBackupSnapshotDir ); strictAssert( isEqual(filesWritten, filesRead), diff --git a/ts/services/backups/types.std.ts b/ts/services/backups/types.std.ts index 33a5746c14..cd2d356ffd 100644 --- a/ts/services/backups/types.std.ts +++ b/ts/services/backups/types.std.ts @@ -25,10 +25,22 @@ export type AboutMe = { e164?: string; }; -export enum BackupType { - Ciphertext = 'Ciphertext', - TestOnlyPlaintext = 'TestOnlyPlaintext', -} +export type BackupExportOptions = + | { type: 'remote' | 'cross-client-integration-test'; level: BackupLevel } + | { + type: 'local-encrypted'; + localBackupSnapshotDir: string; + }; +export type BackupImportOptions = ( + | { type: 'remote' | 'cross-client-integration-test' } + | { + type: 'local-encrypted'; + localBackupSnapshotDir: string; + } +) & { + ephemeralKey?: Uint8Array; + onProgress?: (currentBytes: number, totalBytes: number) => void; +}; export type LocalChatStyle = Readonly<{ wallpaperPhotoPointer: Uint8Array | undefined; diff --git a/ts/services/backups/util/filePointers.preload.ts b/ts/services/backups/util/filePointers.preload.ts index e71de2c516..0452270dd5 100644 --- a/ts/services/backups/util/filePointers.preload.ts +++ b/ts/services/backups/util/filePointers.preload.ts @@ -1,17 +1,16 @@ // Copyright 2024 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only -import { existsSync } from 'node:fs'; import { BackupLevel } from '@signalapp/libsignal-client/zkgroup.js'; import { APPLICATION_OCTET_STREAM, stringToMIMEType, } from '../../../types/MIME.std.js'; -import { createLogger } from '../../../logging/log.std.js'; import type { AttachmentType } from '../../../types/Attachment.std.js'; -import { getAbsoluteAttachmentPath } from '../../../util/migrations.preload.js'; +import { doesAttachmentExist } from '../../../util/migrations.preload.js'; import { - hasRequiredInformationForBackup, + hasRequiredInformationForLocalBackup, + hasRequiredInformationForRemoteBackup, hasRequiredInformationToDownloadFromTransitTier, } from '../../../util/Attachment.std.js'; import { Backups, SignalService } from '../../../protobuf/index.std.js'; @@ -29,6 +28,10 @@ import { type GetBackupCdnInfoType, getMediaIdFromMediaName, getMediaName, + getMediaNameForAttachment, + type BackupCdnInfoType, + getLocalBackupFileNameForAttachment, + getLocalBackupFileName, } from './mediaId.preload.js'; import { missingCaseError } from '../../../util/missingCaseError.std.js'; import { bytesToUuid } from '../../../util/uuidToBytes.std.js'; @@ -37,21 +40,16 @@ import { generateAttachmentKeys } from '../../../AttachmentCrypto.node.js'; import { getAttachmentLocalBackupPathFromSnapshotDir } from './localBackup.node.js'; import { isValidAttachmentKey, + isValidDigest, isValidPlaintextHash, } from '../../../types/Crypto.std.js'; +import type { BackupExportOptions, BackupImportOptions } from '../types.std.js'; import { isTestOrMockEnvironment } from '../../../environment.std.js'; -const log = createLogger('filePointers'); - -type ConvertFilePointerToAttachmentOptions = { - // Only for testing - _createName: (suffix?: string) => string; - localBackupSnapshotDir: string | undefined; -}; - export function convertFilePointerToAttachment( filePointer: Backups.FilePointer, - options: Partial = {} + options: BackupImportOptions, + testDependencies?: { _createName: (suffix?: string) => string } ): AttachmentType { const { contentType, @@ -64,7 +62,7 @@ export function convertFilePointerToAttachment( incrementalMacChunkSize, locatorInfo, } = filePointer; - const doCreateName = options._createName ?? createName; + const doCreateName = testDependencies?._createName ?? createName; const commonProps: AttachmentType = { size: 0, @@ -124,24 +122,16 @@ export function convertFilePointerToAttachment( } let localBackupPath: string | undefined; - if (Bytes.isNotEmpty(localKey)) { - const { localBackupSnapshotDir } = options; - - strictAssert( - localBackupSnapshotDir, - 'localBackupSnapshotDir is required for filePointer.localLocator' + if ( + options.type === 'local-encrypted' && + Bytes.isNotEmpty(localKey) && + Bytes.isNotEmpty(plaintextHash) + ) { + const localMediaName = getLocalBackupFileName({ plaintextHash, localKey }); + localBackupPath = getAttachmentLocalBackupPathFromSnapshotDir( + localMediaName, + options.localBackupSnapshotDir ); - - if (mediaName) { - localBackupPath = getAttachmentLocalBackupPathFromSnapshotDir( - mediaName, - localBackupSnapshotDir - ); - } else { - log.error( - 'convertFilePointerToAttachment: localKey but no plaintextHash' - ); - } } return { @@ -172,7 +162,7 @@ export function convertFilePointerToAttachment( export function convertBackupMessageAttachmentToAttachment( messageAttachment: Backups.IMessageAttachment, - options: Partial = {} + options: BackupImportOptions ): AttachmentType | null { const { clientUuid } = messageAttachment; @@ -207,21 +197,21 @@ export function convertBackupMessageAttachmentToAttachment( } export async function getFilePointerForAttachment({ - attachment, + attachment: rawAttachment, getBackupCdnInfo, - backupLevel, + backupOptions, messageReceivedAt, - isLocalBackup = false, }: { attachment: Readonly; getBackupCdnInfo: GetBackupCdnInfoType; - backupLevel: BackupLevel; + backupOptions: BackupExportOptions; messageReceivedAt: number; - isLocalBackup?: boolean; }): Promise<{ filePointer: Backups.FilePointer; backupJob?: CoreAttachmentBackupJobType | PartialAttachmentLocalBackupJobType; }> { + const attachment = maybeFixupAttachment(rawAttachment); + const filePointer = new Backups.FilePointer({ contentType: attachment.contentType, fileName: attachment.fileName, @@ -241,67 +231,79 @@ export async function getFilePointerForAttachment({ } } - const locatorInfo = getLocatorInfoForAttachment({ + const isAttachmentOnDisk = + attachment.path != null && (await doesAttachmentExist(attachment.path)); + + const remoteMediaName = hasRequiredInformationForRemoteBackup(attachment) + ? getMediaNameForAttachment(attachment) + : undefined; + + const remoteMediaId = remoteMediaName + ? getMediaIdFromMediaName(remoteMediaName) + : undefined; + + const remoteBackupStatus: BackupCdnInfoType = remoteMediaId + ? await getBackupCdnInfo(remoteMediaId.string) + : { isInBackupTier: false }; + + const isLocalBackup = backupOptions.type === 'local-encrypted'; + filePointer.locatorInfo = getLocatorInfoForAttachment({ attachment, - isLocalBackup, + backupOptions, + isOnDisk: isAttachmentOnDisk, + backupTierInfo: remoteBackupStatus, }); - if (locatorInfo) { - filePointer.locatorInfo = locatorInfo; - } - - let backupJob: - | CoreAttachmentBackupJobType - | PartialAttachmentLocalBackupJobType - | undefined; - - if (backupLevel !== BackupLevel.Paid && !isLocalBackup) { - return { filePointer, backupJob: undefined }; - } - - if (!Bytes.isNotEmpty(locatorInfo.plaintextHash)) { - return { filePointer, backupJob: undefined }; - } - - const mediaName = getMediaName({ - plaintextHash: locatorInfo.plaintextHash, - key: locatorInfo.key, - }); - - const backupInfo = await getBackupCdnInfo( - getMediaIdFromMediaName(mediaName).string - ); - - if (backupInfo.isInBackupTier) { - if (locatorInfo.mediaTierCdnNumber !== backupInfo.cdnNumber) { - log.warn( - 'backupCdnNumber on attachment differs from cdnNumber from list endpoint' - ); - // Prefer the one from the list endpoint - locatorInfo.mediaTierCdnNumber = backupInfo.cdnNumber; - } - return { filePointer, backupJob: undefined }; - } - - const { path, localKey, version, size } = attachment; - - if (!path || !isValidAttachmentKey(localKey)) { - return { filePointer, backupJob: undefined }; - } - if (isLocalBackup) { - backupJob = { - mediaName, - type: 'local', - data: { - path, - size, - localKey, - }, + if ( + isAttachmentOnDisk && + hasRequiredInformationForLocalBackup(attachment) + ) { + return { + filePointer, + backupJob: { + mediaName: getLocalBackupFileNameForAttachment(attachment), + type: 'local', + data: { + path: attachment.path, + }, + }, + }; + } + return { + filePointer, + backupJob: undefined, }; - } else { - backupJob = { - mediaName, + } + + if (backupOptions.level !== BackupLevel.Paid) { + return { filePointer, backupJob: undefined }; + } + + if (remoteBackupStatus.isInBackupTier) { + return { filePointer, backupJob: undefined }; + } + + if (!isAttachmentOnDisk) { + return { filePointer, backupJob: undefined }; + } + + if (!remoteMediaName) { + return { filePointer, backupJob: undefined }; + } + + const { path, localKey, key, version } = attachment; + + strictAssert(path, 'Path must exist for attachment on disk'); + strictAssert(key, 'Key must exist for remote backupable attachment'); + + const { transitCdnKey, transitCdnNumber, transitTierUploadTimestamp } = + filePointer.locatorInfo; + + return { + filePointer, + backupJob: { + mediaName: remoteMediaName, receivedAt: messageReceivedAt, type: 'standard', data: { @@ -309,74 +311,86 @@ export async function getFilePointerForAttachment({ localKey, version, contentType: attachment.contentType, - keys: Bytes.toBase64(locatorInfo.key), - size: locatorInfo.size, + keys: key, + size: attachment.size, transitCdnInfo: - locatorInfo.transitCdnKey && locatorInfo.transitCdnNumber != null + transitCdnKey && transitCdnNumber != null ? { - cdnKey: locatorInfo.transitCdnKey, - cdnNumber: locatorInfo.transitCdnNumber, - uploadTimestamp: - locatorInfo.transitTierUploadTimestamp?.toNumber(), + cdnKey: transitCdnKey, + cdnNumber: transitCdnNumber, + uploadTimestamp: transitTierUploadTimestamp?.toNumber(), } : undefined, }, - }; - } - - return { filePointer, backupJob }; + }, + }; } +function maybeFixupAttachment(attachment: AttachmentType): AttachmentType { + // Fixup attachment which has plaintextHash but no key + if ( + isValidPlaintextHash(attachment.plaintextHash) && + !isValidAttachmentKey(attachment.key) + ) { + const fixedUpAttachment = { ...attachment }; + fixedUpAttachment.key = Bytes.toBase64(generateAttachmentKeys()); + // Delete all info dependent on key + delete fixedUpAttachment.cdnKey; + delete fixedUpAttachment.cdnNumber; + delete fixedUpAttachment.uploadTimestamp; + delete fixedUpAttachment.digest; + delete fixedUpAttachment.backupCdnNumber; + + strictAssert( + hasRequiredInformationForRemoteBackup(fixedUpAttachment), + 'should be backupable with new key' + ); + return fixedUpAttachment; + } + return attachment; +} function getLocatorInfoForAttachment({ - attachment: _rawAttachment, - isLocalBackup, + attachment, + backupOptions, + isOnDisk, + backupTierInfo, }: { attachment: AttachmentType; - isLocalBackup: boolean; + backupOptions: BackupExportOptions; + isOnDisk: boolean; + backupTierInfo: BackupCdnInfoType; }): Backups.FilePointer.LocatorInfo { const locatorInfo = new Backups.FilePointer.LocatorInfo(); - const attachment = { ..._rawAttachment }; - if (attachment.error) { - return locatorInfo; - } + const isLocalBackup = backupOptions.type === 'local-encrypted'; - { - const isBackupable = hasRequiredInformationForBackup(attachment); - const isDownloadableFromTransitTier = - hasRequiredInformationToDownloadFromTransitTier(attachment); + const shouldBeLocallyBackedUp = + isLocalBackup && + isOnDisk && + hasRequiredInformationForLocalBackup(attachment); - if (!isBackupable && !isDownloadableFromTransitTier) { - // TODO: DESKTOP-8914 - if ( - isValidPlaintextHash(attachment.plaintextHash) && - !isValidAttachmentKey(attachment.key) - ) { - attachment.key = Bytes.toBase64(generateAttachmentKeys()); - // Delete all info dependent on key - delete attachment.cdnKey; - delete attachment.cdnNumber; - delete attachment.uploadTimestamp; - delete attachment.digest; - delete attachment.backupCdnNumber; - - strictAssert( - hasRequiredInformationForBackup(attachment), - 'should be backupable with new key' - ); - } - } - } - const isBackupable = hasRequiredInformationForBackup(attachment); const isDownloadableFromTransitTier = hasRequiredInformationToDownloadFromTransitTier(attachment); - if (!isBackupable && !isDownloadableFromTransitTier) { + if ( + !shouldBeLocallyBackedUp && + !isDownloadableFromTransitTier && + !hasRequiredInformationForRemoteBackup(attachment) + ) { return locatorInfo; } locatorInfo.size = attachment.size; - locatorInfo.key = Bytes.fromBase64(attachment.key); + + if (isValidAttachmentKey(attachment.key)) { + locatorInfo.key = Bytes.fromBase64(attachment.key); + } + + if (isValidPlaintextHash(attachment.plaintextHash)) { + locatorInfo.plaintextHash = Bytes.fromHex(attachment.plaintextHash); + } else if (isValidDigest(attachment.digest)) { + locatorInfo.encryptedDigest = Bytes.fromBase64(attachment.digest); + } if (isDownloadableFromTransitTier) { locatorInfo.transitCdnKey = attachment.cdnKey; @@ -386,25 +400,14 @@ function getLocatorInfoForAttachment({ ); } - if (isBackupable) { - locatorInfo.plaintextHash = Bytes.fromHex(attachment.plaintextHash); - // TODO: DESKTOP-8887 - if (attachment.backupCdnNumber != null) { - locatorInfo.mediaTierCdnNumber = attachment.backupCdnNumber; - } - } else { - locatorInfo.encryptedDigest = Bytes.fromBase64(attachment.digest); + if (shouldBeLocallyBackedUp) { + locatorInfo.localKey = Bytes.fromBase64(attachment.localKey); } - // TODO: DESKTOP-8904 - if (isLocalBackup && isBackupable) { - const attachmentExistsLocally = - attachment.path != null && - existsSync(getAbsoluteAttachmentPath(attachment.path)); - - if (attachmentExistsLocally && attachment.localKey) { - locatorInfo.localKey = Bytes.fromBase64(attachment.localKey); - } + if (backupTierInfo.isInBackupTier && backupTierInfo.cdnNumber != null) { + locatorInfo.mediaTierCdnNumber = backupTierInfo.cdnNumber; + } else if (backupOptions.type === 'cross-client-integration-test') { + locatorInfo.mediaTierCdnNumber = attachment.backupCdnNumber; } return locatorInfo; diff --git a/ts/services/backups/util/mediaId.preload.ts b/ts/services/backups/util/mediaId.preload.ts index cf323b637b..e4826f6048 100644 --- a/ts/services/backups/util/mediaId.preload.ts +++ b/ts/services/backups/util/mediaId.preload.ts @@ -4,7 +4,11 @@ import { DataReader } from '../../../sql/Client.preload.js'; import * as Bytes from '../../../Bytes.std.js'; import { getBackupMediaRootKey } from '../crypto.preload.js'; -import type { BackupableAttachmentType } from '../../../types/Attachment.std.js'; +import type { + BackupableAttachmentType, + AttachmentReadyForLocalBackup, +} from '../../../types/Attachment.std.js'; +import { sha256 } from '../../../Crypto.node.js'; export function getMediaIdFromMediaName(mediaName: string): { string: string; @@ -56,6 +60,25 @@ export function getMediaName({ return Bytes.toHex(Bytes.concatenate([plaintextHash, key])); } +export function getLocalBackupFileNameForAttachment( + attachment: AttachmentReadyForLocalBackup +): string { + return getLocalBackupFileName({ + plaintextHash: Bytes.fromHex(attachment.plaintextHash), + localKey: Bytes.fromBase64(attachment.localKey), + }); +} + +export function getLocalBackupFileName({ + plaintextHash, + localKey, +}: { + plaintextHash: Uint8Array; + localKey: Uint8Array; +}): string { + return Bytes.toHex(sha256(Bytes.concatenate([plaintextHash, localKey]))); +} + export function getMediaNameForAttachmentThumbnail( fullsizeMediaName: string ): `${string}_thumbnail` { diff --git a/ts/test-electron/backup/attachments_test.preload.ts b/ts/test-electron/backup/attachments_test.preload.ts index 549d822b33..b9ed5281ba 100644 --- a/ts/test-electron/backup/attachments_test.preload.ts +++ b/ts/test-electron/backup/attachments_test.preload.ts @@ -26,7 +26,7 @@ import type { QuotedMessageType, } from '../../model-types.d.ts'; import { - hasRequiredInformationForBackup, + hasRequiredInformationForRemoteBackup, isVoiceMessage, } from '../../util/Attachment.std.js'; import type { AttachmentType } from '../../types/Attachment.std.js'; @@ -128,7 +128,7 @@ describe('backup/attachments', () => { attachment: AttachmentType ): AttachmentType { const base = omit(attachment, NON_ROUNDTRIPPED_FIELDS); - if (hasRequiredInformationForBackup(attachment)) { + if (hasRequiredInformationForRemoteBackup(attachment)) { delete base.digest; } else { delete base.plaintextHash; @@ -793,7 +793,7 @@ describe('backup/attachments', () => { strictAssert(key, 'thumbnail key was created'); strictAssert(plaintextHash, 'quote plaintextHash was roundtripped'); strictAssert( - hasRequiredInformationForBackup(thumbnail), + hasRequiredInformationForRemoteBackup(thumbnail), 'has key and plaintextHash' ); assert.deepStrictEqual(thumbnail, { diff --git a/ts/test-electron/backup/filePointer_test.preload.ts b/ts/test-electron/backup/filePointer_test.preload.ts index bd1feac3e2..22dcab61ae 100644 --- a/ts/test-electron/backup/filePointer_test.preload.ts +++ b/ts/test-electron/backup/filePointer_test.preload.ts @@ -6,6 +6,7 @@ import * as sinon from 'sinon'; import { BackupLevel } from '@signalapp/libsignal-client/zkgroup.js'; import { randomBytes } from 'node:crypto'; import { join } from 'node:path'; +import { emptyDir, ensureFile } from 'fs-extra'; import { Backups } from '../../protobuf/index.std.js'; @@ -22,6 +23,9 @@ import type { GetBackupCdnInfoType } from '../../services/backups/util/mediaId.p import { strictAssert } from '../../util/assert.std.js'; import { isValidAttachmentKey } from '../../types/Crypto.std.js'; import { itemStorage } from '../../textsecure/Storage.preload.js'; +import { getAbsoluteAttachmentPath } from '../../util/migrations.preload.js'; +import { getPath } from '../../../app/attachments.node.js'; +import { sha256 } from '../../Crypto.node.js'; describe('convertFilePointerToAttachment', () => { const commonFilePointerProps = { @@ -52,6 +56,7 @@ describe('convertFilePointerToAttachment', () => { ...commonFilePointerProps, locatorInfo: {}, }), + { type: 'remote' }, { _createName: () => 'downloadPath' } ); @@ -65,6 +70,7 @@ describe('convertFilePointerToAttachment', () => { it('processes filepointer with missing locatorInfo', () => { const result = convertFilePointerToAttachment( new Backups.FilePointer(commonFilePointerProps), + { type: 'remote' }, { _createName: () => 'downloadPath' } ); @@ -90,6 +96,7 @@ describe('convertFilePointerToAttachment', () => { mediaTierCdnNumber: 43, }, }), + { type: 'remote' }, { _createName: () => 'downloadPath' } ); @@ -123,17 +130,19 @@ describe('convertFilePointerToAttachment', () => { localKey: Bytes.fromString('localKey'), }, }), + { type: 'local-encrypted', localBackupSnapshotDir: '/root/backups' }, { _createName: () => 'downloadPath', - localBackupSnapshotDir: '/root/backups', } ); const mediaName = Bytes.toHex( - Bytes.concatenate([ - Bytes.fromString('plaintextHash'), - Bytes.fromString('key'), - ]) + sha256( + Bytes.concatenate([ + Bytes.fromString('plaintextHash'), + Bytes.fromString('localKey'), + ]) + ) ); assert.deepStrictEqual(result, { ...commonAttachmentProps, @@ -207,7 +216,7 @@ const notInBackupCdn: GetBackupCdnInfoType = async () => { describe('getFilePointerForAttachment', () => { let sandbox: sinon.SinonSandbox; - beforeEach(() => { + beforeEach(async () => { sandbox = sinon.createSandbox(); sandbox.stub(itemStorage, 'get').callsFake(key => { if (key === 'masterKey') { @@ -218,16 +227,21 @@ describe('getFilePointerForAttachment', () => { } return undefined; }); + await ensureFile(getAbsoluteAttachmentPath(defaultAttachment.path)); }); - afterEach(() => { + afterEach(async () => { sandbox.restore(); + await emptyDir(getPath(window.SignalContext.config.userDataPath)); }); it('if missing key, generates a new one and removes transit info & digest', async () => { const { filePointer } = await getFilePointerForAttachment({ attachment: { ...defaultAttachment, key: undefined }, - backupLevel: BackupLevel.Paid, + backupOptions: { + type: 'remote', + level: BackupLevel.Paid, + }, getBackupCdnInfo: notInBackupCdn, messageReceivedAt: 100, }); @@ -254,7 +268,10 @@ describe('getFilePointerForAttachment', () => { assert.deepEqual( await getFilePointerForAttachment({ attachment: { ...defaultAttachment, plaintextHash: undefined }, - backupLevel: BackupLevel.Paid, + backupOptions: { + type: 'remote', + level: BackupLevel.Paid, + }, getBackupCdnInfo: notInBackupCdn, messageReceivedAt: 100, }), @@ -278,7 +295,10 @@ describe('getFilePointerForAttachment', () => { assert.deepEqual( await getFilePointerForAttachment({ attachment: defaultAttachment, - backupLevel: BackupLevel.Free, + backupOptions: { + type: 'remote', + level: BackupLevel.Free, + }, getBackupCdnInfo: notInBackupCdn, messageReceivedAt: 100, }), @@ -292,7 +312,6 @@ describe('getFilePointerForAttachment', () => { transitCdnKey: 'cdnKey', transitCdnNumber: 2, transitTierUploadTimestamp: Long.fromNumber(1234), - mediaTierCdnNumber: 42, }), }), backupJob: undefined, @@ -304,7 +323,10 @@ describe('getFilePointerForAttachment', () => { assert.deepEqual( await getFilePointerForAttachment({ attachment: { ...defaultAttachment, digest: undefined }, - backupLevel: BackupLevel.Free, + backupOptions: { + type: 'remote', + level: BackupLevel.Free, + }, getBackupCdnInfo: notInBackupCdn, messageReceivedAt: 100, }), @@ -318,7 +340,6 @@ describe('getFilePointerForAttachment', () => { transitCdnKey: 'cdnKey', transitCdnNumber: 2, transitTierUploadTimestamp: Long.fromNumber(1234), - mediaTierCdnNumber: 42, }), }), backupJob: undefined, @@ -330,7 +351,10 @@ describe('getFilePointerForAttachment', () => { assert.deepEqual( await getFilePointerForAttachment({ attachment: { ...defaultAttachment, cdnKey: undefined }, - backupLevel: BackupLevel.Free, + backupOptions: { + type: 'remote', + level: BackupLevel.Free, + }, getBackupCdnInfo: notInBackupCdn, messageReceivedAt: 100, }), @@ -341,7 +365,6 @@ describe('getFilePointerForAttachment', () => { plaintextHash: Bytes.fromHex(defaultAttachment.plaintextHash), key: Bytes.fromBase64(defaultAttachment.key), size: 100, - mediaTierCdnNumber: 42, }), }), backupJob: undefined, @@ -352,7 +375,10 @@ describe('getFilePointerForAttachment', () => { assert.deepEqual( await getFilePointerForAttachment({ attachment: defaultAttachment, - backupLevel: BackupLevel.Paid, + backupOptions: { + type: 'remote', + level: BackupLevel.Paid, + }, getBackupCdnInfo: notInBackupCdn, messageReceivedAt: 100, }), @@ -366,7 +392,6 @@ describe('getFilePointerForAttachment', () => { transitCdnKey: 'cdnKey', transitCdnNumber: 2, transitTierUploadTimestamp: Long.fromNumber(1234), - mediaTierCdnNumber: 42, }), }), backupJob: { @@ -390,14 +415,16 @@ describe('getFilePointerForAttachment', () => { } ); }); - it('if local backup includes local backup job', async () => { + it('does not include backup job if file does not exist', async () => { assert.deepEqual( await getFilePointerForAttachment({ - attachment: defaultAttachment, - backupLevel: BackupLevel.Paid, + attachment: { ...defaultAttachment, path: 'not/here' }, + backupOptions: { + type: 'remote', + level: BackupLevel.Paid, + }, getBackupCdnInfo: notInBackupCdn, messageReceivedAt: 100, - isLocalBackup: true, }), { filePointer: new FilePointer({ @@ -409,19 +436,82 @@ describe('getFilePointerForAttachment', () => { transitCdnKey: 'cdnKey', transitCdnNumber: 2, transitTierUploadTimestamp: Long.fromNumber(1234), - mediaTierCdnNumber: 42, }), }), - backupJob: { - data: { - localKey: defaultAttachment.localKey, - path: defaultAttachment.path, - size: 100, - }, - mediaName: defaultMediaName, - type: 'local', - }, + backupJob: undefined, } ); }); + describe('local backups', () => { + const defaultLocalMediaName = Bytes.toHex( + sha256( + Bytes.concatenate([ + Bytes.fromHex(defaultAttachment.plaintextHash), + Bytes.fromBase64(defaultAttachment.localKey), + ]) + ) + ); + + it('generates local backup locatorInfo and a local backup job', async () => { + assert.deepEqual( + await getFilePointerForAttachment({ + attachment: defaultAttachment, + backupOptions: { + type: 'local-encrypted', + localBackupSnapshotDir: '/root/backups', + }, + getBackupCdnInfo: notInBackupCdn, + messageReceivedAt: 100, + }), + { + filePointer: new FilePointer({ + ...defaultFilePointer, + locatorInfo: new LocatorInfo({ + plaintextHash: Bytes.fromHex(defaultAttachment.plaintextHash), + localKey: Bytes.fromBase64(defaultAttachment.localKey), + key: Bytes.fromBase64(defaultAttachment.key), + size: 100, + transitCdnKey: 'cdnKey', + transitCdnNumber: 2, + transitTierUploadTimestamp: Long.fromNumber(1234), + }), + }), + backupJob: { + data: { + path: defaultAttachment.path, + }, + mediaName: defaultLocalMediaName, + type: 'local', + }, + } + ); + }); + it('if file does not exist, does not include localKey or backup job', async () => { + assert.deepEqual( + await getFilePointerForAttachment({ + attachment: { ...defaultAttachment, path: 'no/file/here' }, + backupOptions: { + type: 'local-encrypted', + localBackupSnapshotDir: '/root/backups', + }, + getBackupCdnInfo: notInBackupCdn, + messageReceivedAt: 100, + }), + { + filePointer: new FilePointer({ + ...defaultFilePointer, + locatorInfo: new LocatorInfo({ + plaintextHash: Bytes.fromHex(defaultAttachment.plaintextHash), + key: Bytes.fromBase64(defaultAttachment.key), + size: 100, + transitCdnKey: 'cdnKey', + transitCdnNumber: 2, + transitTierUploadTimestamp: Long.fromNumber(1234), + }), + }), + backupJob: undefined, + } + ); + }); + }); }); diff --git a/ts/test-electron/backup/helpers.preload.ts b/ts/test-electron/backup/helpers.preload.ts index f2c662f6a8..e2c0e893fb 100644 --- a/ts/test-electron/backup/helpers.preload.ts +++ b/ts/test-electron/backup/helpers.preload.ts @@ -228,13 +228,19 @@ export async function asymmetricRoundtripHarness( postSaveUpdates, }); - await backupsService.exportToDisk(targetOutputFile, options.backupLevel); + await backupsService.exportToDisk(targetOutputFile, { + type: 'remote', + level: options.backupLevel, + }); await updateConvoIdToTitle(); await clearData(); - await backupsService.importBackup(() => createReadStream(targetOutputFile)); + await backupsService.importBackup( + () => createReadStream(targetOutputFile), + { type: 'remote' } + ); const messagesFromDatabase = await DataReader._getAllMessages(); diff --git a/ts/test-electron/backup/integration_test.preload.ts b/ts/test-electron/backup/integration_test.preload.ts index 13e3d67287..15cd9455bc 100644 --- a/ts/test-electron/backup/integration_test.preload.ts +++ b/ts/test-electron/backup/integration_test.preload.ts @@ -14,10 +14,7 @@ import { assert } from 'chai'; import { clearData } from './helpers.preload.js'; import { loadAllAndReinitializeRedux } from '../../services/allLoaders.preload.js'; -import { - backupsService, - BackupType, -} from '../../services/backups/index.preload.js'; +import { backupsService } from '../../services/backups/index.preload.js'; import { initialize as initializeExpiringMessageService } from '../../services/expiringMessagesDeletion.preload.js'; import { MemoryStream } from '../../util/MemoryStream.node.js'; @@ -56,13 +53,13 @@ describe('backup/integration', () => { const expectedBuffer = await readFile(fullPath); await backupsService.importBackup(() => Readable.from([expectedBuffer]), { - backupType: BackupType.TestOnlyPlaintext, + type: 'cross-client-integration-test', }); - const { data: exported } = await backupsService.exportBackupData( - BackupLevel.Paid, - BackupType.TestOnlyPlaintext - ); + const { data: exported } = await backupsService.exportBackupData({ + type: 'cross-client-integration-test', + level: BackupLevel.Paid, + }); const actualStream = new MemoryStream(Buffer.from(exported)); const expectedStream = new MemoryStream(expectedBuffer); diff --git a/ts/textsecure/downloadAttachment.preload.ts b/ts/textsecure/downloadAttachment.preload.ts index a895db1d24..39ea8e5ef3 100644 --- a/ts/textsecure/downloadAttachment.preload.ts +++ b/ts/textsecure/downloadAttachment.preload.ts @@ -16,7 +16,7 @@ import { getAbsoluteDownloadsPath, getAbsoluteAttachmentPath, } from '../util/migrations.preload.js'; -import { hasRequiredInformationForBackup } from '../util/Attachment.std.js'; +import { hasRequiredInformationForRemoteBackup } from '../util/Attachment.std.js'; import { AttachmentSizeError, type AttachmentType, @@ -200,7 +200,7 @@ export async function downloadAttachment( } if (mediaTier === MediaTier.BACKUP) { strictAssert( - hasRequiredInformationForBackup(attachment), + hasRequiredInformationForRemoteBackup(attachment), `${logId}: attachment missing critical information for backup tier` ); } diff --git a/ts/types/Attachment.std.ts b/ts/types/Attachment.std.ts index 52273db6df..57788f2563 100644 --- a/ts/types/Attachment.std.ts +++ b/ts/types/Attachment.std.ts @@ -233,6 +233,10 @@ export type BackupableAttachmentType = WithRequiredProperties< AttachmentType, 'plaintextHash' | 'key' >; +export type AttachmentReadyForLocalBackup = WithRequiredProperties< + AttachmentType, + 'plaintextHash' | 'localKey' | 'path' +>; export type AttachmentDownloadableFromTransitTier = WithRequiredProperties< AttachmentType, diff --git a/ts/types/AttachmentBackup.std.ts b/ts/types/AttachmentBackup.std.ts index 1d96bfb4ad..65de176513 100644 --- a/ts/types/AttachmentBackup.std.ts +++ b/ts/types/AttachmentBackup.std.ts @@ -48,8 +48,6 @@ export type CoreAttachmentLocalBackupJobType = { mediaName: string; data: { path: string | null; - size: number; - localKey: string; }; backupsBaseDir: string; }; diff --git a/ts/util/Attachment.std.ts b/ts/util/Attachment.std.ts index 91f1d3f044..355890a753 100644 --- a/ts/util/Attachment.std.ts +++ b/ts/util/Attachment.std.ts @@ -12,6 +12,7 @@ import type { BackupableAttachmentType, AttachmentDownloadableFromTransitTier, LocallySavedAttachment, + AttachmentReadyForLocalBackup, } from '../types/Attachment.std.js'; import type { LoggerType } from '../types/Logging.std.js'; import { createLogger } from '../logging/log.std.js'; @@ -847,7 +848,7 @@ export function getCachedAttachmentBySignature( return undefined; } -export function hasRequiredInformationForBackup( +export function hasRequiredInformationForRemoteBackup( attachment: AttachmentType ): attachment is BackupableAttachmentType { return ( @@ -856,13 +857,23 @@ export function hasRequiredInformationForBackup( ); } +export function hasRequiredInformationForLocalBackup( + attachment: AttachmentType +): attachment is AttachmentReadyForLocalBackup { + return ( + isValidAttachmentKey(attachment.localKey) && + isValidPlaintextHash(attachment.plaintextHash) && + Boolean(attachment.path) + ); +} + export function wasImportedFromLocalBackup( attachment: AttachmentType ): attachment is BackupableAttachmentType { return ( - hasRequiredInformationForBackup(attachment) && - Boolean(attachment.localBackupPath) && - isValidAttachmentKey(attachment.localKey) + isValidPlaintextHash(attachment.plaintextHash) && + isValidAttachmentKey(attachment.localKey) && + Boolean(attachment.localBackupPath) ); } @@ -900,7 +911,7 @@ export function shouldAttachmentEndUpInRemoteBackup({ attachment: AttachmentType; hasMediaBackups: boolean; }): boolean { - return hasMediaBackups && hasRequiredInformationForBackup(attachment); + return hasMediaBackups && hasRequiredInformationForRemoteBackup(attachment); } export function isDownloadable(attachment: AttachmentType): boolean { diff --git a/ts/util/downloadAttachment.preload.ts b/ts/util/downloadAttachment.preload.ts index 993cc61878..5af5bc6413 100644 --- a/ts/util/downloadAttachment.preload.ts +++ b/ts/util/downloadAttachment.preload.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: AGPL-3.0-only import { ErrorCode, LibSignalErrorBase } from '@signalapp/libsignal-client'; import { - hasRequiredInformationForBackup, + hasRequiredInformationForRemoteBackup, wasImportedFromLocalBackup, } from './Attachment.std.js'; import { @@ -58,7 +58,7 @@ export async function downloadAttachment({ variant !== AttachmentVariant.Default ? `[${variant}]` : ''; const logId = `${_logId}${variantForLogging}`; - const isBackupable = hasRequiredInformationForBackup(attachment); + const isBackupable = hasRequiredInformationForRemoteBackup(attachment); const mightBeOnBackupTierNow = isBackupable && hasMediaBackups; const mightBeOnBackupTierInTheFuture = isBackupable;