diff --git a/_locales/en/messages.json b/_locales/en/messages.json index 10fe79e63a..02d8dad7e9 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -6652,6 +6652,26 @@ "messageformat": "Internal", "description": "Button to switch the settings view to control internal configuration" }, + "icu:Preferences__internal__local-backups": { + "messageformat": "Local backups", + "description": "Text header for internal local backup tools" + }, + "icu:Preferences__internal__export-local-backup": { + "messageformat": "Export…", + "description": "Button to run internal local backup export tool" + }, + "icu:Preferences__internal__export-local-backup--description": { + "messageformat": "Export local encrypted backup to a folder and validate it", + "description": "Description of the internal local backup export tool" + }, + "icu:Preferences__internal__import-local-backup": { + "messageformat": "Import…", + "description": "Button to run internal local backup import tool" + }, + "icu:Preferences__internal__import-local-backup--description": { + "messageformat": "Stage a local encrypted backup for import on link", + "description": "Description of the internal local backup export tool" + }, "icu:Preferences__internal__validate-backup--description": { "messageformat": "Export encrypted backup to memory and run validation suite on it", "description": "Description of the internal backup validation tool" diff --git a/app/main.ts b/app/main.ts index 44b8c4cc85..802bc412db 100644 --- a/app/main.ts +++ b/app/main.ts @@ -3110,31 +3110,50 @@ ipc.handle('show-save-dialog', async (_event, { defaultPath }) => { return { canceled: false, filePath: finalFilePath }; }); -ipc.handle('show-save-multi-dialog', async _event => { - if (!mainWindow) { - getLogger().warn('show-save-multi-dialog: no main window'); +ipc.handle( + 'show-open-folder-dialog', + async ( + _event, + { useMainWindow }: { useMainWindow: boolean } = { useMainWindow: false } + ) => { + let canceled: boolean; + let selectedDirPaths: ReadonlyArray; - return { canceled: true }; - } - const { canceled, filePaths: selectedDirPaths } = await dialog.showOpenDialog( - mainWindow, - { - defaultPath: app.getPath('downloads'), - properties: ['openDirectory', 'createDirectory'], + if (useMainWindow) { + if (!mainWindow) { + getLogger().warn('show-open-folder-dialog: no main window'); + return { canceled: true }; + } + + ({ canceled, filePaths: selectedDirPaths } = await dialog.showOpenDialog( + mainWindow, + { + defaultPath: app.getPath('downloads'), + properties: ['openDirectory', 'createDirectory'], + } + )); + } else { + ({ canceled, filePaths: selectedDirPaths } = await dialog.showOpenDialog({ + defaultPath: app.getPath('downloads'), + properties: ['openDirectory', 'createDirectory'], + })); } - ); - if (canceled || selectedDirPaths.length === 0) { - return { canceled: true }; + + if (canceled || selectedDirPaths.length === 0) { + return { canceled: true }; + } + + if (selectedDirPaths.length > 1) { + getLogger().warn( + 'show-open-folder-dialog: multiple directories selected' + ); + + return { canceled: true }; + } + + return { canceled: false, dirPath: selectedDirPaths[0] }; } - - if (selectedDirPaths.length > 1) { - getLogger().warn('show-save-multi-dialog: multiple directories selected'); - - return { canceled: true }; - } - - return { canceled: false, dirPath: selectedDirPaths[0] }; -}); +); ipc.handle('executeMenuRole', async ({ sender }, untypedRole) => { const role = untypedRole as MenuItemConstructorOptions['role']; diff --git a/protos/Backups.proto b/protos/Backups.proto index 01057d3630..9cde938185 100644 --- a/protos/Backups.proto +++ b/protos/Backups.proto @@ -725,11 +725,30 @@ message FilePointer { message InvalidAttachmentLocator { } + // References attachments in a local encrypted backup. + // Importers should first attempt to read the file from the local backup, + // and on failure fallback to backup and transit cdn if possible. + message LocalLocator { + string mediaName = 1; + // Separate key used to encrypt this file for the local backup. + // Generally required. Missing field indicates attachment was not + // available locally when the backup was generated, but remote + // backup or transit info was available. + optional bytes localKey = 2; + bytes remoteKey = 3; + bytes remoteDigest = 4; + uint32 size = 5; + optional uint32 backupCdnNumber = 6; + optional string transitCdnKey = 7; + optional uint32 transitCdnNumber = 8; + } + // If unset, importers should consider it to be an InvalidAttachmentLocator without throwing an error. oneof locator { BackupLocator backupLocator = 1; AttachmentLocator attachmentLocator = 2; InvalidAttachmentLocator invalidAttachmentLocator = 3; + LocalLocator localLocator = 12; } optional string contentType = 4; diff --git a/protos/LocalBackup.proto b/protos/LocalBackup.proto new file mode 100644 index 0000000000..5381d79071 --- /dev/null +++ b/protos/LocalBackup.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package signal.backup.local; + +option java_package = "org.thoughtcrime.securesms.backup.v2.local.proto"; +option swift_prefix = "LocalBackupProto_"; + +message Metadata { + message EncryptedBackupId { + bytes iv = 1; // 12 bytes, randomly generated + bytes encryptedId = 2; // AES-256-CTR, key = local backup metadata key, message = backup ID bytes + // local backup metadata key = hkdf(input: K_B, info: UTF8("20241011_SIGNAL_LOCAL_BACKUP_METADATA_KEY"), length: 32) + // No hash of the ID; if it's decrypted incorrectly, the main backup will fail to decrypt anyway. + } + + uint32 version = 1; + EncryptedBackupId backupId = 2; // used to decrypt the backup file knowing only the Account Entropy Pool +} + +message FilesFrame { + oneof item { + string mediaName = 1; + } +} diff --git a/ts/CI.ts b/ts/CI.ts index ed9460c5aa..237d003490 100644 --- a/ts/CI.ts +++ b/ts/CI.ts @@ -43,6 +43,8 @@ export type CIType = { ) => unknown; openSignalRoute(url: string): Promise; migrateAllMessages(): Promise; + exportLocalBackup(backupsBaseDir: string): Promise; + stageLocalBackupForImport(snapshotDir: string): Promise; uploadBackup(): Promise; unlink: () => void; print: (...args: ReadonlyArray) => void; @@ -193,6 +195,20 @@ export function getCI({ document.body.removeChild(a); } + async function exportLocalBackup(backupsBaseDir: string): Promise { + const { snapshotDir } = + await backupsService.exportLocalBackup(backupsBaseDir); + return snapshotDir; + } + + async function stageLocalBackupForImport(snapshotDir: string): Promise { + const { error } = + await backupsService.stageLocalBackupForImport(snapshotDir); + if (error) { + throw error; + } + } + async function uploadBackup() { await backupsService.upload(); await AttachmentBackupManager.waitForIdle(); @@ -237,6 +253,8 @@ export function getCI({ waitForEvent, openSignalRoute, migrateAllMessages, + exportLocalBackup, + stageLocalBackupForImport, uploadBackup, unlink, getPendingEventCount, diff --git a/ts/background.ts b/ts/background.ts index 4958161fe9..64a16490a1 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -214,6 +214,7 @@ import { MessageModel } from './models/messages'; import { waitForEvent } from './shims/events'; import { sendSyncRequests } from './textsecure/syncRequests'; import { handleServerAlerts } from './util/handleServerAlerts'; +import { isLocalBackupsEnabled } from './util/isLocalBackupsEnabled'; export function isOverHourIntoPast(timestamp: number): boolean { return isNumber(timestamp) && isOlderThan(timestamp, HOUR); @@ -1758,7 +1759,7 @@ export async function startApp(): Promise { hasSentSyncRequests = true; } - // 4. Download (or resume download) of link & sync backup + // 4. Download (or resume download) of link & sync backup or local backup const { wasBackupImported } = await maybeDownloadAndImportBackup(); log.info(logId, { wasBackupImported, @@ -1836,20 +1837,29 @@ export async function startApp(): Promise { wasBackupImported: boolean; }> { const backupDownloadPath = window.storage.get('backupDownloadPath'); - if (backupDownloadPath) { + const isLocalBackupAvailable = + backupsService.isLocalBackupStaged() && isLocalBackupsEnabled(); + + if (isLocalBackupAvailable || backupDownloadPath) { tapToViewMessagesDeletionService.pause(); // Download backup before enabling request handler and storage service try { - const { wasBackupImported } = await backupsService.downloadAndImport({ - onProgress: (backupStep, currentBytes, totalBytes) => { - window.reduxActions.installer.updateBackupImportProgress({ - backupStep, - currentBytes, - totalBytes, - }); - }, - }); + let wasBackupImported = false; + if (isLocalBackupAvailable) { + await backupsService.importLocalBackup(); + wasBackupImported = true; + } else { + ({ wasBackupImported } = await backupsService.downloadAndImport({ + onProgress: (backupStep, currentBytes, totalBytes) => { + window.reduxActions.installer.updateBackupImportProgress({ + backupStep, + currentBytes, + totalBytes, + }); + }, + })); + } log.info('afterAppStart: backup download attempt completed, resolving'); backupReady.resolve({ wasBackupImported }); diff --git a/ts/components/Preferences.stories.tsx b/ts/components/Preferences.stories.tsx index 5acbc90e6b..a33e98271a 100644 --- a/ts/components/Preferences.stories.tsx +++ b/ts/components/Preferences.stories.tsx @@ -43,6 +43,28 @@ const availableSpeakers = [ }, ]; +const validateBackupResult = { + totalBytes: 100, + duration: 10000, + stats: { + adHocCalls: 1, + callLinks: 2, + conversations: 3, + chats: 4, + distributionLists: 5, + messages: 6, + notificationProfiles: 2, + skippedMessages: 7, + stickerPacks: 8, + fixedDirectMessages: 9, + }, +}; + +const exportLocalBackupResult = { + ...validateBackupResult, + snapshotDir: '/home/signaluser/SignalBackups/signal-backup-1745618069169', +}; + export default { title: 'Components/Preferences', component: Preferences, @@ -138,6 +160,18 @@ export default { doDeleteAllData: action('doDeleteAllData'), doneRendering: action('doneRendering'), editCustomColor: action('editCustomColor'), + exportLocalBackup: async () => { + return { + result: exportLocalBackupResult, + }; + }, + importLocalBackup: async () => { + return { + success: true, + error: undefined, + snapshotDir: exportLocalBackupResult.snapshotDir, + }; + }, makeSyncRequest: action('makeSyncRequest'), onAudioNotificationsChange: action('onAudioNotificationsChange'), onAutoConvertEmojiChange: action('onAutoConvertEmojiChange'), @@ -192,22 +226,7 @@ export default { ), validateBackup: async () => { return { - result: { - totalBytes: 100, - duration: 10000, - stats: { - adHocCalls: 1, - callLinks: 2, - conversations: 3, - chats: 4, - distributionLists: 5, - messages: 6, - notificationProfiles: 2, - skippedMessages: 7, - stickerPacks: 8, - fixedDirectMessages: 9, - }, - }, + result: validateBackupResult, }; }, } satisfies PropsType, diff --git a/ts/components/Preferences.tsx b/ts/components/Preferences.tsx index c40397a040..857ac814b9 100644 --- a/ts/components/Preferences.tsx +++ b/ts/components/Preferences.tsx @@ -74,6 +74,7 @@ import { import { PreferencesBackups } from './PreferencesBackups'; import { PreferencesInternal } from './PreferencesInternal'; import { FunEmojiLocalizationProvider } from './fun/FunEmojiLocalizationProvider'; +import type { ValidateLocalBackupStructureResultType } from '../services/backups/util/localBackup'; type CheckboxChangeHandlerType = (value: boolean) => unknown; type SelectChangeHandlerType = (value: T) => unknown; @@ -157,9 +158,11 @@ type PropsFunctionType = { doDeleteAllData: () => unknown; doneRendering: () => unknown; editCustomColor: (colorId: string, color: CustomColorType) => unknown; + exportLocalBackup: () => Promise; getConversationsWithCustomColor: ( colorId: string ) => Promise>; + importLocalBackup: () => Promise; makeSyncRequest: () => unknown; refreshCloudBackupStatus: () => void; refreshBackupSubscriptionStatus: () => void; @@ -286,6 +289,7 @@ export function Preferences({ doneRendering, editCustomColor, emojiSkinToneDefault, + exportLocalBackup, getConversationsWithCustomColor, hasAudioNotifications, hasAutoConvertEmoji, @@ -311,6 +315,7 @@ export function Preferences({ hasTextFormatting, hasTypingIndicators, i18n, + importLocalBackup, initialPage = Page.General, initialSpellCheckSetting, isAutoDownloadUpdatesSupported, @@ -1736,7 +1741,12 @@ export function Preferences({ ); } else if (page === Page.Internal) { settings = ( - + ); } diff --git a/ts/components/PreferencesInternal.tsx b/ts/components/PreferencesInternal.tsx index d5dff2e928..8cfe39e024 100644 --- a/ts/components/PreferencesInternal.tsx +++ b/ts/components/PreferencesInternal.tsx @@ -7,17 +7,30 @@ import { toLogFormat } from '../types/errors'; import { formatFileSize } from '../util/formatFileSize'; import { SECOND } from '../util/durations'; import type { ValidationResultType as BackupValidationResultType } from '../services/backups'; +import type { ValidateLocalBackupStructureResultType } from '../services/backups/util/localBackup'; import { SettingsRow, SettingsControl } from './PreferencesUtil'; import { Button, ButtonVariant } from './Button'; import { Spinner } from './Spinner'; export function PreferencesInternal({ i18n, + exportLocalBackup: doExportLocalBackup, + importLocalBackup: doImportLocalBackup, validateBackup: doValidateBackup, }: { i18n: LocalizerType; + exportLocalBackup: () => Promise; + importLocalBackup: () => Promise; validateBackup: () => Promise; }): JSX.Element { + const [isExportPending, setIsExportPending] = useState(false); + const [exportResult, setExportResult] = useState< + BackupValidationResultType | undefined + >(); + const [importResult, setImportResult] = useState< + ValidateLocalBackupStructureResultType | undefined + >(); + const [isValidationPending, setIsValidationPending] = useState(false); const [validationResult, setValidationResult] = useState< BackupValidationResultType | undefined @@ -35,34 +48,110 @@ export function PreferencesInternal({ } }, [doValidateBackup]); - let validationElem: JSX.Element | undefined; - if (validationResult != null) { - if ('result' in validationResult) { - const { - result: { totalBytes, stats, duration }, - } = validationResult; + const renderValidationResult = useCallback( + ( + backupResult: BackupValidationResultType | undefined + ): JSX.Element | undefined => { + if (backupResult == null) { + return; + } - validationElem = ( -
-

File size: {formatFileSize(totalBytes)}

-

Duration: {Math.round(duration / SECOND)}s

- 
-            {JSON.stringify(stats, null, 2)}
-
-
- ); - } else { - const { error } = validationResult; + if ('result' in backupResult) { + const { + result: { totalBytes, stats, duration }, + } = backupResult; - validationElem = ( + let snapshotDirEl: JSX.Element | undefined; + if ('snapshotDir' in backupResult.result) { + snapshotDirEl = ( +

+ Backup path: + 

+                {backupResult.result.snapshotDir}
+
+

+ ); + } + + return ( +
+ {snapshotDirEl} +

Main file size: {formatFileSize(totalBytes)}

+

Duration: {Math.round(duration / SECOND)}s

+ 
+              {JSON.stringify(stats, null, 2)}
+
+
+ ); + } + + const { error } = backupResult; + + return ( 
             {error}
); + }, + [] + ); + + const exportLocalBackup = useCallback(async () => { + setIsExportPending(true); + setExportResult(undefined); + try { + setExportResult(await doExportLocalBackup()); + } catch (error) { + setExportResult({ error: toLogFormat(error) }); + } finally { + setIsExportPending(false); } - } + }, [doExportLocalBackup]); + + const importLocalBackup = useCallback(async () => { + setImportResult(undefined); + try { + setImportResult(await doImportLocalBackup()); + } catch (error) { + setImportResult({ + success: false, + error: toLogFormat(error), + snapshotDir: undefined, + }); + } + }, [doImportLocalBackup]); + + const renderImportResult = useCallback( + ( + didImportResult: ValidateLocalBackupStructureResultType | undefined + ): JSX.Element | undefined => { + if (didImportResult == null) { + return; + } + + const { success, error, snapshotDir } = didImportResult; + if (success) { + return ( +
+ 
+              {`Staged: ${snapshotDir}\n\nPlease link to finish import.`}
+
+
+ ); + } + + return ( +
+ 
+            {`Failed: ${error}`}
+
+
+ ); + }, + [] + ); return ( <> @@ -93,7 +182,49 @@ export function PreferencesInternal({ } /> - {validationElem} + {renderValidationResult(validationResult)} + + + + + {isExportPending ? ( + + ) : ( + i18n('icu:Preferences__internal__export-local-backup') + )} + + } + /> + + {renderValidationResult(exportResult)} + + + {i18n('icu:Preferences__internal__import-local-backup')} + + } + /> + + {renderImportResult(importResult)} ); diff --git a/ts/jobs/AttachmentDownloadManager.ts b/ts/jobs/AttachmentDownloadManager.ts index f3eb5b47dc..85989a90b2 100644 --- a/ts/jobs/AttachmentDownloadManager.ts +++ b/ts/jobs/AttachmentDownloadManager.ts @@ -24,6 +24,7 @@ import { AttachmentVariant, AttachmentPermanentlyUndownloadableError, mightBeOnBackupTier, + mightBeInLocalBackup, } from '../types/Attachment'; import { type ReadonlyMessageAttributesType } from '../model-types.d'; import { getMessageById } from '../messages/getMessageById'; @@ -285,9 +286,10 @@ export class AttachmentDownloadManager extends JobManager { const downloadedThumbnail = await dependencies.downloadAttachment({ attachment, diff --git a/ts/jobs/AttachmentLocalBackupManager.ts b/ts/jobs/AttachmentLocalBackupManager.ts new file mode 100644 index 0000000000..311931072a --- /dev/null +++ b/ts/jobs/AttachmentLocalBackupManager.ts @@ -0,0 +1,279 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only +/* eslint-disable max-classes-per-file */ + +import { existsSync } from 'node:fs'; +import { PassThrough } from 'node:stream'; +import { constants as FS_CONSTANTS, copyFile, mkdir } from 'fs/promises'; + +import * as durations from '../util/durations'; +import * as log from '../logging/log'; + +import * as Errors from '../types/errors'; +import { redactGenericText } from '../util/privacy'; +import { + JobManager, + type JobManagerParamsType, + type JobManagerJobResultType, +} from './JobManager'; +import { type BackupsService, backupsService } from '../services/backups'; +import { decryptAttachmentV2ToSink } from '../AttachmentCrypto'; +import { + type AttachmentLocalBackupJobType, + type CoreAttachmentLocalBackupJobType, +} from '../types/AttachmentBackup'; +import { isInCall as isInCallSelector } from '../state/selectors/calling'; +import { encryptAndUploadAttachment } from '../util/uploadAttachment'; +import type { WebAPIType } from '../textsecure/WebAPI'; +import type { LocallySavedAttachment } from '../types/Attachment'; +import { + getLocalBackupDirectoryForMediaName, + getLocalBackupPathForMediaName, +} from '../services/backups/util/localBackup'; + +const MAX_CONCURRENT_JOBS = 3; +const RETRY_CONFIG = { + maxAttempts: 3, + backoffConfig: { + // 1 minute, 5 minutes, 25 minutes, every hour + multiplier: 3, + firstBackoffs: [10 * durations.SECOND], + maxBackoffTime: durations.MINUTE, + }, +}; + +export class AttachmentLocalBackupManager extends JobManager { + static #instance: AttachmentLocalBackupManager | undefined; + readonly #jobsByMediaName = new Map(); + + static defaultParams: JobManagerParamsType = + { + markAllJobsInactive: AttachmentLocalBackupManager.markAllJobsInactive, + saveJob: AttachmentLocalBackupManager.saveJob, + removeJob: AttachmentLocalBackupManager.removeJob, + getNextJobs: AttachmentLocalBackupManager.getNextJobs, + runJob: runAttachmentBackupJob, + shouldHoldOffOnStartingQueuedJobs: () => { + const reduxState = window.reduxStore?.getState(); + if (reduxState) { + return isInCallSelector(reduxState); + } + return false; + }, + getJobId, + getJobIdForLogging, + getRetryConfig: () => RETRY_CONFIG, + maxConcurrentJobs: MAX_CONCURRENT_JOBS, + }; + + override logPrefix = 'AttachmentLocalBackupManager'; + + static get instance(): AttachmentLocalBackupManager { + if (!AttachmentLocalBackupManager.#instance) { + AttachmentLocalBackupManager.#instance = new AttachmentLocalBackupManager( + AttachmentLocalBackupManager.defaultParams + ); + } + return AttachmentLocalBackupManager.#instance; + } + + static get jobs(): Map { + return AttachmentLocalBackupManager.instance.#jobsByMediaName; + } + + static async start(): Promise { + log.info('AttachmentLocalBackupManager/starting'); + await AttachmentLocalBackupManager.instance.start(); + } + + static async stop(): Promise { + log.info('AttachmentLocalBackupManager/stopping'); + return AttachmentLocalBackupManager.#instance?.stop(); + } + + static async addJob(newJob: CoreAttachmentLocalBackupJobType): Promise { + return AttachmentLocalBackupManager.instance.addJob(newJob); + } + + static async waitForIdle(): Promise { + return AttachmentLocalBackupManager.instance.waitForIdle(); + } + + static async markAllJobsInactive(): Promise { + for (const [mediaName, job] of AttachmentLocalBackupManager.jobs) { + AttachmentLocalBackupManager.jobs.set(mediaName, { + ...job, + active: false, + }); + } + } + + static async saveJob(job: AttachmentLocalBackupJobType): Promise { + AttachmentLocalBackupManager.jobs.set(job.mediaName, job); + } + + static async removeJob( + job: Pick + ): Promise { + AttachmentLocalBackupManager.jobs.delete(job.mediaName); + } + + static clearAllJobs(): void { + AttachmentLocalBackupManager.jobs.clear(); + } + + static async getNextJobs({ + limit, + timestamp, + }: { + limit: number; + timestamp: number; + }): Promise> { + let countRemaining = limit; + const nextJobs: Array = []; + for (const job of AttachmentLocalBackupManager.jobs.values()) { + if (job.active || (job.retryAfter && job.retryAfter > timestamp)) { + continue; + } + + nextJobs.push(job); + countRemaining -= 1; + if (countRemaining <= 0) { + break; + } + } + return nextJobs; + } +} + +function getJobId(job: CoreAttachmentLocalBackupJobType): string { + return job.mediaName; +} + +function getJobIdForLogging(job: CoreAttachmentLocalBackupJobType): string { + return `${redactGenericText(job.mediaName)}.${job.type}`; +} + +/** + * Backup-specific methods + */ +class AttachmentPermanentlyMissingError extends Error {} + +type RunAttachmentBackupJobDependenciesType = { + getAbsoluteAttachmentPath: typeof window.Signal.Migrations.getAbsoluteAttachmentPath; + backupMediaBatch?: WebAPIType['backupMediaBatch']; + backupsService: BackupsService; + encryptAndUploadAttachment: typeof encryptAndUploadAttachment; + decryptAttachmentV2ToSink: typeof decryptAttachmentV2ToSink; +}; + +export async function runAttachmentBackupJob( + job: AttachmentLocalBackupJobType, + _options: { + isLastAttempt: boolean; + abortSignal: AbortSignal; + }, + dependencies: RunAttachmentBackupJobDependenciesType = { + getAbsoluteAttachmentPath: + window.Signal.Migrations.getAbsoluteAttachmentPath, + backupsService, + backupMediaBatch: window.textsecure.server?.backupMediaBatch, + encryptAndUploadAttachment, + decryptAttachmentV2ToSink, + } +): Promise> { + const jobIdForLogging = getJobIdForLogging(job); + const logId = `AttachmentLocalBackupManager/runAttachmentBackupJob/${jobIdForLogging}`; + try { + await runAttachmentBackupJobInner(job, dependencies); + return { status: 'finished' }; + } catch (error) { + log.error( + `${logId}: Failed to backup attachment, attempt ${job.attempts}`, + Errors.toLogFormat(error) + ); + + if (error instanceof AttachmentPermanentlyMissingError) { + log.error(`${logId}: Attachment unable to be found, giving up on job`); + return { status: 'finished' }; + } + + return { status: 'retry' }; + } +} + +async function runAttachmentBackupJobInner( + job: AttachmentLocalBackupJobType, + dependencies: RunAttachmentBackupJobDependenciesType +): Promise { + const jobIdForLogging = getJobIdForLogging(job); + const logId = `AttachmentLocalBackupManager.runAttachmentBackupJobInner(${jobIdForLogging})`; + + log.info(`${logId}: starting`); + + const { backupsBaseDir, mediaName } = job; + const { contentType, digest, iv, keys, localKey, path, size } = job.data; + + if (!path) { + throw new AttachmentPermanentlyMissingError('No path property'); + } + + const absolutePath = dependencies.getAbsoluteAttachmentPath(path); + if (!existsSync(absolutePath)) { + throw new AttachmentPermanentlyMissingError('No file at provided path'); + } + + if (!localKey) { + throw new Error('No localKey property, required for test decryption'); + } + + const localBackupFileDir = getLocalBackupDirectoryForMediaName({ + backupsBaseDir, + mediaName, + }); + await mkdir(localBackupFileDir, { recursive: true }); + + const localBackupFilePath = getLocalBackupPathForMediaName({ + backupsBaseDir, + mediaName, + }); + + const attachment: LocallySavedAttachment = { + path, + iv, + key: keys, + localKey, + digest, + contentType, + size, + }; + + // TODO: Add check in local FS to prevent double backup + + // File is already encrypted with localKey, so we just have to copy it to the backup dir + const attachmentPath = window.Signal.Migrations.getAbsoluteAttachmentPath( + attachment.path + ); + + // Set COPYFILE_FICLONE for Copy on Write (OS dependent, gracefully falls back to copy) + await copyFile( + attachmentPath, + localBackupFilePath, + FS_CONSTANTS.COPYFILE_FICLONE + ); + + // TODO: Optimize this check -- it can be expensive to test decrypt on every export + log.info(`${logId}: Verifying file restored from local backup`); + const sink = new PassThrough(); + sink.resume(); + await decryptAttachmentV2ToSink( + { + ciphertextPath: localBackupFilePath, + idForLogging: 'attachments/readAndDecryptDataFromDisk', + keysBase64: localKey, + size, + type: 'local', + }, + sink + ); +} diff --git a/ts/main/settingsChannel.ts b/ts/main/settingsChannel.ts index f7293d8548..be40789087 100644 --- a/ts/main/settingsChannel.ts +++ b/ts/main/settingsChannel.ts @@ -68,6 +68,8 @@ export class SettingsChannel extends EventEmitter { this.#installCallback('syncRequest'); this.#installCallback('setEmojiSkinToneDefault'); this.#installCallback('getEmojiSkinToneDefault'); + this.#installCallback('exportLocalBackup'); + this.#installCallback('importLocalBackup'); this.#installCallback('validateBackup'); // Backups diff --git a/ts/services/backups/constants.ts b/ts/services/backups/constants.ts index ea64bf8854..f5ba4ce8c3 100644 --- a/ts/services/backups/constants.ts +++ b/ts/services/backups/constants.ts @@ -6,6 +6,10 @@ import type { ConversationColorType } from '../../types/Colors'; export const BACKUP_VERSION = 1; +export const LOCAL_BACKUP_VERSION = 1; + +export const LOCAL_BACKUP_BACKUP_ID_IV_LENGTH = 16; + const { WallpaperPreset } = Backups.ChatStyle; // See https://github.com/signalapp/Signal-Android-Private/blob/4a41e9f9a1ed0aba7cae0e0dc4dbcac50fddc469/app/src/main/java/org/thoughtcrime/securesms/conversation/colors/ChatColorsMapper.kt#L32 diff --git a/ts/services/backups/crypto.ts b/ts/services/backups/crypto.ts index 02ccfd059d..2b77b39572 100644 --- a/ts/services/backups/crypto.ts +++ b/ts/services/backups/crypto.ts @@ -125,3 +125,12 @@ export function deriveBackupThumbnailTransitKeyMaterial( ), }; } + +export function getBackupId(): Uint8Array { + const aci = window.storage.user.getCheckedAci(); + return getBackupKey().deriveBackupId(toAciObject(aci)); +} + +export function getLocalBackupMetadataKey(): Uint8Array { + return getBackupKey().deriveLocalBackupMetadataKey(); +} diff --git a/ts/services/backups/export.ts b/ts/services/backups/export.ts index 17f64e22cc..f2d5fe072f 100644 --- a/ts/services/backups/export.ts +++ b/ts/services/backups/export.ts @@ -4,6 +4,7 @@ import Long from 'long'; import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client'; import type { BackupLevel } from '@signalapp/libsignal-client/zkgroup'; +import { dirname } from 'path'; import pMap from 'p-map'; import pTimeout from 'p-timeout'; import { Readable } from 'stream'; @@ -125,11 +126,13 @@ import { } from '../../types/Attachment'; import { getFilePointerForAttachment, + getLocalBackupFilePointerForAttachment, maybeGetBackupJobForAttachmentAndFilePointer, } from './util/filePointers'; import { getBackupMediaRootKey } from './crypto'; import type { CoreAttachmentBackupJobType } from '../../types/AttachmentBackup'; import { AttachmentBackupManager } from '../../jobs/AttachmentBackupManager'; +import { AttachmentLocalBackupManager } from '../../jobs/AttachmentLocalBackupManager'; import { getBackupCdnInfo } from './util/mediaId'; import { calculateExpirationTimestamp } from '../../util/expirationTimer'; import { ReadStatus } from '../../messages/MessageReadStatus'; @@ -177,6 +180,7 @@ type ToChatItemOptionsType = Readonly<{ aboutMe: AboutMe; callHistoryByCallId: Record; backupLevel: BackupLevel; + isLocalBackup: boolean; }>; type NonBubbleOptionsType = Pick< @@ -227,6 +231,7 @@ export class BackupExportStream extends Readable { readonly #serviceIdToRecipientId = new Map(); readonly #e164ToRecipientId = new Map(); readonly #roomIdToRecipientId = new Map(); + readonly #mediaNamesToFilePointers = new Map(); readonly #stats: StatsType = { adHocCalls: 0, callLinks: 0, @@ -253,43 +258,82 @@ export class BackupExportStream extends Readable { super(); } - public run(backupLevel: BackupLevel): void { + public run( + backupLevel: BackupLevel, + localBackupSnapshotDir: string | undefined = undefined + ): void { + const localBackupsBaseDir = localBackupSnapshotDir + ? dirname(localBackupSnapshotDir) + : undefined; + const isLocalBackup = localBackupsBaseDir != null; drop( (async () => { log.info('BackupExportStream: starting...'); drop(AttachmentBackupManager.stop()); + drop(AttachmentLocalBackupManager.stop()); log.info('BackupExportStream: message migration starting...'); await migrateAllMessages(); await pauseWriteAccess(); try { - await this.#unsafeRun(backupLevel); + await this.#unsafeRun(backupLevel, isLocalBackup); } catch (error) { this.emit('error', error); } finally { await resumeWriteAccess(); - // TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction - await DataWriter.clearAllAttachmentBackupJobs(); - if (this.backupType !== BackupType.TestOnlyPlaintext) { - await Promise.all( - this.#attachmentBackupJobs.map(job => - AttachmentBackupManager.addJobAndMaybeThumbnailJob(job) - ) + if (isLocalBackup) { + log.info( + `BackupExportStream: Adding ${this.#attachmentBackupJobs.length} jobs for AttachmentLocalBackupManager` ); - drop(AttachmentBackupManager.start()); + AttachmentLocalBackupManager.clearAllJobs(); + await Promise.all( + this.#attachmentBackupJobs.map(job => { + if (job.type === 'thumbnail') { + log.error( + "BackupExportStream: Can't backup thumbnails to local backup, skipping" + ); + return Promise.resolve(); + } + + return AttachmentLocalBackupManager.addJob({ + ...job, + backupsBaseDir: localBackupsBaseDir, + }); + }) + ); + drop(AttachmentLocalBackupManager.start()); + } else { + // TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction + await DataWriter.clearAllAttachmentBackupJobs(); + if (this.backupType !== BackupType.TestOnlyPlaintext) { + await Promise.all( + this.#attachmentBackupJobs.map(job => + AttachmentBackupManager.addJobAndMaybeThumbnailJob(job) + ) + ); + drop(AttachmentBackupManager.start()); + } } + log.info('BackupExportStream: finished'); } })() ); } + public getMediaNamesIterator(): MapIterator { + return this.#mediaNamesToFilePointers.keys(); + } + public getStats(): Readonly { return this.#stats; } - async #unsafeRun(backupLevel: BackupLevel): Promise { + async #unsafeRun( + backupLevel: BackupLevel, + isLocalBackup: boolean + ): Promise { this.#ourConversation = window.ConversationController.getOurConversationOrThrow().attributes; this.push( @@ -663,6 +707,7 @@ export class BackupExportStream extends Readable { aboutMe, callHistoryByCallId, backupLevel, + isLocalBackup, }), { concurrency: MAX_CONCURRENCY } ); @@ -1093,7 +1138,12 @@ export class BackupExportStream extends Readable { async #toChatItem( message: MessageAttributesType, - { aboutMe, callHistoryByCallId, backupLevel }: ToChatItemOptionsType + { + aboutMe, + callHistoryByCallId, + backupLevel, + isLocalBackup, + }: ToChatItemOptionsType ): Promise { const conversation = window.ConversationController.get( message.conversationId @@ -1253,6 +1303,7 @@ export class BackupExportStream extends Readable { result.viewOnceMessage = await this.#toViewOnceMessage({ message, backupLevel, + isLocalBackup, }); } else if (message.deletedForEveryone) { result.remoteDeletedMessage = {}; @@ -1314,6 +1365,7 @@ export class BackupExportStream extends Readable { ? await this.#processAttachment({ attachment: contactDetails.avatar.avatar, backupLevel, + isLocalBackup, messageReceivedAt: message.received_at, }) : undefined, @@ -1335,6 +1387,7 @@ export class BackupExportStream extends Readable { ? await this.#processAttachment({ attachment: sticker.data, backupLevel, + isLocalBackup, messageReceivedAt: message.received_at, }) : undefined; @@ -1378,23 +1431,27 @@ export class BackupExportStream extends Readable { result.directStoryReplyMessage = await this.#toDirectStoryReplyMessage({ message, backupLevel, + isLocalBackup, }); result.revisions = await this.#toChatItemRevisions( result, message, - backupLevel + backupLevel, + isLocalBackup ); } else { result.standardMessage = await this.#toStandardMessage({ message, backupLevel, + isLocalBackup, }); result.revisions = await this.#toChatItemRevisions( result, message, - backupLevel + backupLevel, + isLocalBackup ); } @@ -2297,9 +2354,11 @@ export class BackupExportStream extends Readable { async #toQuote({ message, backupLevel, + isLocalBackup, }: { message: Pick; backupLevel: BackupLevel; + isLocalBackup: boolean; }): Promise { const { quote } = message; if (!quote) { @@ -2359,6 +2418,7 @@ export class BackupExportStream extends Readable { attachment: attachment.thumbnail, backupLevel, message, + isLocalBackup, }) : undefined, }; @@ -2417,16 +2477,19 @@ export class BackupExportStream extends Readable { attachment, backupLevel, message, + isLocalBackup, }: { attachment: AttachmentType; backupLevel: BackupLevel; message: Pick; + isLocalBackup: boolean; }): Promise { const { clientUuid } = attachment; const filePointer = await this.#processAttachment({ attachment, backupLevel, messageReceivedAt: message.received_at, + isLocalBackup, }); return new Backups.MessageAttachment({ @@ -2440,18 +2503,51 @@ export class BackupExportStream extends Readable { async #processAttachment({ attachment, backupLevel, + isLocalBackup, messageReceivedAt, }: { attachment: AttachmentType; backupLevel: BackupLevel; + isLocalBackup: boolean; messageReceivedAt: number; }): Promise { - const { filePointer, updatedAttachment } = - await getFilePointerForAttachment({ - attachment, - backupLevel, - getBackupCdnInfo, - }); + // We need to always get updatedAttachment in case the attachment wasn't reencryptable + // to the original digest. In that case mediaName will be based on updatedAttachment. + const { filePointer, updatedAttachment } = isLocalBackup + ? await getLocalBackupFilePointerForAttachment({ + attachment, + backupLevel, + getBackupCdnInfo, + }) + : await getFilePointerForAttachment({ + attachment, + backupLevel, + getBackupCdnInfo, + }); + + if (isLocalBackup && filePointer.localLocator) { + // Duplicate attachment check. Local backups can only contain 1 file per mediaName, + // so if we see a duplicate mediaName then we must reuse the previous FilePointer. + const { mediaName } = filePointer.localLocator; + strictAssert( + mediaName, + 'FilePointer.LocalLocator must contain mediaName' + ); + const existingFilePointer = this.#mediaNamesToFilePointers.get(mediaName); + if (existingFilePointer) { + strictAssert( + existingFilePointer.localLocator, + 'Local backup existing mediaName FilePointer must contain LocalLocator' + ); + strictAssert( + existingFilePointer.localLocator.size === attachment.size, + 'Local backup existing mediaName FilePointer size must match attachment' + ); + return existingFilePointer; + } + + this.#mediaNamesToFilePointers.set(mediaName, filePointer); + } if (updatedAttachment) { // TODO (DESKTOP-6688): ensure that we update the message/attachment in DB with the @@ -2639,6 +2735,7 @@ export class BackupExportStream extends Readable { async #toStandardMessage({ message, backupLevel, + isLocalBackup, }: { message: Pick< MessageAttributesType, @@ -2652,11 +2749,13 @@ export class BackupExportStream extends Readable { | 'received_at' >; backupLevel: BackupLevel; + isLocalBackup: boolean; }): Promise { return { quote: await this.#toQuote({ message, backupLevel, + isLocalBackup, }), attachments: message.attachments?.length ? await Promise.all( @@ -2665,6 +2764,7 @@ export class BackupExportStream extends Readable { attachment, backupLevel, message, + isLocalBackup, }); }) ) @@ -2673,6 +2773,7 @@ export class BackupExportStream extends Readable { ? await this.#processAttachment({ attachment: message.bodyAttachment, backupLevel, + isLocalBackup, messageReceivedAt: message.received_at, }) : undefined, @@ -2697,6 +2798,7 @@ export class BackupExportStream extends Readable { ? await this.#processAttachment({ attachment: preview.image, backupLevel, + isLocalBackup, messageReceivedAt: message.received_at, }) : undefined, @@ -2711,6 +2813,7 @@ export class BackupExportStream extends Readable { async #toDirectStoryReplyMessage({ message, backupLevel, + isLocalBackup, }: { message: Pick< MessageAttributesType, @@ -2722,6 +2825,7 @@ export class BackupExportStream extends Readable { | 'reactions' >; backupLevel: BackupLevel; + isLocalBackup: boolean; }): Promise { const result = new Backups.DirectStoryReplyMessage({ reactions: this.#getMessageReactions(message), @@ -2735,6 +2839,7 @@ export class BackupExportStream extends Readable { ? await this.#processAttachment({ attachment: message.bodyAttachment, backupLevel, + isLocalBackup, messageReceivedAt: message.received_at, }) : undefined, @@ -2755,12 +2860,14 @@ export class BackupExportStream extends Readable { async #toViewOnceMessage({ message, backupLevel, + isLocalBackup, }: { message: Pick< MessageAttributesType, 'attachments' | 'received_at' | 'reactions' >; backupLevel: BackupLevel; + isLocalBackup: boolean; }): Promise { const attachment = message.attachments?.at(0); return { @@ -2771,6 +2878,7 @@ export class BackupExportStream extends Readable { attachment, backupLevel, message, + isLocalBackup, }), reactions: this.#getMessageReactions(message), }; @@ -2779,7 +2887,8 @@ export class BackupExportStream extends Readable { async #toChatItemRevisions( parent: Backups.IChatItem, message: MessageAttributesType, - backupLevel: BackupLevel + backupLevel: BackupLevel, + isLocalBackup: boolean ): Promise | undefined> { const { editHistory } = message; if (editHistory == null) { @@ -2818,11 +2927,13 @@ export class BackupExportStream extends Readable { await this.#toDirectStoryReplyMessage({ message: history, backupLevel, + isLocalBackup, }); } else { result.standardMessage = await this.#toStandardMessage({ message: history, backupLevel, + isLocalBackup, }); } return result; diff --git a/ts/services/backups/import.ts b/ts/services/backups/import.ts index 50a85f3588..7882876a42 100644 --- a/ts/services/backups/import.ts +++ b/ts/services/backups/import.ts @@ -244,18 +244,22 @@ export class BackupImportStream extends Writable { #pendingGroupAvatars = new Map(); #frameErrorCount: number = 0; - private constructor(private readonly backupType: BackupType) { + private constructor( + private readonly backupType: BackupType, + private readonly localBackupSnapshotDir: string | undefined + ) { super({ objectMode: true }); } public static async create( - backupType = BackupType.Ciphertext + backupType = BackupType.Ciphertext, + localBackupSnapshotDir: string | undefined = undefined ): Promise { await AttachmentDownloadManager.stop(); await DataWriter.removeAllBackupAttachmentDownloadJobs(); await resetBackupMediaDownloadProgress(); - return new BackupImportStream(backupType); + return new BackupImportStream(backupType, localBackupSnapshotDir); } override async _write( @@ -1854,11 +1858,19 @@ export class BackupImportStream extends Writable { bodyRanges: this.#fromBodyRanges(data.text), })), bodyAttachment: data.longText - ? convertFilePointerToAttachment(data.longText) + ? convertFilePointerToAttachment( + data.longText, + this.#getFilePointerOptions() + ) : undefined, attachments: data.attachments?.length ? data.attachments - .map(convertBackupMessageAttachmentToAttachment) + .map(attachment => + convertBackupMessageAttachmentToAttachment( + attachment, + this.#getFilePointerOptions() + ) + ) .filter(isNotNil) : undefined, preview: data.linkPreview?.length @@ -1902,7 +1914,10 @@ export class BackupImportStream extends Writable { description: dropNull(preview.description), date: getCheckedTimestampOrUndefinedFromLong(preview.date), image: preview.image - ? convertFilePointerToAttachment(preview.image) + ? convertFilePointerToAttachment( + preview.image, + this.#getFilePointerOptions() + ) : undefined, }; }) @@ -1917,7 +1932,10 @@ export class BackupImportStream extends Writable { ...(attachment ? { attachments: [ - convertBackupMessageAttachmentToAttachment(attachment), + convertBackupMessageAttachmentToAttachment( + attachment, + this.#getFilePointerOptions() + ), ].filter(isNotNil), } : { @@ -1948,7 +1966,10 @@ export class BackupImportStream extends Writable { result.body = textReply.text?.body ?? undefined; result.bodyRanges = this.#fromBodyRanges(textReply.text); result.bodyAttachment = textReply.longText - ? convertFilePointerToAttachment(textReply.longText) + ? convertFilePointerToAttachment( + textReply.longText, + this.#getFilePointerOptions() + ) : undefined; } else if (emoji) { result.storyReaction = { @@ -1978,7 +1999,10 @@ export class BackupImportStream extends Writable { body: textReply.text?.body ?? undefined, bodyRanges: this.#fromBodyRanges(textReply.text), bodyAttachment: textReply.longText - ? convertFilePointerToAttachment(textReply.longText) + ? convertFilePointerToAttachment( + textReply.longText, + this.#getFilePointerOptions() + ) : undefined, }; } @@ -2101,7 +2125,10 @@ export class BackupImportStream extends Writable { ? stringToMIMEType(contentType) : APPLICATION_OCTET_STREAM, thumbnail: thumbnail?.pointer - ? convertFilePointerToAttachment(thumbnail.pointer) + ? convertFilePointerToAttachment( + thumbnail.pointer, + this.#getFilePointerOptions() + ) : undefined, }; }) ?? [], @@ -2263,7 +2290,10 @@ export class BackupImportStream extends Writable { organization: organization || undefined, avatar: avatar ? { - avatar: convertFilePointerToAttachment(avatar), + avatar: convertFilePointerToAttachment( + avatar, + this.#getFilePointerOptions() + ), isProfile: false, } : undefined, @@ -2310,7 +2340,12 @@ export class BackupImportStream extends Writable { packId: Bytes.toHex(packId), packKey: Bytes.toBase64(packKey), stickerId, - data: data ? convertFilePointerToAttachment(data) : undefined, + data: data + ? convertFilePointerToAttachment( + data, + this.#getFilePointerOptions() + ) + : undefined, }, reactions: this.#fromReactions(chatItem.stickerMessage.reactions), }, @@ -3691,6 +3726,14 @@ export class BackupImportStream extends Writable { autoBubbleColor, }; } + + #getFilePointerOptions() { + if (this.localBackupSnapshotDir != null) { + return { localBackupSnapshotDir: this.localBackupSnapshotDir }; + } + + return {}; + } } function rgbIntToDesktopHSL(intValue: number): { diff --git a/ts/services/backups/index.ts b/ts/services/backups/index.ts index 38e996e5ee..ae5bfaaaf6 100644 --- a/ts/services/backups/index.ts +++ b/ts/services/backups/index.ts @@ -5,15 +5,16 @@ import { pipeline } from 'stream/promises'; import { PassThrough } from 'stream'; import type { Readable, Writable } from 'stream'; import { createReadStream, createWriteStream } from 'fs'; -import { unlink, stat } from 'fs/promises'; +import { mkdir, stat, unlink } from 'fs/promises'; import { ensureFile } from 'fs-extra'; import { join } from 'path'; import { createGzip, createGunzip } from 'zlib'; import { createCipheriv, createHmac, randomBytes } from 'crypto'; -import { noop } from 'lodash'; +import { isEqual, noop } from 'lodash'; import { BackupLevel } from '@signalapp/libsignal-client/zkgroup'; import { BackupKey } from '@signalapp/libsignal-client/dist/AccountKeys'; import { throttle } from 'lodash/fp'; +import { ipcRenderer } from 'electron'; import { DataReader, DataWriter } from '../../sql/Client'; import * as log from '../../logging/log'; @@ -49,7 +50,11 @@ import { isTestOrMockEnvironment } from '../../environment'; import { runStorageServiceSyncJob } from '../storage'; import { BackupExportStream, type StatsType } from './export'; import { BackupImportStream } from './import'; -import { getKeyMaterial } from './crypto'; +import { + getBackupId, + getKeyMaterial, + getLocalBackupMetadataKey, +} from './crypto'; import { BackupCredentials } from './credentials'; import { BackupAPI } from './api'; import { validateBackup, ValidationType } from './validator'; @@ -66,6 +71,16 @@ import { MemoryStream } from './util/MemoryStream'; import { ToastType } from '../../types/Toast'; import { isAdhoc, isNightly } from '../../util/version'; import { getMessageQueueTime } from '../../util/getMessageQueueTime'; +import { isLocalBackupsEnabled } from '../../util/isLocalBackupsEnabled'; +import type { ValidateLocalBackupStructureResultType } from './util/localBackup'; +import { + writeLocalBackupMetadata, + verifyLocalBackupMetadata, + writeLocalBackupFilesList, + readLocalBackupFilesList, + validateLocalBackupStructure, +} from './util/localBackup'; +import { AttachmentLocalBackupManager } from '../../jobs/AttachmentLocalBackupManager'; export { BackupType }; @@ -94,6 +109,7 @@ type DoDownloadOptionsType = Readonly<{ export type ImportOptionsType = Readonly<{ backupType?: BackupType; + localBackupSnapshotDir?: string; ephemeralKey?: Uint8Array; onProgress?: (currentBytes: number, totalBytes: number) => void; }>; @@ -104,9 +120,13 @@ export type ExportResultType = Readonly<{ stats: Readonly; }>; +export type LocalBackupExportResultType = ExportResultType & { + snapshotDir: string; +}; + export type ValidationResultType = Readonly< | { - result: ExportResultType; + result: ExportResultType | LocalBackupExportResultType; } | { error: string; @@ -123,6 +143,8 @@ export class BackupsService { | ExplodePromiseResultType | undefined; + #localBackupSnapshotDir: string | undefined; + public readonly credentials = new BackupCredentials(); public readonly api = new BackupAPI(this.credentials); public readonly throttledFetchCloudBackupStatus = throttle( @@ -263,23 +285,7 @@ export class BackupsService { } public async upload(): Promise { - // Make sure we are up-to-date on storage service - { - const { promise: storageService, resolve } = explodePromise(); - window.Whisper.events.once('storageService:syncComplete', resolve); - - runStorageServiceSyncJob({ reason: 'backups.upload' }); - await storageService; - } - - // Clear message queue - await window.waitForEmptyEventQueue(); - - // Make sure all batches are flushed - await Promise.all([ - window.waitForAllBatchers(), - window.flushAllWaitBatchers(), - ]); + await this.#waitForEmptyQueues('backups.upload'); const fileName = `backup-${randomBytes(32).toString('hex')}`; const filePath = join(window.BasePaths.temp, fileName); @@ -290,9 +296,9 @@ export class BackupsService { log.info(`exportBackup: starting, backup level: ${backupLevel}...`); try { - const fileSize = await this.exportToDisk(filePath, backupLevel); + const { totalBytes } = await this.exportToDisk(filePath, backupLevel); - await this.api.upload(filePath, fileSize); + await this.api.upload(filePath, totalBytes); } finally { try { await unlink(filePath); @@ -302,6 +308,95 @@ export class BackupsService { } } + public async exportLocalBackup( + backupsBaseDir: string | undefined = undefined, + backupLevel: BackupLevel = BackupLevel.Free + ): Promise { + strictAssert(isLocalBackupsEnabled(), 'Local backups must be enabled'); + + await this.#waitForEmptyQueues('backups.exportLocalBackup'); + + const baseDir = + backupsBaseDir ?? + join(window.SignalContext.getPath('userData'), 'SignalBackups'); + const snapshotDir = join(baseDir, `signal-backup-${new Date().getTime()}`); + await mkdir(snapshotDir, { recursive: true }); + const mainProtoPath = join(snapshotDir, 'main'); + + log.info('exportLocalBackup: starting'); + + const exportResult = await this.exportToDisk( + mainProtoPath, + backupLevel, + BackupType.Ciphertext, + snapshotDir + ); + + log.info('exportLocalBackup: writing metadata'); + const metadataArgs = { + snapshotDir, + backupId: getBackupId(), + metadataKey: getLocalBackupMetadataKey(), + }; + await writeLocalBackupMetadata(metadataArgs); + await verifyLocalBackupMetadata(metadataArgs); + + log.info( + 'exportLocalBackup: waiting for AttachmentLocalBackupManager to finish' + ); + await AttachmentLocalBackupManager.waitForIdle(); + + log.info(`exportLocalBackup: exported to disk: ${snapshotDir}`); + return { ...exportResult, snapshotDir }; + } + + public async stageLocalBackupForImport( + snapshotDir: string + ): Promise { + const result = await validateLocalBackupStructure(snapshotDir); + const { success, error } = result; + if (success) { + this.#localBackupSnapshotDir = snapshotDir; + log.info( + `stageLocalBackupForImport: Staged ${snapshotDir} for import. Please link to perform import.` + ); + } else { + this.#localBackupSnapshotDir = undefined; + log.info( + `stageLocalBackupForImport: Invalid snapshot ${snapshotDir}. Error: ${error}.` + ); + } + return result; + } + + public isLocalBackupStaged(): boolean { + return Boolean(this.#localBackupSnapshotDir); + } + + public async importLocalBackup(): Promise { + strictAssert( + this.#localBackupSnapshotDir, + 'importLocalBackup: Staged backup is required, use stageLocalBackupForImport()' + ); + + log.info(`importLocalBackup: Importing ${this.#localBackupSnapshotDir}`); + + const backupFile = join(this.#localBackupSnapshotDir, 'main'); + await this.importFromDisk(backupFile, { + localBackupSnapshotDir: this.#localBackupSnapshotDir, + }); + + await verifyLocalBackupMetadata({ + snapshotDir: this.#localBackupSnapshotDir, + backupId: getBackupId(), + metadataKey: getLocalBackupMetadataKey(), + }); + + this.#localBackupSnapshotDir = undefined; + + log.info('importLocalBackup: Done'); + } + // Test harness public async exportBackupData( backupLevel: BackupLevel = BackupLevel.Free, @@ -319,29 +414,63 @@ export class BackupsService { }; } - // Test harness public async exportToDisk( path: string, backupLevel: BackupLevel = BackupLevel.Free, - backupType = BackupType.Ciphertext - ): Promise { - const { totalBytes } = await this.#exportBackup( + backupType = BackupType.Ciphertext, + localBackupSnapshotDir: string | undefined = undefined + ): Promise { + const exportResult = await this.#exportBackup( createWriteStream(path), backupLevel, - backupType + backupType, + localBackupSnapshotDir ); if (backupType === BackupType.Ciphertext) { await validateBackup( () => new FileStream(path), - totalBytes, + exportResult.totalBytes, isTestOrMockEnvironment() ? ValidationType.Internal : ValidationType.Export ); } - return totalBytes; + return exportResult; + } + + public async _internalExportLocalBackup( + backupLevel: BackupLevel = BackupLevel.Free + ): Promise { + try { + const { canceled, dirPath: backupsBaseDir } = await ipcRenderer.invoke( + 'show-open-folder-dialog' + ); + if (canceled || !backupsBaseDir) { + return { error: 'Backups directory not selected' }; + } + + const result = await this.exportLocalBackup(backupsBaseDir, backupLevel); + return { result }; + } catch (error) { + return { error: Errors.toLogFormat(error) }; + } + } + + public async _internalStageLocalBackupForImport(): Promise { + const { canceled, dirPath: snapshotDir } = await ipcRenderer.invoke( + 'show-open-folder-dialog' + ); + if (canceled || !snapshotDir) { + return { + success: false, + error: 'File dialog canceled', + snapshotDir: undefined, + }; + } + + return this.stageLocalBackupForImport(snapshotDir); } // Test harness @@ -417,6 +546,7 @@ export class BackupsService { backupType = BackupType.Ciphertext, ephemeralKey, onProgress, + localBackupSnapshotDir = undefined, }: ImportOptionsType = {} ): Promise { strictAssert(!this.#isRunning, 'BackupService is already running'); @@ -438,7 +568,10 @@ export class BackupsService { window.ConversationController.setReadOnly(true); - const importStream = await BackupImportStream.create(backupType); + const importStream = await BackupImportStream.create( + backupType, + localBackupSnapshotDir + ); if (backupType === BackupType.Ciphertext) { const { aesKey, macKey } = getKeyMaterial( ephemeralKey ? new BackupKey(Buffer.from(ephemeralKey)) : undefined @@ -761,7 +894,8 @@ export class BackupsService { async #exportBackup( sink: Writable, backupLevel: BackupLevel = BackupLevel.Free, - backupType = BackupType.Ciphertext + backupType = BackupType.Ciphertext, + localBackupSnapshotDir: string | undefined = undefined ): Promise { strictAssert(!this.#isRunning, 'BackupService is already running'); @@ -786,7 +920,7 @@ export class BackupsService { const { aesKey, macKey } = getKeyMaterial(); const recordStream = new BackupExportStream(backupType); - recordStream.run(backupLevel); + recordStream.run(backupLevel, localBackupSnapshotDir); const iv = randomBytes(IV_LENGTH); @@ -817,6 +951,21 @@ export class BackupsService { throw missingCaseError(backupType); } + if (localBackupSnapshotDir) { + log.info('exportBackup: writing local backup files list'); + const filesWritten = await writeLocalBackupFilesList({ + snapshotDir: localBackupSnapshotDir, + mediaNamesIterator: recordStream.getMediaNamesIterator(), + }); + const filesRead = await readLocalBackupFilesList( + localBackupSnapshotDir + ); + strictAssert( + isEqual(filesWritten, filesRead), + 'exportBackup: Local backup files proto must match files written' + ); + } + const duration = Date.now() - start; return { totalBytes, stats: recordStream.getStats(), duration }; } finally { @@ -866,6 +1015,28 @@ export class BackupsService { window.reduxActions.installer.startInstaller(); } + async #waitForEmptyQueues( + reason: 'backups.upload' | 'backups.exportLocalBackup' + ) { + // Make sure we are up-to-date on storage service + { + const { promise: storageService, resolve } = explodePromise(); + window.Whisper.events.once('storageService:syncComplete', resolve); + + runStorageServiceSyncJob({ reason }); + await storageService; + } + + // Clear message queue + await window.waitForEmptyEventQueue(); + + // Make sure all batches are flushed + await Promise.all([ + window.waitForAllBatchers(), + window.flushAllWaitBatchers(), + ]); + } + public isImportRunning(): boolean { return this.#isRunning === 'import'; } diff --git a/ts/services/backups/util/filePointers.ts b/ts/services/backups/util/filePointers.ts index 2a7096f2be..f6f3a24e28 100644 --- a/ts/services/backups/util/filePointers.ts +++ b/ts/services/backups/util/filePointers.ts @@ -40,11 +40,17 @@ import { bytesToUuid } from '../../../util/uuidToBytes'; import { createName } from '../../../util/attachmentPath'; import { ensureAttachmentIsReencryptable } from '../../../util/ensureAttachmentIsReencryptable'; import type { ReencryptionInfo } from '../../../AttachmentCrypto'; +import { getAttachmentLocalBackupPathFromSnapshotDir } from './localBackup'; + +type ConvertFilePointerToAttachmentOptions = { + // Only for testing + _createName: (suffix?: string) => string; + localBackupSnapshotDir: string | undefined; +}; export function convertFilePointerToAttachment( filePointer: Backups.FilePointer, - // Only for testing - { _createName: doCreateName = createName } = {} + options: Partial = {} ): AttachmentType { const { contentType, @@ -58,7 +64,9 @@ export function convertFilePointerToAttachment( attachmentLocator, backupLocator, invalidAttachmentLocator, + localLocator, } = filePointer; + const doCreateName = options._createName ?? createName; const commonProps: Omit = { contentType: contentType @@ -122,6 +130,57 @@ export function convertFilePointerToAttachment( }; } + if (localLocator) { + const { + mediaName, + localKey, + backupCdnNumber, + remoteKey: key, + remoteDigest: digest, + size, + transitCdnKey, + transitCdnNumber, + } = localLocator; + + const { localBackupSnapshotDir } = options; + strictAssert( + localBackupSnapshotDir, + 'localBackupSnapshotDir is required for filePointer.localLocator' + ); + + if (mediaName == null) { + log.error( + 'convertFilePointerToAttachment: filePointer.localLocator missing mediaName!' + ); + return { + ...omit(commonProps, 'downloadPath'), + error: true, + size: 0, + }; + } + const localBackupPath = getAttachmentLocalBackupPathFromSnapshotDir( + mediaName, + localBackupSnapshotDir + ); + + return { + ...commonProps, + cdnKey: transitCdnKey ?? undefined, + cdnNumber: transitCdnNumber ?? undefined, + key: key?.length ? Bytes.toBase64(key) : undefined, + digest: digest?.length ? Bytes.toBase64(digest) : undefined, + size: size ?? 0, + localBackupPath, + localKey: localKey?.length ? Bytes.toBase64(localKey) : undefined, + backupLocator: backupCdnNumber + ? { + mediaName, + cdnNumber: backupCdnNumber, + } + : undefined, + }; + } + if (!invalidAttachmentLocator) { log.error('convertFilePointerToAttachment: filePointer had no locator'); } @@ -134,7 +193,8 @@ export function convertFilePointerToAttachment( } export function convertBackupMessageAttachmentToAttachment( - messageAttachment: Backups.IMessageAttachment + messageAttachment: Backups.IMessageAttachment, + options: Partial = {} ): AttachmentType | null { const { clientUuid } = messageAttachment; @@ -142,7 +202,7 @@ export function convertBackupMessageAttachmentToAttachment( return null; } const result = { - ...convertFilePointerToAttachment(messageAttachment.pointer), + ...convertFilePointerToAttachment(messageAttachment.pointer, options), clientUuid: clientUuid ? bytesToUuid(clientUuid) : undefined, }; @@ -372,6 +432,99 @@ export async function getFilePointerForAttachment({ }; } +// Given a remote backup FilePointer, return a FilePointer referencing a local backup +export async function getLocalBackupFilePointerForAttachment({ + attachment, + backupLevel, + getBackupCdnInfo, +}: { + attachment: Readonly; + backupLevel: BackupLevel; + getBackupCdnInfo: GetBackupCdnInfoType; +}): Promise<{ + filePointer: Backups.FilePointer; + updatedAttachment?: AttachmentType; +}> { + const { filePointer: remoteFilePointer, updatedAttachment } = + await getFilePointerForAttachment({ + attachment, + backupLevel, + getBackupCdnInfo, + }); + + if (attachment.localKey == null) { + return { filePointer: remoteFilePointer, updatedAttachment }; + } + + strictAssert( + attachment.localKey != null, + 'getLocalBackupFilePointerForAttachment: attachment must have localKey' + ); + + if (remoteFilePointer.backupLocator) { + const { backupLocator } = remoteFilePointer; + const { mediaName } = backupLocator; + strictAssert( + mediaName, + 'getLocalBackupFilePointerForAttachment: BackupLocator must have mediaName' + ); + + const localLocator = new Backups.FilePointer.LocalLocator({ + mediaName, + localKey: Bytes.fromBase64(attachment.localKey), + remoteKey: backupLocator.key, + remoteDigest: backupLocator.digest, + size: backupLocator.size, + backupCdnNumber: backupLocator.cdnNumber, + transitCdnKey: backupLocator.transitCdnKey, + transitCdnNumber: backupLocator.transitCdnNumber, + }); + + return { + filePointer: { + ...omit(remoteFilePointer, 'backupLocator'), + localLocator, + }, + updatedAttachment, + }; + } + + if (remoteFilePointer.attachmentLocator) { + const { attachmentLocator } = remoteFilePointer; + const { digest } = attachmentLocator; + strictAssert( + digest, + 'getLocalBackupFilePointerForAttachment: AttachmentLocator must have digest' + ); + const mediaName = getMediaNameFromDigest(Bytes.toBase64(digest)); + strictAssert( + mediaName, + 'getLocalBackupFilePointerForAttachment: mediaName must be derivable from AttachmentLocator' + ); + + const localLocator = new Backups.FilePointer.LocalLocator({ + mediaName, + localKey: Bytes.fromBase64(attachment.localKey), + remoteKey: attachmentLocator.key, + remoteDigest: attachmentLocator.digest, + size: attachmentLocator.size, + backupCdnNumber: undefined, + transitCdnKey: attachmentLocator.cdnKey, + transitCdnNumber: attachmentLocator.cdnNumber, + }); + + return { + filePointer: { + ...omit(remoteFilePointer, 'attachmentLocator'), + localLocator, + }, + updatedAttachment, + }; + } + + return { filePointer: remoteFilePointer, updatedAttachment }; +} + function getAttachmentLocator( attachment: AttachmentDownloadableFromTransitTier ) { @@ -419,21 +572,33 @@ export async function maybeGetBackupJobForAttachmentAndFilePointer({ getBackupCdnInfo: GetBackupCdnInfoType; messageReceivedAt: number; }): Promise { - if (!filePointer.backupLocator) { + let locator: + | Backups.FilePointer.IBackupLocator + | Backups.FilePointer.ILocalLocator; + if (filePointer.backupLocator) { + locator = filePointer.backupLocator; + } else if (filePointer.localLocator) { + locator = filePointer.localLocator; + } else { return null; } - const { mediaName } = filePointer.backupLocator; + const { mediaName } = locator; strictAssert(mediaName, 'mediaName must exist'); - const { isInBackupTier } = await getBackupCdnInfo( - getMediaIdFromMediaName(mediaName).string - ); - - if (isInBackupTier) { - return null; + if (filePointer.backupLocator) { + const { isInBackupTier } = await getBackupCdnInfo( + getMediaIdFromMediaName(mediaName).string + ); + if (isInBackupTier) { + return null; + } } + // TODO: For local backups we don't want to double back up the same file, so + // we could check for the same file here and if it's found then return early. + // Also ok to skip downstream when the job runs. + strictAssert( isAttachmentLocallySaved(attachment), 'Attachment must be saved locally for it to be backed up' @@ -455,19 +620,38 @@ export async function maybeGetBackupJobForAttachmentAndFilePointer({ encryptionInfo = attachment.reencryptionInfo; } - strictAssert( - filePointer.backupLocator.digest, - 'digest must exist on backupLocator' - ); - strictAssert( - encryptionInfo.digest === Bytes.toBase64(filePointer.backupLocator.digest), - 'digest on job and backupLocator must match' - ); + if (filePointer.backupLocator) { + strictAssert( + filePointer.backupLocator.digest, + 'digest must exist on backupLocator' + ); + strictAssert( + encryptionInfo.digest === + Bytes.toBase64(filePointer.backupLocator.digest), + 'digest on job and backupLocator must match' + ); + } else if (filePointer.localLocator) { + strictAssert( + filePointer.localLocator.remoteDigest, + 'digest must exist on backupLocator' + ); + strictAssert( + encryptionInfo.digest === + Bytes.toBase64(filePointer.localLocator.remoteDigest), + 'digest on job and localLocator must match' + ); + } - const { path, contentType, size, uploadTimestamp, version, localKey } = - attachment; + const { path, contentType, size, uploadTimestamp, version } = attachment; - const { transitCdnKey, transitCdnNumber } = filePointer.backupLocator; + const { transitCdnKey, transitCdnNumber } = locator; + + let localKey: string | undefined; + if (filePointer.localLocator && filePointer.localLocator.localKey != null) { + localKey = Bytes.toBase64(filePointer.localLocator.localKey); + } else { + localKey = attachment.localKey; + } return { mediaName, diff --git a/ts/services/backups/util/localBackup.ts b/ts/services/backups/util/localBackup.ts new file mode 100644 index 0000000000..88a1180745 --- /dev/null +++ b/ts/services/backups/util/localBackup.ts @@ -0,0 +1,296 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { randomBytes } from 'crypto'; +import { dirname, join } from 'path'; +import { readFile, stat, writeFile } from 'fs/promises'; +import { createReadStream, createWriteStream } from 'fs'; +import { Transform } from 'stream'; +import { pipeline } from 'stream/promises'; +import * as log from '../../../logging/log'; +import * as Bytes from '../../../Bytes'; +import * as Errors from '../../../types/errors'; +import { Signal } from '../../../protobuf'; +import protobuf from '../../../protobuf/wrap'; +import { strictAssert } from '../../../util/assert'; +import { decryptAesCtr, encryptAesCtr } from '../../../Crypto'; +import type { LocalBackupMetadataVerificationType } from '../../../types/backups'; +import { + LOCAL_BACKUP_VERSION, + LOCAL_BACKUP_BACKUP_ID_IV_LENGTH, +} from '../constants'; +import { explodePromise } from '../../../util/explodePromise'; + +const { Reader } = protobuf; + +export function getLocalBackupDirectoryForMediaName({ + backupsBaseDir, + mediaName, +}: { + backupsBaseDir: string; + mediaName: string; +}): string { + if (mediaName.length < 2) { + throw new Error('Invalid mediaName input'); + } + + return join(backupsBaseDir, 'files', mediaName.substring(0, 2)); +} + +export function getLocalBackupPathForMediaName({ + backupsBaseDir, + mediaName, +}: { + backupsBaseDir: string; + mediaName: string; +}): string { + return join( + getLocalBackupDirectoryForMediaName({ backupsBaseDir, mediaName }), + mediaName + ); +} + +/** + * Given a target local backup import e.g. /etc/SignalBackups/signal-backup-1743119037066 + * and an attachment, return the attachment's file path within the local backup + * e.g. /etc/SignalBackups/files/a1/[a1bcdef...] + * + * @param {string} snapshotDir - Timestamped local backup directory + */ +export function getAttachmentLocalBackupPathFromSnapshotDir( + mediaName: string, + snapshotDir: string +): string { + return join( + dirname(snapshotDir), + 'files', + mediaName.substring(0, 2), + mediaName + ); +} + +export async function writeLocalBackupMetadata({ + snapshotDir, + backupId, + metadataKey, +}: LocalBackupMetadataVerificationType): Promise { + const iv = randomBytes(LOCAL_BACKUP_BACKUP_ID_IV_LENGTH); + const encryptedId = encryptAesCtr(metadataKey, backupId, iv); + + const metadataSerialized = Signal.backup.local.Metadata.encode({ + backupId: new Signal.backup.local.Metadata.EncryptedBackupId({ + iv, + encryptedId, + }), + version: LOCAL_BACKUP_VERSION, + }).finish(); + + const metadataPath = join(snapshotDir, 'metadata'); + await writeFile(metadataPath, metadataSerialized); +} + +export async function verifyLocalBackupMetadata({ + snapshotDir, + backupId, + metadataKey, +}: LocalBackupMetadataVerificationType): Promise { + const metadataPath = join(snapshotDir, 'metadata'); + const metadataSerialized = await readFile(metadataPath); + + const metadata = Signal.backup.local.Metadata.decode(metadataSerialized); + strictAssert( + metadata.version === LOCAL_BACKUP_VERSION, + 'verifyLocalBackupMetadata: Local backup version must match' + ); + strictAssert( + metadata.backupId, + 'verifyLocalBackupMetadata: Must have backupId' + ); + + const { iv, encryptedId } = metadata.backupId; + strictAssert(iv, 'verifyLocalBackupMetadata: Must have backupId.iv'); + strictAssert( + encryptedId, + 'verifyLocalBackupMetadata: Must have backupId.encryptedId' + ); + + const localBackupBackupId = decryptAesCtr(metadataKey, encryptedId, iv); + strictAssert( + Bytes.areEqual(backupId, localBackupBackupId), + 'verifyLocalBackupMetadata: backupId must match the local backup backupId' + ); + + return true; +} + +export async function writeLocalBackupFilesList({ + snapshotDir, + mediaNamesIterator, +}: { + snapshotDir: string; + mediaNamesIterator: MapIterator; +}): Promise> { + const { promise, resolve, reject } = explodePromise>(); + + const filesListPath = join(snapshotDir, 'files'); + const writeStream = createWriteStream(filesListPath); + writeStream.on('error', error => { + reject(error); + }); + + const files: Array = []; + for (const mediaName of mediaNamesIterator) { + const data = Signal.backup.local.FilesFrame.encodeDelimited({ + mediaName, + }).finish(); + if (!writeStream.write(data)) { + // eslint-disable-next-line no-await-in-loop + await new Promise(resolveStream => + writeStream.once('drain', resolveStream) + ); + } + files.push(mediaName); + } + + writeStream.end(() => { + resolve(files); + }); + + await promise; + return files; +} + +export async function readLocalBackupFilesList( + snapshotDir: string +): Promise> { + const filesListPath = join(snapshotDir, 'files'); + const readStream = createReadStream(filesListPath); + const parseFilesTransform = new ParseFilesListTransform(); + + try { + await pipeline(readStream, parseFilesTransform); + } catch (error) { + try { + readStream.close(); + } catch (closeError) { + log.error( + 'readLocalBackupFilesList: Error when closing readStream', + Errors.toLogFormat(closeError) + ); + } + + throw error; + } + + readStream.close(); + + return parseFilesTransform.mediaNames; +} + +export class ParseFilesListTransform extends Transform { + public mediaNames: Array = []; + + public activeFile: Signal.backup.local.FilesFrame | undefined; + #unused: Uint8Array | undefined; + + override async _transform( + chunk: Buffer | undefined, + _encoding: string, + done: (error?: Error) => void + ): Promise { + if (!chunk || chunk.byteLength === 0) { + done(); + return; + } + + try { + let data = chunk; + if (this.#unused) { + data = Buffer.concat([this.#unused, data]); + this.#unused = undefined; + } + + const reader = Reader.create(data); + while (reader.pos < reader.len) { + const startPos = reader.pos; + + if (!this.activeFile) { + try { + this.activeFile = + Signal.backup.local.FilesFrame.decodeDelimited(reader); + } catch (err) { + // We get a RangeError if there wasn't enough data to read the next record. + if (err instanceof RangeError) { + // Note: A failed decodeDelimited() does in fact update reader.pos, so we + // must reset to startPos + this.#unused = data.subarray(startPos); + done(); + return; + } + + // Something deeper has gone wrong; the proto is malformed or something + done(err); + return; + } + } + + if (!this.activeFile) { + done( + new Error( + 'ParseFilesListTransform: No active file after successful decode!' + ) + ); + return; + } + + if (this.activeFile.mediaName) { + this.mediaNames.push(this.activeFile.mediaName); + } else { + log.warn( + 'ParseFilesListTransform: Active file had empty mediaName, ignoring' + ); + } + + this.activeFile = undefined; + } + } catch (error) { + done(error); + return; + } + + done(); + } +} + +export type ValidateLocalBackupStructureResultType = + | { success: true; error: undefined; snapshotDir: string | undefined } + | { success: false; error: string; snapshotDir: string | undefined }; + +export async function validateLocalBackupStructure( + snapshotDir: string +): Promise { + try { + await stat(snapshotDir); + } catch (error) { + return { + success: false, + error: 'Snapshot directory does not exist', + snapshotDir, + }; + } + + for (const file of ['main', 'metadata', 'files']) { + try { + // eslint-disable-next-line no-await-in-loop + await stat(join(snapshotDir, 'main')); + } catch (error) { + return { + success: false, + error: `Snapshot directory does not contain ${file} file`, + snapshotDir, + }; + } + } + + return { success: true, error: undefined, snapshotDir }; +} diff --git a/ts/state/ducks/conversations.ts b/ts/state/ducks/conversations.ts index 92d9c46f51..34362f6d27 100644 --- a/ts/state/ducks/conversations.ts +++ b/ts/state/ducks/conversations.ts @@ -4088,7 +4088,7 @@ const showSaveMultiDialog = (): Promise<{ canceled: boolean; dirPath?: string; }> => { - return ipcRenderer.invoke('show-save-multi-dialog'); + return ipcRenderer.invoke('show-open-folder-dialog', { useMainWindow: true }); }; export type SaveAttachmentsActionCreatorType = ReadonlyDeep< diff --git a/ts/test-electron/util/downloadAttachment_test.ts b/ts/test-electron/util/downloadAttachment_test.ts index 0eca5e9379..042bba8b97 100644 --- a/ts/test-electron/util/downloadAttachment_test.ts +++ b/ts/test-electron/util/downloadAttachment_test.ts @@ -50,6 +50,7 @@ describe('utils/downloadAttachment', () => { abortSignal: abortController.signal, }, dependencies: { + downloadAttachmentFromLocalBackup: stubDownload, downloadAttachmentFromServer: stubDownload, }, }); @@ -86,6 +87,7 @@ describe('utils/downloadAttachment', () => { abortSignal: abortController.signal, }, dependencies: { + downloadAttachmentFromLocalBackup: stubDownload, downloadAttachmentFromServer: stubDownload, }, }), @@ -123,6 +125,7 @@ describe('utils/downloadAttachment', () => { abortSignal: abortController.signal, }, dependencies: { + downloadAttachmentFromLocalBackup: stubDownload, downloadAttachmentFromServer: stubDownload, }, }); @@ -161,6 +164,7 @@ describe('utils/downloadAttachment', () => { abortSignal: abortController.signal, }, dependencies: { + downloadAttachmentFromLocalBackup: stubDownload, downloadAttachmentFromServer: stubDownload, }, }); @@ -210,6 +214,7 @@ describe('utils/downloadAttachment', () => { abortSignal: abortController.signal, }, dependencies: { + downloadAttachmentFromLocalBackup: stubDownload, downloadAttachmentFromServer: stubDownload, }, }); @@ -260,6 +265,7 @@ describe('utils/downloadAttachment', () => { abortSignal: abortController.signal, }, dependencies: { + downloadAttachmentFromLocalBackup: stubDownload, downloadAttachmentFromServer: stubDownload, }, }), diff --git a/ts/test-mock/backups/backups_test.ts b/ts/test-mock/backups/backups_test.ts index 5a836e6328..92b9d85152 100644 --- a/ts/test-mock/backups/backups_test.ts +++ b/ts/test-mock/backups/backups_test.ts @@ -1,12 +1,15 @@ // Copyright 2023 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only +import fs from 'fs/promises'; import { randomBytes } from 'node:crypto'; import { join } from 'node:path'; +import os from 'os'; import { readFile } from 'node:fs/promises'; import createDebug from 'debug'; import Long from 'long'; import { Proto, StorageState } from '@signalapp/mock-server'; +import { assert } from 'chai'; import { expect } from 'playwright/test'; import { generateStoryDistributionId } from '../../types/StoryDistributionId'; @@ -17,7 +20,7 @@ import { IMAGE_JPEG } from '../../types/MIME'; import { uuidToBytes } from '../../util/uuidToBytes'; import * as durations from '../../util/durations'; import type { App } from '../playwright'; -import { Bootstrap } from '../bootstrap'; +import { Bootstrap, type LinkOptionsType } from '../bootstrap'; import { getMessageInTimelineByTimestamp, sendTextMessage, @@ -60,7 +63,11 @@ describe('backups', function (this: Mocha.Suite) { await bootstrap.teardown(); }); - it('exports and imports regular backup', async function () { + async function generateTestDataThenRestoreBackup( + thisVal: Mocha.Context, + exportBackupFn: () => void, + getBootstrapLinkParams: () => LinkOptionsType + ) { let state = StorageState.getEmpty(); const { phone, contacts } = bootstrap; @@ -237,7 +244,7 @@ describe('backups', function (this: Mocha.Suite) { .waitFor(); } - await app.uploadBackup(); + await exportBackupFn(); const comparator = await bootstrap.createScreenshotComparator( app, @@ -288,7 +295,7 @@ describe('backups', function (this: Mocha.Suite) { await snapshot('story privacy'); }, - this.test + thisVal.test ); await app.close(); @@ -296,7 +303,7 @@ describe('backups', function (this: Mocha.Suite) { // Restart await bootstrap.eraseStorage(); await server.removeAllCDNAttachments(); - app = await bootstrap.link(); + app = await bootstrap.link(getBootstrapLinkParams()); await app.waitForBackupImportComplete(); // Make sure that contact sync happens after backup import, otherwise the @@ -312,6 +319,35 @@ describe('backups', function (this: Mocha.Suite) { } await comparator(app); + } + + it('exports and imports local backup', async function () { + let snapshotDir: string; + + await generateTestDataThenRestoreBackup( + this, + async () => { + const backupsBaseDir = await fs.mkdtemp( + join(os.tmpdir(), 'SignalBackups') + ); + snapshotDir = await app.exportLocalBackup(backupsBaseDir); + assert.exists( + snapshotDir, + 'Local backup export should return backup dir' + ); + }, + () => ({ localBackup: snapshotDir }) + ); + }); + + it('exports and imports regular backup', async function () { + await generateTestDataThenRestoreBackup( + this, + async () => { + await app.uploadBackup(); + }, + () => ({}) + ); }); it('imports ephemeral backup', async function () { diff --git a/ts/test-mock/bootstrap.ts b/ts/test-mock/bootstrap.ts index 563fa0bd14..0522f97bf0 100644 --- a/ts/test-mock/bootstrap.ts +++ b/ts/test-mock/bootstrap.ts @@ -132,6 +132,7 @@ export type EphemeralBackupType = Readonly< export type LinkOptionsType = Readonly<{ extraConfig?: Partial; ephemeralBackup?: EphemeralBackupType; + localBackup?: string; }>; type BootstrapInternalOptions = BootstrapOptions & @@ -376,6 +377,7 @@ export class Bootstrap { public async link({ extraConfig, ephemeralBackup, + localBackup, }: LinkOptionsType = {}): Promise { debug('linking'); @@ -383,6 +385,10 @@ export class Bootstrap { const window = await app.getWindow(); + if (localBackup != null) { + await app.stageLocalBackupForImport(localBackup); + } + debug('looking for QR code or relink button'); const qrCode = window.locator( '.module-InstallScreenQrCodeNotScannedStep__qr-code__code' diff --git a/ts/test-mock/playwright.ts b/ts/test-mock/playwright.ts index 0d5b55774f..aec7be460e 100644 --- a/ts/test-mock/playwright.ts +++ b/ts/test-mock/playwright.ts @@ -207,6 +207,20 @@ export class App extends EventEmitter { return window.evaluate(`window.SignalCI.getMessagesBySentAt(${timestamp})`); } + public async exportLocalBackup(backupsBaseDir: string): Promise { + const window = await this.getWindow(); + return window.evaluate( + `window.SignalCI.exportLocalBackup('${backupsBaseDir}')` + ); + } + + public async stageLocalBackupForImport(snapshotDir: string): Promise { + const window = await this.getWindow(); + return window.evaluate( + `window.SignalCI.stageLocalBackupForImport('${snapshotDir}')` + ); + } + public async uploadBackup(): Promise { const window = await this.getWindow(); await window.evaluate('window.SignalCI.uploadBackup()'); diff --git a/ts/textsecure/AccountManager.ts b/ts/textsecure/AccountManager.ts index 82ffccf69b..65a00a91ed 100644 --- a/ts/textsecure/AccountManager.ts +++ b/ts/textsecure/AccountManager.ts @@ -1209,6 +1209,8 @@ export default class AccountManager extends EventTarget { } if (accountEntropyPool) { await storage.put('accountEntropyPool', accountEntropyPool); + } else { + log.warn('createAccount: accountEntropyPool was missing!'); } let derivedMasterKey = masterKey; if (derivedMasterKey == null) { diff --git a/ts/textsecure/downloadAttachment.ts b/ts/textsecure/downloadAttachment.ts index 5a39fca99b..41a671f655 100644 --- a/ts/textsecure/downloadAttachment.ts +++ b/ts/textsecure/downloadAttachment.ts @@ -55,7 +55,7 @@ export function getCdnKey(attachment: ProcessedAttachment): string { return cdnKey; } -function getBackupMediaOuterEncryptionKeyMaterial( +export function getBackupMediaOuterEncryptionKeyMaterial( attachment: AttachmentType ): BackupMediaKeyMaterialType { const mediaId = getMediaIdForAttachment(attachment); diff --git a/ts/types/Attachment.ts b/ts/types/Attachment.ts index 572abcd309..3bec07d84e 100644 --- a/ts/types/Attachment.ts +++ b/ts/types/Attachment.ts @@ -109,6 +109,7 @@ export type AttachmentType = { mediaName: string; cdnNumber?: number; }; + localBackupPath?: string; // See app/attachment_channel.ts version?: 1 | 2; @@ -1281,6 +1282,12 @@ export function mightBeOnBackupTier( return Boolean(attachment.backupLocator?.mediaName); } +export function mightBeInLocalBackup( + attachment: Pick +): boolean { + return Boolean(attachment.localBackupPath && attachment.localKey); +} + export function isDownloadableFromTransitTier( attachment: AttachmentType ): attachment is AttachmentDownloadableFromTransitTier { diff --git a/ts/types/AttachmentBackup.ts b/ts/types/AttachmentBackup.ts index d80c58b30c..5295b3836f 100644 --- a/ts/types/AttachmentBackup.ts +++ b/ts/types/AttachmentBackup.ts @@ -45,6 +45,14 @@ export type ThumbnailAttachmentBackupJobType = { }; }; +export type CoreAttachmentLocalBackupJobType = + StandardAttachmentBackupJobType & { + backupsBaseDir: string; + }; + +export type AttachmentLocalBackupJobType = CoreAttachmentLocalBackupJobType & + JobManagerJobType; + const standardBackupJobDataSchema = z.object({ type: z.literal('standard'), mediaName: z.string(), diff --git a/ts/types/backups.ts b/ts/types/backups.ts index 290c73cd5e..9120a3ed32 100644 --- a/ts/types/backups.ts +++ b/ts/types/backups.ts @@ -61,3 +61,9 @@ export type BackupsSubscriptionType = cost?: SubscriptionCostType; } ); + +export type LocalBackupMetadataVerificationType = { + snapshotDir: string; + backupId: Uint8Array; + metadataKey: Uint8Array; +}; diff --git a/ts/util/createIPCEvents.ts b/ts/util/createIPCEvents.ts index 44df51dabf..14bd6d8f68 100644 --- a/ts/util/createIPCEvents.ts +++ b/ts/util/createIPCEvents.ts @@ -69,7 +69,8 @@ import type { BackupStatusType, } from '../types/backups'; import { isBackupFeatureEnabled } from './isBackupEnabled'; -import * as RemoteConfig from '../RemoteConfig'; +import { isSettingsInternalEnabled } from './isSettingsInternalEnabled'; +import type { ValidateLocalBackupStructureResultType } from '../services/backups/util/localBackup'; type SentMediaQualityType = 'standard' | 'high'; type NotificationSettingType = 'message' | 'name' | 'count' | 'off'; @@ -164,6 +165,8 @@ export type IPCEventsCallbacksType = { unknownSignalLink: () => void; getCustomColors: () => Record; syncRequest: () => Promise; + exportLocalBackup: () => Promise; + importLocalBackup: () => Promise; validateBackup: () => Promise; setGlobalDefaultConversationColor: ( color: ConversationColorType, @@ -542,7 +545,7 @@ export function createIPCEvents( }, isPrimary: () => window.textsecure.storage.user.getDeviceId() === 1, - isInternalUser: () => RemoteConfig.isEnabled('desktop.internalUser'), + isInternalUser: () => isSettingsInternalEnabled(), syncRequest: async () => { const contactSyncComplete = waitForEvent( 'contactSync:complete', @@ -552,6 +555,9 @@ export function createIPCEvents( return contactSyncComplete; }, // Only for internal use + exportLocalBackup: () => backupsService._internalExportLocalBackup(), + importLocalBackup: () => + backupsService._internalStageLocalBackupForImport(), validateBackup: () => backupsService._internalValidate(), getLastSyncTime: () => window.storage.get('synced_at'), setLastSyncTime: value => window.storage.put('synced_at', value), diff --git a/ts/util/downloadAttachment.ts b/ts/util/downloadAttachment.ts index 671195cc62..497fbf62c5 100644 --- a/ts/util/downloadAttachment.ts +++ b/ts/util/downloadAttachment.ts @@ -7,8 +7,10 @@ import { AttachmentVariant, AttachmentPermanentlyUndownloadableError, getAttachmentIdForLogging, + mightBeInLocalBackup, } from '../types/Attachment'; import { downloadAttachment as doDownloadAttachment } from '../textsecure/downloadAttachment'; +import { downloadAttachmentFromLocalBackup as doDownloadAttachmentFromLocalBackup } from './downloadAttachmentFromLocalBackup'; import { MediaTier } from '../types/AttachmentDownload'; import * as log from '../logging/log'; import { HTTPError } from '../textsecure/Errors'; @@ -18,7 +20,10 @@ import type { ReencryptedAttachmentV2 } from '../AttachmentCrypto'; export async function downloadAttachment({ attachment, options: { variant = AttachmentVariant.Default, onSizeUpdate, abortSignal }, - dependencies = { downloadAttachmentFromServer: doDownloadAttachment }, + dependencies = { + downloadAttachmentFromServer: doDownloadAttachment, + downloadAttachmentFromLocalBackup: doDownloadAttachmentFromLocalBackup, + }, }: { attachment: AttachmentType; options: { @@ -26,7 +31,10 @@ export async function downloadAttachment({ onSizeUpdate: (totalBytes: number) => void; abortSignal: AbortSignal; }; - dependencies?: { downloadAttachmentFromServer: typeof doDownloadAttachment }; + dependencies?: { + downloadAttachmentFromServer: typeof doDownloadAttachment; + downloadAttachmentFromLocalBackup: typeof doDownloadAttachmentFromLocalBackup; + }; }): Promise { const attachmentId = getAttachmentIdForLogging(attachment); const variantForLogging = @@ -51,6 +59,23 @@ export async function downloadAttachment({ }; } + if (mightBeInLocalBackup(attachment)) { + log.info(`${logId}: Downloading attachment from local backup`); + try { + const result = + await dependencies.downloadAttachmentFromLocalBackup(attachment); + onSizeUpdate(attachment.size); + return result; + } catch (error) { + // We also just log this error instead of throwing, since we want to still try to + // find it on the backup then transit tiers. + log.error( + `${logId}: error when downloading from local backup; will try backup and transit tier`, + toLogFormat(error) + ); + } + } + if (mightBeOnBackupTier(migratedAttachment)) { try { return await dependencies.downloadAttachmentFromServer( diff --git a/ts/util/downloadAttachmentFromLocalBackup.ts b/ts/util/downloadAttachmentFromLocalBackup.ts new file mode 100644 index 0000000000..c0aff5eb13 --- /dev/null +++ b/ts/util/downloadAttachmentFromLocalBackup.ts @@ -0,0 +1,61 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { isNumber } from 'lodash'; +import { + type AttachmentType, + getAttachmentIdForLogging, +} from '../types/Attachment'; +import * as log from '../logging/log'; +import { toLogFormat } from '../types/errors'; +import { + decryptAndReencryptLocally, + type ReencryptedAttachmentV2, +} from '../AttachmentCrypto'; +import { strictAssert } from './assert'; + +export class AttachmentPermanentlyUndownloadableError extends Error {} + +export async function downloadAttachmentFromLocalBackup( + attachment: AttachmentType +): Promise { + const attachmentId = getAttachmentIdForLogging(attachment); + const dataId = `${attachmentId}`; + const logId = `downloadAttachmentFromLocalBackup(${dataId})`; + + try { + return await doDownloadFromLocalBackup(attachment, { logId }); + } catch (error) { + log.error( + `${logId}: error when copying from local backup`, + toLogFormat(error) + ); + throw new AttachmentPermanentlyUndownloadableError(); + } +} + +async function doDownloadFromLocalBackup( + attachment: AttachmentType, + { + logId, + }: { + logId: string; + } +): Promise { + const { digest, localBackupPath, localKey, size } = attachment; + + strictAssert(digest, `${logId}: missing digest`); + strictAssert(localKey, `${logId}: missing localKey`); + strictAssert(localBackupPath, `${logId}: missing localBackupPath`); + strictAssert(isNumber(size), `${logId}: missing size`); + + return decryptAndReencryptLocally({ + type: 'local', + ciphertextPath: localBackupPath, + idForLogging: logId, + keysBase64: localKey, + size, + getAbsoluteAttachmentPath: + window.Signal.Migrations.getAbsoluteAttachmentPath, + }); +} diff --git a/ts/util/isLocalBackupsEnabled.ts b/ts/util/isLocalBackupsEnabled.ts new file mode 100644 index 0000000000..89d28644c7 --- /dev/null +++ b/ts/util/isLocalBackupsEnabled.ts @@ -0,0 +1,24 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import * as RemoteConfig from '../RemoteConfig'; +import { isTestOrMockEnvironment } from '../environment'; +import { isStagingServer } from './isStagingServer'; +import { isNightly } from './version'; + +export function isLocalBackupsEnabled(): boolean { + if (isStagingServer() || isTestOrMockEnvironment()) { + return true; + } + + if (RemoteConfig.isEnabled('desktop.internalUser')) { + return true; + } + + const version = window.getVersion?.(); + if (version != null) { + return isNightly(version); + } + + return false; +} diff --git a/ts/util/isSettingsInternalEnabled.ts b/ts/util/isSettingsInternalEnabled.ts new file mode 100644 index 0000000000..eb881e4cdb --- /dev/null +++ b/ts/util/isSettingsInternalEnabled.ts @@ -0,0 +1,18 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import * as RemoteConfig from '../RemoteConfig'; +import { isNightly } from './version'; + +export function isSettingsInternalEnabled(): boolean { + if (RemoteConfig.isEnabled('desktop.internalUser')) { + return true; + } + + const version = window.getVersion?.(); + if (version != null) { + return isNightly(version); + } + + return false; +} diff --git a/ts/windows/preload.ts b/ts/windows/preload.ts index 6e55218282..0412c537ee 100644 --- a/ts/windows/preload.ts +++ b/ts/windows/preload.ts @@ -51,6 +51,8 @@ installCallback('isInternalUser'); installCallback('syncRequest'); installCallback('getEmojiSkinToneDefault'); installCallback('setEmojiSkinToneDefault'); +installCallback('exportLocalBackup'); +installCallback('importLocalBackup'); installCallback('validateBackup'); installSetting('alwaysRelayCalls'); diff --git a/ts/windows/settings/app.tsx b/ts/windows/settings/app.tsx index 61a9ba9bd2..b7bb1f28d6 100644 --- a/ts/windows/settings/app.tsx +++ b/ts/windows/settings/app.tsx @@ -43,6 +43,7 @@ SettingsWindowProps.onRender( doDeleteAllData, doneRendering, editCustomColor, + exportLocalBackup, getConversationsWithCustomColor, hasAudioNotifications, hasAutoConvertEmoji, @@ -67,6 +68,7 @@ SettingsWindowProps.onRender( hasStoriesDisabled, hasTextFormatting, hasTypingIndicators, + importLocalBackup, initialSpellCheckSetting, isAutoDownloadUpdatesSupported, isAutoLaunchSupported, @@ -152,6 +154,7 @@ SettingsWindowProps.onRender( defaultConversationColor={defaultConversationColor} deviceName={deviceName} emojiSkinToneDefault={emojiSkinToneDefault} + exportLocalBackup={exportLocalBackup} phoneNumber={phoneNumber} doDeleteAllData={doDeleteAllData} doneRendering={doneRendering} @@ -181,6 +184,7 @@ SettingsWindowProps.onRender( hasTextFormatting={hasTextFormatting} hasTypingIndicators={hasTypingIndicators} i18n={i18n} + importLocalBackup={importLocalBackup} initialSpellCheckSetting={initialSpellCheckSetting} isAutoDownloadUpdatesSupported={isAutoDownloadUpdatesSupported} isAutoLaunchSupported={isAutoLaunchSupported} diff --git a/ts/windows/settings/preload.ts b/ts/windows/settings/preload.ts index cbc606801a..9ac4894606 100644 --- a/ts/windows/settings/preload.ts +++ b/ts/windows/settings/preload.ts @@ -100,6 +100,8 @@ const ipcGetEmojiSkinToneDefault = createCallback('getEmojiSkinToneDefault'); const ipcIsSyncNotSupported = createCallback('isPrimary'); const ipcIsInternalUser = createCallback('isInternalUser'); const ipcMakeSyncRequest = createCallback('syncRequest'); +const ipcExportLocalBackup = createCallback('exportLocalBackup'); +const ipcImportLocalBackup = createCallback('importLocalBackup'); const ipcValidateBackup = createCallback('validateBackup'); const ipcDeleteAllMyStories = createCallback('deleteAllMyStories'); const ipcRefreshCloudBackupStatus = createCallback('refreshCloudBackupStatus'); @@ -369,6 +371,8 @@ async function renderPreferences() { resetAllChatColors: ipcResetAllChatColors, resetDefaultChatColor: ipcResetDefaultChatColor, setGlobalDefaultConversationColor: ipcSetGlobalDefaultConversationColor, + exportLocalBackup: ipcExportLocalBackup, + importLocalBackup: ipcImportLocalBackup, validateBackup: ipcValidateBackup, // Limited support features isAutoDownloadUpdatesSupported: Settings.isAutoDownloadUpdatesSupported(