Refactor backup import/export options

This commit is contained in:
trevor-signal
2025-10-31 09:16:33 -04:00
committed by GitHub
parent 644702199a
commit 0a5f3ccccc
17 changed files with 539 additions and 516 deletions

View File

@@ -38,7 +38,7 @@ import {
shouldAttachmentEndUpInRemoteBackup,
getUndownloadedAttachmentSignature,
isIncremental,
hasRequiredInformationForBackup,
hasRequiredInformationForRemoteBackup,
} from '../util/Attachment.std.js';
import type { ReadonlyMessageAttributesType } from '../model-types.d.ts';
import { backupsService } from '../services/backups/index.preload.js';
@@ -310,7 +310,7 @@ export class AttachmentDownloadManager extends JobManager<CoreAttachmentDownload
if (
source === AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA &&
!hasRequiredInformationForBackup(attachment)
!hasRequiredInformationForRemoteBackup(attachment)
) {
source = AttachmentDownloadSource.BACKUP_IMPORT_NO_MEDIA;
}

View File

@@ -3,8 +3,12 @@
/* eslint-disable max-classes-per-file */
import { existsSync } from 'node:fs';
import { PassThrough } from 'node:stream';
import { constants as FS_CONSTANTS, copyFile, mkdir } from 'node:fs/promises';
import {
constants as FS_CONSTANTS,
copyFile,
mkdir,
rename,
} from 'node:fs/promises';
import * as durations from '../util/durations/index.std.js';
import { createLogger } from '../logging/log.std.js';
@@ -14,6 +18,7 @@ import { redactGenericText } from '../util/privacy.node.js';
import {
getAbsoluteAttachmentPath,
getAbsoluteAttachmentPath as doGetAbsoluteAttachmentPath,
getAbsoluteTempPath,
} from '../util/migrations.preload.js';
import {
JobManager,
@@ -36,6 +41,7 @@ import {
getLocalBackupDirectoryForMediaName,
getLocalBackupPathForMediaName,
} from '../services/backups/util/localBackup.node.js';
import { createName } from '../util/attachmentPath.node.js';
const log = createLogger('AttachmentLocalBackupManager');
@@ -219,7 +225,7 @@ async function runAttachmentBackupJobInner(
log.info(`${logId}: starting`);
const { backupsBaseDir, mediaName } = job;
const { localKey, path, size } = job.data;
const { path } = job.data;
if (!path) {
throw new AttachmentPermanentlyMissingError('No path property');
@@ -230,45 +236,26 @@ async function runAttachmentBackupJobInner(
throw new AttachmentPermanentlyMissingError('No file at provided path');
}
if (!localKey) {
throw new Error('No localKey property, required for test decryption');
}
const localBackupFileDir = getLocalBackupDirectoryForMediaName({
backupsBaseDir,
mediaName,
});
await mkdir(localBackupFileDir, { recursive: true });
const localBackupFilePath = getLocalBackupPathForMediaName({
const destinationLocalBackupFilePath = getLocalBackupPathForMediaName({
backupsBaseDir,
mediaName,
});
// TODO: Add check in local FS to prevent double backup
// File is already encrypted with localKey, so we just have to copy it to the backup dir
const attachmentPath = getAbsoluteAttachmentPath(path);
const sourceAttachmentPath = getAbsoluteAttachmentPath(path);
const tempPath = getAbsoluteTempPath(createName());
// A unique constraint on the DB table should enforce that only one job is writing to
// the same mediaName at a time, but just to be safe, we copy to temp file and rename to
// ensure the atomicity of the copy operation
// Set COPYFILE_FICLONE for Copy on Write (OS dependent, gracefully falls back to copy)
await copyFile(
attachmentPath,
localBackupFilePath,
FS_CONSTANTS.COPYFILE_FICLONE
);
// TODO: Optimize this check -- it can be expensive to test decrypt on every export
log.info(`${logId}: Verifying file in local backup`);
const sink = new PassThrough();
sink.resume();
await decryptAttachmentV2ToSink(
{
ciphertextPath: localBackupFilePath,
idForLogging: 'AttachmentLocalBackupManager',
keysBase64: localKey,
size,
type: 'local',
},
sink
);
await copyFile(sourceAttachmentPath, tempPath, FS_CONSTANTS.COPYFILE_FICLONE);
await rename(tempPath, destinationLocalBackupFilePath);
}

View File

@@ -3,7 +3,6 @@
import Long from 'long';
import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client';
import type { BackupLevel } from '@signalapp/libsignal-client/zkgroup.js';
import { dirname } from 'node:path';
import pMap from 'p-map';
import pTimeout from 'p-timeout';
@@ -117,8 +116,11 @@ import {
} from '../../types/CallDisposition.std.js';
import { isAciString } from '../../util/isAciString.std.js';
import { hslToRGBInt } from '../../util/hslToRGB.std.js';
import type { AboutMe, LocalChatStyle } from './types.std.js';
import { BackupType } from './types.std.js';
import type {
AboutMe,
BackupExportOptions,
LocalChatStyle,
} from './types.std.js';
import { messageHasPaymentEvent } from '../../messages/payments.std.js';
import {
numberToAddressType,
@@ -129,7 +131,8 @@ import type { AttachmentType } from '../../types/Attachment.std.js';
import {
isGIF,
isDownloaded,
hasRequiredInformationForBackup,
hasRequiredInformationForLocalBackup,
hasRequiredInformationForRemoteBackup,
} from '../../util/Attachment.std.js';
import { getFilePointerForAttachment } from './util/filePointers.preload.js';
import { getBackupMediaRootKey } from './crypto.preload.js';
@@ -141,6 +144,7 @@ import { AttachmentBackupManager } from '../../jobs/AttachmentBackupManager.prel
import { AttachmentLocalBackupManager } from '../../jobs/AttachmentLocalBackupManager.preload.js';
import {
getBackupCdnInfo,
getLocalBackupFileNameForAttachment,
getMediaNameForAttachment,
} from './util/mediaId.preload.js';
import { calculateExpirationTimestamp } from '../../util/expirationTimer.std.js';
@@ -210,8 +214,6 @@ type GetRecipientIdOptionsType =
type ToChatItemOptionsType = Readonly<{
aboutMe: AboutMe;
callHistoryByCallId: Record<string, CallHistoryDetails>;
backupLevel: BackupLevel;
isLocalBackup: boolean;
}>;
type NonBubbleOptionsType = Pick<
@@ -287,18 +289,11 @@ export class BackupExportStream extends Readable {
// array.
#customColorIdByUuid = new Map<string, Long>();
constructor(private readonly backupType: BackupType) {
constructor(private readonly options: Readonly<BackupExportOptions>) {
super();
}
public run(
backupLevel: BackupLevel,
localBackupSnapshotDir: string | undefined = undefined
): void {
const localBackupsBaseDir = localBackupSnapshotDir
? dirname(localBackupSnapshotDir)
: undefined;
const isLocalBackup = localBackupsBaseDir != null;
public run(): void {
drop(
(async () => {
log.info('BackupExportStream: starting...');
@@ -309,37 +304,44 @@ export class BackupExportStream extends Readable {
await pauseWriteAccess();
try {
await this.#unsafeRun(backupLevel, isLocalBackup);
await this.#unsafeRun();
} catch (error) {
this.emit('error', error);
} finally {
await resumeWriteAccess();
// TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction
const { type } = this.options;
switch (type) {
case 'local-encrypted':
{
log.info(
`BackupExportStream: Adding ${this.#attachmentBackupJobs.length} jobs for AttachmentLocalBackupManager`
);
const backupsBaseDir = dirname(
this.options.localBackupSnapshotDir
);
if (isLocalBackup) {
log.info(
`BackupExportStream: Adding ${this.#attachmentBackupJobs.length} jobs for AttachmentLocalBackupManager`
);
AttachmentLocalBackupManager.clearAllJobs();
await Promise.all(
this.#attachmentBackupJobs.map(job => {
if (job.type !== 'local') {
log.error(
"BackupExportStream: Can't enqueue remote backup jobs during local backup, skipping"
);
return Promise.resolve();
}
AttachmentLocalBackupManager.clearAllJobs();
await Promise.all(
this.#attachmentBackupJobs.map(job => {
if (job.type !== 'local') {
log.error(
"BackupExportStream: Can't enqueue remote backup jobs during local backup, skipping"
);
return Promise.resolve();
}
return AttachmentLocalBackupManager.addJob({
...job,
backupsBaseDir: localBackupsBaseDir,
});
})
);
drop(AttachmentLocalBackupManager.start());
} else {
// TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction
await DataWriter.clearAllAttachmentBackupJobs();
if (this.backupType !== BackupType.TestOnlyPlaintext) {
return AttachmentLocalBackupManager.addJob({
...job,
backupsBaseDir,
});
})
);
drop(AttachmentLocalBackupManager.start());
}
break;
case 'remote':
await DataWriter.clearAllAttachmentBackupJobs();
await Promise.all(
this.#attachmentBackupJobs.map(job => {
if (job.type === 'local') {
@@ -355,9 +357,13 @@ export class BackupExportStream extends Readable {
})
);
drop(AttachmentBackupManager.start());
}
break;
case 'cross-client-integration-test':
break;
default:
// eslint-disable-next-line no-unsafe-finally
throw missingCaseError(type);
}
log.info('BackupExportStream: finished');
}
})()
@@ -372,10 +378,7 @@ export class BackupExportStream extends Readable {
return this.#stats;
}
async #unsafeRun(
backupLevel: BackupLevel,
isLocalBackup: boolean
): Promise<void> {
async #unsafeRun(): Promise<void> {
this.#ourConversation =
window.ConversationController.getOurConversationOrThrow().attributes;
this.push(
@@ -752,8 +755,6 @@ export class BackupExportStream extends Readable {
const chatItem = await this.#toChatItem(message, {
aboutMe,
callHistoryByCallId,
backupLevel,
isLocalBackup,
});
if (chatItem === undefined) {
@@ -1214,12 +1215,7 @@ export class BackupExportStream extends Readable {
async #toChatItem(
message: MessageAttributesType,
{
aboutMe,
callHistoryByCallId,
backupLevel,
isLocalBackup,
}: ToChatItemOptionsType
{ aboutMe, callHistoryByCallId }: ToChatItemOptionsType
): Promise<Backups.IChatItem | undefined> {
const conversation = window.ConversationController.get(
message.conversationId
@@ -1378,8 +1374,6 @@ export class BackupExportStream extends Readable {
if (isTapToView(message)) {
result.viewOnceMessage = await this.#toViewOnceMessage({
message,
backupLevel,
isLocalBackup,
});
} else if (message.deletedForEveryone) {
result.remoteDeletedMessage = {};
@@ -1440,8 +1434,6 @@ export class BackupExportStream extends Readable {
avatar: contactDetails.avatar?.avatar
? await this.#processAttachment({
attachment: contactDetails.avatar.avatar,
backupLevel,
isLocalBackup,
messageReceivedAt: message.received_at,
})
: undefined,
@@ -1462,8 +1454,6 @@ export class BackupExportStream extends Readable {
stickerProto.data = sticker.data
? await this.#processAttachment({
attachment: sticker.data,
backupLevel,
isLocalBackup,
messageReceivedAt: message.received_at,
})
: undefined;
@@ -1506,29 +1496,15 @@ export class BackupExportStream extends Readable {
} else if (message.storyReplyContext) {
result.directStoryReplyMessage = await this.#toDirectStoryReplyMessage({
message,
backupLevel,
isLocalBackup,
});
result.revisions = await this.#toChatItemRevisions(
result,
message,
backupLevel,
isLocalBackup
);
result.revisions = await this.#toChatItemRevisions(result, message);
} else {
result.standardMessage = await this.#toStandardMessage({
message,
backupLevel,
isLocalBackup,
});
result.revisions = await this.#toChatItemRevisions(
result,
message,
backupLevel,
isLocalBackup
);
result.revisions = await this.#toChatItemRevisions(result, message);
}
if (isOutgoing) {
@@ -2446,12 +2422,8 @@ export class BackupExportStream extends Readable {
async #toQuote({
message,
backupLevel,
isLocalBackup,
}: {
message: Pick<MessageAttributesType, 'quote' | 'received_at' | 'body'>;
backupLevel: BackupLevel;
isLocalBackup: boolean;
}): Promise<Backups.IQuote | null> {
const { quote } = message;
if (!quote) {
@@ -2513,9 +2485,7 @@ export class BackupExportStream extends Readable {
thumbnail: attachment.thumbnail
? await this.#processMessageAttachment({
attachment: attachment.thumbnail,
backupLevel,
message,
isLocalBackup,
})
: undefined,
};
@@ -2572,21 +2542,15 @@ export class BackupExportStream extends Readable {
async #processMessageAttachment({
attachment,
backupLevel,
message,
isLocalBackup,
}: {
attachment: AttachmentType;
backupLevel: BackupLevel;
message: Pick<MessageAttributesType, 'quote' | 'received_at' | 'body'>;
isLocalBackup: boolean;
}): Promise<Backups.MessageAttachment> {
const { clientUuid } = attachment;
const filePointer = await this.#processAttachment({
attachment,
backupLevel,
messageReceivedAt: message.received_at,
isLocalBackup,
});
return new Backups.MessageAttachment({
@@ -2599,26 +2563,28 @@ export class BackupExportStream extends Readable {
async #processAttachment({
attachment,
backupLevel,
isLocalBackup,
messageReceivedAt,
}: {
attachment: AttachmentType;
backupLevel: BackupLevel;
isLocalBackup: boolean;
messageReceivedAt: number;
}): Promise<Backups.FilePointer> {
const { filePointer, backupJob } = await getFilePointerForAttachment({
attachment,
isLocalBackup,
backupLevel,
backupOptions: this.options,
messageReceivedAt,
getBackupCdnInfo,
});
if (hasRequiredInformationForBackup(attachment)) {
const mediaName = getMediaNameForAttachment(attachment);
let mediaName: string | undefined;
if (this.options.type === 'local-encrypted') {
if (hasRequiredInformationForLocalBackup(attachment)) {
mediaName = getLocalBackupFileNameForAttachment(attachment);
}
} else if (hasRequiredInformationForRemoteBackup(attachment)) {
mediaName = getMediaNameForAttachment(attachment);
}
if (mediaName) {
// Re-use existing locatorInfo and backup job if we've already seen this file
const existingFilePointer = this.#mediaNamesToFilePointers.get(mediaName);
@@ -2632,7 +2598,6 @@ export class BackupExportStream extends Readable {
if (filePointer.locatorInfo) {
this.#mediaNamesToFilePointers.set(mediaName, filePointer);
}
if (backupJob) {
this.#attachmentBackupJobs.push(backupJob);
}
@@ -2814,8 +2779,6 @@ export class BackupExportStream extends Readable {
async #toStandardMessage({
message,
backupLevel,
isLocalBackup,
}: {
message: Pick<
MessageAttributesType,
@@ -2829,8 +2792,6 @@ export class BackupExportStream extends Readable {
| 'received_at'
| 'timestamp'
>;
backupLevel: BackupLevel;
isLocalBackup: boolean;
}): Promise<Backups.IStandardMessage> {
if (
message.body &&
@@ -2842,17 +2803,13 @@ export class BackupExportStream extends Readable {
return {
quote: await this.#toQuote({
message,
backupLevel,
isLocalBackup,
}),
attachments: message.attachments?.length
? await Promise.all(
message.attachments.map(attachment => {
return this.#processMessageAttachment({
attachment,
backupLevel,
message,
isLocalBackup,
});
})
)
@@ -2863,8 +2820,6 @@ export class BackupExportStream extends Readable {
message.bodyAttachment && !isDownloaded(message.bodyAttachment)
? await this.#processAttachment({
attachment: message.bodyAttachment,
backupLevel,
isLocalBackup,
messageReceivedAt: message.received_at,
})
: undefined,
@@ -2890,8 +2845,6 @@ export class BackupExportStream extends Readable {
image: preview.image
? await this.#processAttachment({
attachment: preview.image,
backupLevel,
isLocalBackup,
messageReceivedAt: message.received_at,
})
: undefined,
@@ -2905,8 +2858,6 @@ export class BackupExportStream extends Readable {
async #toDirectStoryReplyMessage({
message,
backupLevel,
isLocalBackup,
}: {
message: Pick<
MessageAttributesType,
@@ -2917,8 +2868,6 @@ export class BackupExportStream extends Readable {
| 'received_at'
| 'reactions'
>;
backupLevel: BackupLevel;
isLocalBackup: boolean;
}): Promise<Backups.IDirectStoryReplyMessage> {
const result = new Backups.DirectStoryReplyMessage({
reactions: this.#getMessageReactions(message),
@@ -2931,8 +2880,6 @@ export class BackupExportStream extends Readable {
longText: message.bodyAttachment
? await this.#processAttachment({
attachment: message.bodyAttachment,
backupLevel,
isLocalBackup,
messageReceivedAt: message.received_at,
})
: undefined,
@@ -2952,15 +2899,11 @@ export class BackupExportStream extends Readable {
async #toViewOnceMessage({
message,
backupLevel,
isLocalBackup,
}: {
message: Pick<
MessageAttributesType,
'attachments' | 'received_at' | 'reactions'
>;
backupLevel: BackupLevel;
isLocalBackup: boolean;
}): Promise<Backups.IViewOnceMessage> {
const attachment = message.attachments?.at(0);
// Integration tests use the 'link-and-sync' version of export, which will include
@@ -2972,9 +2915,7 @@ export class BackupExportStream extends Readable {
? null
: await this.#processMessageAttachment({
attachment,
backupLevel,
message,
isLocalBackup,
}),
reactions: this.#getMessageReactions(message),
};
@@ -2982,9 +2923,7 @@ export class BackupExportStream extends Readable {
async #toChatItemRevisions(
parent: Backups.IChatItem,
message: MessageAttributesType,
backupLevel: BackupLevel,
isLocalBackup: boolean
message: MessageAttributesType
): Promise<Array<Backups.IChatItem> | undefined> {
const { editHistory } = message;
if (editHistory == null) {
@@ -3022,14 +2961,10 @@ export class BackupExportStream extends Readable {
result.directStoryReplyMessage =
await this.#toDirectStoryReplyMessage({
message: history,
backupLevel,
isLocalBackup,
});
} else {
result.standardMessage = await this.#toStandardMessage({
message: history,
backupLevel,
isLocalBackup,
});
}
return result;

View File

@@ -92,8 +92,11 @@ import { signalProtocolStore } from '../../SignalProtocolStore.preload.js';
import * as Bytes from '../../Bytes.std.js';
import { BACKUP_VERSION, WALLPAPER_TO_BUBBLE_COLOR } from './constants.std.js';
import { UnsupportedBackupVersion } from './errors.std.js';
import type { AboutMe, LocalChatStyle } from './types.std.js';
import { BackupType } from './types.std.js';
import type {
AboutMe,
BackupImportOptions,
LocalChatStyle,
} from './types.std.js';
import { getBackupMediaRootKey } from './crypto.preload.js';
import type { GroupV2ChangeDetailType } from '../../types/groups.std.js';
import { queueAttachmentDownloads } from '../../util/queueAttachmentDownloads.preload.js';
@@ -274,22 +277,18 @@ export class BackupImportStream extends Writable {
#frameErrorCount: number = 0;
#backupTier: BackupLevel | undefined;
private constructor(
private readonly backupType: BackupType,
private readonly localBackupSnapshotDir: string | undefined
) {
private constructor(private readonly options: BackupImportOptions) {
super({ objectMode: true });
}
public static async create(
backupType = BackupType.Ciphertext,
localBackupSnapshotDir: string | undefined = undefined
options: BackupImportOptions
): Promise<BackupImportStream> {
await AttachmentDownloadManager.stop();
await DataWriter.removeAllBackupAttachmentDownloadJobs();
await resetBackupMediaDownloadStats();
return new BackupImportStream(backupType, localBackupSnapshotDir);
return new BackupImportStream(options);
}
override async _write(
@@ -424,7 +423,7 @@ export class BackupImportStream extends Writable {
await pMap(
[...this.#pendingGroupAvatars.entries()],
async ([conversationId, newAvatarUrl]) => {
if (this.backupType === BackupType.TestOnlyPlaintext) {
if (this.options.type === 'cross-client-integration-test') {
return;
}
await groupAvatarJobQueue.add({ conversationId, newAvatarUrl });
@@ -446,7 +445,7 @@ export class BackupImportStream extends Writable {
);
if (
this.backupType !== BackupType.TestOnlyPlaintext &&
this.options.type !== 'cross-client-integration-test' &&
!isTestEnvironment(getEnvironment())
) {
await startBackupMediaDownload();
@@ -1906,17 +1905,14 @@ export class BackupImportStream extends Writable {
bodyRanges: this.#fromBodyRanges(data.text),
})),
bodyAttachment: data.longText
? convertFilePointerToAttachment(
data.longText,
this.#getFilePointerOptions()
)
? convertFilePointerToAttachment(data.longText, this.options)
: undefined,
attachments: data.attachments?.length
? data.attachments
.map(attachment =>
convertBackupMessageAttachmentToAttachment(
attachment,
this.#getFilePointerOptions()
this.options
)
)
.filter(isNotNil)
@@ -1962,10 +1958,7 @@ export class BackupImportStream extends Writable {
description: dropNull(preview.description),
date: getCheckedTimestampOrUndefinedFromLong(preview.date),
image: preview.image
? convertFilePointerToAttachment(
preview.image,
this.#getFilePointerOptions()
)
? convertFilePointerToAttachment(preview.image, this.options)
: undefined,
};
})
@@ -1984,7 +1977,7 @@ export class BackupImportStream extends Writable {
? [
convertBackupMessageAttachmentToAttachment(
attachment,
this.#getFilePointerOptions()
this.options
),
].filter(isNotNil)
: undefined,
@@ -2023,10 +2016,7 @@ export class BackupImportStream extends Writable {
result.body = textReply.text?.body ?? undefined;
result.bodyRanges = this.#fromBodyRanges(textReply.text);
result.bodyAttachment = textReply.longText
? convertFilePointerToAttachment(
textReply.longText,
this.#getFilePointerOptions()
)
? convertFilePointerToAttachment(textReply.longText, this.options)
: undefined;
} else if (emoji) {
result.storyReaction = {
@@ -2056,10 +2046,7 @@ export class BackupImportStream extends Writable {
body: textReply.text?.body ?? undefined,
bodyRanges: this.#fromBodyRanges(textReply.text),
bodyAttachment: textReply.longText
? convertFilePointerToAttachment(
textReply.longText,
this.#getFilePointerOptions()
)
? convertFilePointerToAttachment(textReply.longText, this.options)
: undefined,
};
}
@@ -2182,10 +2169,7 @@ export class BackupImportStream extends Writable {
? stringToMIMEType(contentType)
: APPLICATION_OCTET_STREAM,
thumbnail: thumbnail?.pointer
? convertFilePointerToAttachment(
thumbnail.pointer,
this.#getFilePointerOptions()
)
? convertFilePointerToAttachment(thumbnail.pointer, this.options)
: undefined,
};
}) ?? [],
@@ -2347,7 +2331,7 @@ export class BackupImportStream extends Writable {
? {
avatar: convertFilePointerToAttachment(
avatar,
this.#getFilePointerOptions()
this.options
),
isProfile: false,
}
@@ -2396,10 +2380,7 @@ export class BackupImportStream extends Writable {
packKey: Bytes.toBase64(packKey),
stickerId,
data: data
? convertFilePointerToAttachment(
data,
this.#getFilePointerOptions()
)
? convertFilePointerToAttachment(data, this.options)
: undefined,
},
reactions: this.#fromReactions(chatItem.stickerMessage.reactions),
@@ -3800,16 +3781,8 @@ export class BackupImportStream extends Writable {
};
}
#getFilePointerOptions() {
if (this.localBackupSnapshotDir != null) {
return { localBackupSnapshotDir: this.localBackupSnapshotDir };
}
return {};
}
#isLocalBackup() {
return this.localBackupSnapshotDir != null;
return this.options.type === 'local-encrypted';
}
#isMediaEnabledBackup() {

View File

@@ -69,7 +69,7 @@ import {
validateBackupStream,
ValidationType,
} from './validator.preload.js';
import { BackupType } from './types.std.js';
import type { BackupExportOptions, BackupImportOptions } from './types.std.js';
import {
BackupInstallerError,
BackupDownloadFailedError,
@@ -103,8 +103,6 @@ const { isEqual, noop } = lodash;
const log = createLogger('backupsService');
export { BackupType };
const IV_LENGTH = 16;
const BACKUP_REFRESH_INTERVAL = 24 * HOUR;
@@ -128,13 +126,6 @@ type DoDownloadOptionsType = Readonly<{
) => void;
}>;
export type ImportOptionsType = Readonly<{
backupType?: BackupType;
localBackupSnapshotDir?: string;
ephemeralKey?: Uint8Array;
onProgress?: (currentBytes: number, totalBytes: number) => void;
}>;
export type ExportResultType = Readonly<{
totalBytes: number;
duration: number;
@@ -318,7 +309,10 @@ export class BackupsService {
log.info(`exportBackup: starting, backup level: ${backupLevel}...`);
try {
const { totalBytes } = await this.exportToDisk(filePath, backupLevel);
const { totalBytes } = await this.exportToDisk(filePath, {
type: 'remote',
level: backupLevel,
});
await this.api.upload(filePath, totalBytes);
} finally {
@@ -331,8 +325,7 @@ export class BackupsService {
}
public async exportLocalBackup(
backupsBaseDir: string | undefined = undefined,
backupLevel: BackupLevel = BackupLevel.Free
backupsBaseDir: string | undefined = undefined
): Promise<LocalBackupExportResultType> {
strictAssert(isLocalBackupsEnabled(), 'Local backups must be enabled');
@@ -347,12 +340,10 @@ export class BackupsService {
log.info('exportLocalBackup: starting');
const exportResult = await this.exportToDisk(
mainProtoPath,
backupLevel,
BackupType.Ciphertext,
snapshotDir
);
const exportResult = await this.exportToDisk(mainProtoPath, {
type: 'local-encrypted',
localBackupSnapshotDir: snapshotDir,
});
log.info('exportLocalBackup: writing metadata');
const metadataArgs = {
@@ -405,6 +396,7 @@ export class BackupsService {
const backupFile = join(this.#localBackupSnapshotDir, 'main');
await this.importFromDisk(backupFile, {
type: 'local-encrypted',
localBackupSnapshotDir: this.#localBackupSnapshotDir,
});
@@ -421,14 +413,13 @@ export class BackupsService {
// Test harness
public async exportBackupData(
backupLevel: BackupLevel = BackupLevel.Free,
backupType = BackupType.Ciphertext
options: BackupExportOptions
): Promise<{ data: Uint8Array } & ExportResultType> {
const sink = new PassThrough();
const chunks = new Array<Uint8Array>();
sink.on('data', chunk => chunks.push(chunk));
const result = await this.#exportBackup(sink, backupLevel, backupType);
const result = await this.#exportBackup(sink, options);
return {
...result,
@@ -438,18 +429,14 @@ export class BackupsService {
public async exportToDisk(
path: string,
backupLevel: BackupLevel = BackupLevel.Free,
backupType = BackupType.Ciphertext,
localBackupSnapshotDir: string | undefined = undefined
options: BackupExportOptions
): Promise<ExportResultType> {
const exportResult = await this.#exportBackup(
createWriteStream(path),
backupLevel,
backupType,
localBackupSnapshotDir
options
);
if (backupType === BackupType.Ciphertext) {
if (options.type !== 'cross-client-integration-test') {
await validateBackup(
() => new FileStream(path),
exportResult.totalBytes,
@@ -462,9 +449,7 @@ export class BackupsService {
return exportResult;
}
public async _internalExportLocalBackup(
backupLevel: BackupLevel = BackupLevel.Free
): Promise<ValidationResultType> {
public async _internalExportLocalBackup(): Promise<ValidationResultType> {
try {
const { canceled, dirPath: backupsBaseDir } = await ipcRenderer.invoke(
'show-open-folder-dialog'
@@ -473,7 +458,7 @@ export class BackupsService {
return { error: 'Backups directory not selected' };
}
const result = await this.exportLocalBackup(backupsBaseDir, backupLevel);
const result = await this.exportLocalBackup(backupsBaseDir);
return { result };
} catch (error) {
return { error: Errors.toLogFormat(error) };
@@ -496,16 +481,16 @@ export class BackupsService {
}
// Test harness
public async _internalValidate(
backupLevel: BackupLevel = BackupLevel.Free,
backupType = BackupType.Ciphertext
): Promise<ValidationResultType> {
public async _internalValidate(): Promise<ValidationResultType> {
try {
const start = Date.now();
const recordStream = new BackupExportStream(backupType);
const recordStream = new BackupExportStream({
type: 'remote',
level: BackupLevel.Free,
});
recordStream.run(backupLevel);
recordStream.run();
const totalBytes = await validateBackupStream(recordStream);
@@ -521,7 +506,10 @@ export class BackupsService {
// Test harness
public async exportWithDialog(): Promise<void> {
const { data } = await this.exportBackupData();
const { data } = await this.exportBackupData({
type: 'remote',
level: BackupLevel.Free,
});
await saveAttachmentToDisk({
name: 'backup.bin',
@@ -531,7 +519,7 @@ export class BackupsService {
public async importFromDisk(
backupFile: string,
options?: ImportOptionsType
options: BackupImportOptions
): Promise<void> {
return this.importBackup(() => createReadStream(backupFile), options);
}
@@ -562,18 +550,13 @@ export class BackupsService {
public async importBackup(
createBackupStream: () => Readable,
{
backupType = BackupType.Ciphertext,
ephemeralKey,
onProgress,
localBackupSnapshotDir = undefined,
}: ImportOptionsType = {}
options: BackupImportOptions
): Promise<void> {
strictAssert(!this.#isRunning, 'BackupService is already running');
window.IPC.startTrackingQueryStats();
log.info(`importBackup: starting ${backupType}...`);
log.info(`importBackup: starting ${options.type}...`);
this.#isRunning = 'import';
const importStart = Date.now();
@@ -588,13 +571,10 @@ export class BackupsService {
window.ConversationController.setReadOnly(true);
const importStream = await BackupImportStream.create(
backupType,
localBackupSnapshotDir
);
if (backupType === BackupType.Ciphertext) {
const importStream = await BackupImportStream.create(options);
if (options.type === 'remote' || options.type === 'local-encrypted') {
const { aesKey, macKey } = getKeyMaterial(
ephemeralKey ? new BackupKey(ephemeralKey) : undefined
options.ephemeralKey ? new BackupKey(options.ephemeralKey) : undefined
);
// First pass - don't decrypt, only verify mac
@@ -621,7 +601,7 @@ export class BackupsService {
throw new BackupImportCanceledError();
}
onProgress?.(0, totalBytes);
options.onProgress?.(0, totalBytes);
strictAssert(theirMac != null, 'importBackup: Missing MAC');
strictAssert(
@@ -638,7 +618,7 @@ export class BackupsService {
let currentBytes = 0;
progressReporter.on('data', chunk => {
currentBytes += chunk.byteLength;
onProgress?.(currentBytes, totalBytes);
options.onProgress?.(currentBytes, totalBytes);
});
await pipeline(
@@ -656,13 +636,13 @@ export class BackupsService {
constantTimeEqual(hmac.digest(), theirMac),
'importBackup: Bad MAC, second pass'
);
} else if (backupType === BackupType.TestOnlyPlaintext) {
} else if (options.type === 'cross-client-integration-test') {
strictAssert(
isTestOrMockEnvironment(),
'Plaintext backups can be imported only in test harness'
);
strictAssert(
ephemeralKey == null,
options.ephemeralKey == null,
'Plaintext backups cannot have ephemeral key'
);
await pipeline(
@@ -671,7 +651,7 @@ export class BackupsService {
importStream
);
} else {
throw missingCaseError(backupType);
throw missingCaseError(options.type);
}
log.info('importBackup: finished...');
@@ -876,6 +856,7 @@ export class BackupsService {
await itemStorage.remove('password');
await this.importFromDisk(downloadPath, {
type: 'remote',
ephemeralKey,
onProgress: (currentBytes, totalBytes) => {
onProgress?.(
@@ -913,9 +894,7 @@ export class BackupsService {
async #exportBackup(
sink: Writable,
backupLevel: BackupLevel = BackupLevel.Free,
backupType = BackupType.Ciphertext,
localBackupSnapshotDir: string | undefined = undefined
options: BackupExportOptions
): Promise<ExportResultType> {
strictAssert(!this.#isRunning, 'BackupService is already running');
@@ -925,7 +904,7 @@ export class BackupsService {
const start = Date.now();
try {
// TODO (DESKTOP-7168): Update mock-server to support this endpoint
if (window.SignalCI || backupType === BackupType.TestOnlyPlaintext) {
if (window.SignalCI || options.type === 'cross-client-integration-test') {
strictAssert(
isTestOrMockEnvironment(),
'Plaintext backups can be exported only in test harness'
@@ -938,47 +917,52 @@ export class BackupsService {
}
const { aesKey, macKey } = getKeyMaterial();
const recordStream = new BackupExportStream(backupType);
const recordStream = new BackupExportStream(options);
recordStream.run(backupLevel, localBackupSnapshotDir);
recordStream.run();
const iv = randomBytes(IV_LENGTH);
let totalBytes = 0;
if (backupType === BackupType.Ciphertext) {
await pipeline(
recordStream,
createGzip(),
appendPaddingStream(),
createCipheriv(CipherType.AES256CBC, aesKey, iv),
prependStream(iv),
appendMacStream(macKey),
measureSize({
onComplete: size => {
totalBytes = size;
},
}),
sink
);
} else if (backupType === BackupType.TestOnlyPlaintext) {
strictAssert(
isTestOrMockEnvironment(),
'Plaintext backups can be exported only in test harness'
);
await pipeline(recordStream, sink);
} else {
throw missingCaseError(backupType);
const { type } = options;
switch (type) {
case 'remote':
case 'local-encrypted':
await pipeline(
recordStream,
createGzip(),
appendPaddingStream(),
createCipheriv(CipherType.AES256CBC, aesKey, iv),
prependStream(iv),
appendMacStream(macKey),
measureSize({
onComplete: size => {
totalBytes = size;
},
}),
sink
);
break;
case 'cross-client-integration-test':
strictAssert(
isTestOrMockEnvironment(),
'Plaintext backups can be exported only in test harness'
);
await pipeline(recordStream, sink);
break;
default:
throw missingCaseError(type);
}
if (localBackupSnapshotDir) {
if (type === 'local-encrypted') {
log.info('exportBackup: writing local backup files list');
const filesWritten = await writeLocalBackupFilesList({
snapshotDir: localBackupSnapshotDir,
snapshotDir: options.localBackupSnapshotDir,
mediaNamesIterator: recordStream.getMediaNamesIterator(),
});
const filesRead = await readLocalBackupFilesList(
localBackupSnapshotDir
options.localBackupSnapshotDir
);
strictAssert(
isEqual(filesWritten, filesRead),

View File

@@ -25,10 +25,22 @@ export type AboutMe = {
e164?: string;
};
export enum BackupType {
Ciphertext = 'Ciphertext',
TestOnlyPlaintext = 'TestOnlyPlaintext',
}
export type BackupExportOptions =
| { type: 'remote' | 'cross-client-integration-test'; level: BackupLevel }
| {
type: 'local-encrypted';
localBackupSnapshotDir: string;
};
export type BackupImportOptions = (
| { type: 'remote' | 'cross-client-integration-test' }
| {
type: 'local-encrypted';
localBackupSnapshotDir: string;
}
) & {
ephemeralKey?: Uint8Array;
onProgress?: (currentBytes: number, totalBytes: number) => void;
};
export type LocalChatStyle = Readonly<{
wallpaperPhotoPointer: Uint8Array | undefined;

View File

@@ -1,17 +1,16 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { existsSync } from 'node:fs';
import { BackupLevel } from '@signalapp/libsignal-client/zkgroup.js';
import {
APPLICATION_OCTET_STREAM,
stringToMIMEType,
} from '../../../types/MIME.std.js';
import { createLogger } from '../../../logging/log.std.js';
import type { AttachmentType } from '../../../types/Attachment.std.js';
import { getAbsoluteAttachmentPath } from '../../../util/migrations.preload.js';
import { doesAttachmentExist } from '../../../util/migrations.preload.js';
import {
hasRequiredInformationForBackup,
hasRequiredInformationForLocalBackup,
hasRequiredInformationForRemoteBackup,
hasRequiredInformationToDownloadFromTransitTier,
} from '../../../util/Attachment.std.js';
import { Backups, SignalService } from '../../../protobuf/index.std.js';
@@ -29,6 +28,10 @@ import {
type GetBackupCdnInfoType,
getMediaIdFromMediaName,
getMediaName,
getMediaNameForAttachment,
type BackupCdnInfoType,
getLocalBackupFileNameForAttachment,
getLocalBackupFileName,
} from './mediaId.preload.js';
import { missingCaseError } from '../../../util/missingCaseError.std.js';
import { bytesToUuid } from '../../../util/uuidToBytes.std.js';
@@ -37,21 +40,16 @@ import { generateAttachmentKeys } from '../../../AttachmentCrypto.node.js';
import { getAttachmentLocalBackupPathFromSnapshotDir } from './localBackup.node.js';
import {
isValidAttachmentKey,
isValidDigest,
isValidPlaintextHash,
} from '../../../types/Crypto.std.js';
import type { BackupExportOptions, BackupImportOptions } from '../types.std.js';
import { isTestOrMockEnvironment } from '../../../environment.std.js';
const log = createLogger('filePointers');
type ConvertFilePointerToAttachmentOptions = {
// Only for testing
_createName: (suffix?: string) => string;
localBackupSnapshotDir: string | undefined;
};
export function convertFilePointerToAttachment(
filePointer: Backups.FilePointer,
options: Partial<ConvertFilePointerToAttachmentOptions> = {}
options: BackupImportOptions,
testDependencies?: { _createName: (suffix?: string) => string }
): AttachmentType {
const {
contentType,
@@ -64,7 +62,7 @@ export function convertFilePointerToAttachment(
incrementalMacChunkSize,
locatorInfo,
} = filePointer;
const doCreateName = options._createName ?? createName;
const doCreateName = testDependencies?._createName ?? createName;
const commonProps: AttachmentType = {
size: 0,
@@ -124,24 +122,16 @@ export function convertFilePointerToAttachment(
}
let localBackupPath: string | undefined;
if (Bytes.isNotEmpty(localKey)) {
const { localBackupSnapshotDir } = options;
strictAssert(
localBackupSnapshotDir,
'localBackupSnapshotDir is required for filePointer.localLocator'
if (
options.type === 'local-encrypted' &&
Bytes.isNotEmpty(localKey) &&
Bytes.isNotEmpty(plaintextHash)
) {
const localMediaName = getLocalBackupFileName({ plaintextHash, localKey });
localBackupPath = getAttachmentLocalBackupPathFromSnapshotDir(
localMediaName,
options.localBackupSnapshotDir
);
if (mediaName) {
localBackupPath = getAttachmentLocalBackupPathFromSnapshotDir(
mediaName,
localBackupSnapshotDir
);
} else {
log.error(
'convertFilePointerToAttachment: localKey but no plaintextHash'
);
}
}
return {
@@ -172,7 +162,7 @@ export function convertFilePointerToAttachment(
export function convertBackupMessageAttachmentToAttachment(
messageAttachment: Backups.IMessageAttachment,
options: Partial<ConvertFilePointerToAttachmentOptions> = {}
options: BackupImportOptions
): AttachmentType | null {
const { clientUuid } = messageAttachment;
@@ -207,21 +197,21 @@ export function convertBackupMessageAttachmentToAttachment(
}
export async function getFilePointerForAttachment({
attachment,
attachment: rawAttachment,
getBackupCdnInfo,
backupLevel,
backupOptions,
messageReceivedAt,
isLocalBackup = false,
}: {
attachment: Readonly<AttachmentType>;
getBackupCdnInfo: GetBackupCdnInfoType;
backupLevel: BackupLevel;
backupOptions: BackupExportOptions;
messageReceivedAt: number;
isLocalBackup?: boolean;
}): Promise<{
filePointer: Backups.FilePointer;
backupJob?: CoreAttachmentBackupJobType | PartialAttachmentLocalBackupJobType;
}> {
const attachment = maybeFixupAttachment(rawAttachment);
const filePointer = new Backups.FilePointer({
contentType: attachment.contentType,
fileName: attachment.fileName,
@@ -241,67 +231,79 @@ export async function getFilePointerForAttachment({
}
}
const locatorInfo = getLocatorInfoForAttachment({
const isAttachmentOnDisk =
attachment.path != null && (await doesAttachmentExist(attachment.path));
const remoteMediaName = hasRequiredInformationForRemoteBackup(attachment)
? getMediaNameForAttachment(attachment)
: undefined;
const remoteMediaId = remoteMediaName
? getMediaIdFromMediaName(remoteMediaName)
: undefined;
const remoteBackupStatus: BackupCdnInfoType = remoteMediaId
? await getBackupCdnInfo(remoteMediaId.string)
: { isInBackupTier: false };
const isLocalBackup = backupOptions.type === 'local-encrypted';
filePointer.locatorInfo = getLocatorInfoForAttachment({
attachment,
isLocalBackup,
backupOptions,
isOnDisk: isAttachmentOnDisk,
backupTierInfo: remoteBackupStatus,
});
if (locatorInfo) {
filePointer.locatorInfo = locatorInfo;
}
let backupJob:
| CoreAttachmentBackupJobType
| PartialAttachmentLocalBackupJobType
| undefined;
if (backupLevel !== BackupLevel.Paid && !isLocalBackup) {
return { filePointer, backupJob: undefined };
}
if (!Bytes.isNotEmpty(locatorInfo.plaintextHash)) {
return { filePointer, backupJob: undefined };
}
const mediaName = getMediaName({
plaintextHash: locatorInfo.plaintextHash,
key: locatorInfo.key,
});
const backupInfo = await getBackupCdnInfo(
getMediaIdFromMediaName(mediaName).string
);
if (backupInfo.isInBackupTier) {
if (locatorInfo.mediaTierCdnNumber !== backupInfo.cdnNumber) {
log.warn(
'backupCdnNumber on attachment differs from cdnNumber from list endpoint'
);
// Prefer the one from the list endpoint
locatorInfo.mediaTierCdnNumber = backupInfo.cdnNumber;
}
return { filePointer, backupJob: undefined };
}
const { path, localKey, version, size } = attachment;
if (!path || !isValidAttachmentKey(localKey)) {
return { filePointer, backupJob: undefined };
}
if (isLocalBackup) {
backupJob = {
mediaName,
type: 'local',
data: {
path,
size,
localKey,
},
if (
isAttachmentOnDisk &&
hasRequiredInformationForLocalBackup(attachment)
) {
return {
filePointer,
backupJob: {
mediaName: getLocalBackupFileNameForAttachment(attachment),
type: 'local',
data: {
path: attachment.path,
},
},
};
}
return {
filePointer,
backupJob: undefined,
};
} else {
backupJob = {
mediaName,
}
if (backupOptions.level !== BackupLevel.Paid) {
return { filePointer, backupJob: undefined };
}
if (remoteBackupStatus.isInBackupTier) {
return { filePointer, backupJob: undefined };
}
if (!isAttachmentOnDisk) {
return { filePointer, backupJob: undefined };
}
if (!remoteMediaName) {
return { filePointer, backupJob: undefined };
}
const { path, localKey, key, version } = attachment;
strictAssert(path, 'Path must exist for attachment on disk');
strictAssert(key, 'Key must exist for remote backupable attachment');
const { transitCdnKey, transitCdnNumber, transitTierUploadTimestamp } =
filePointer.locatorInfo;
return {
filePointer,
backupJob: {
mediaName: remoteMediaName,
receivedAt: messageReceivedAt,
type: 'standard',
data: {
@@ -309,74 +311,86 @@ export async function getFilePointerForAttachment({
localKey,
version,
contentType: attachment.contentType,
keys: Bytes.toBase64(locatorInfo.key),
size: locatorInfo.size,
keys: key,
size: attachment.size,
transitCdnInfo:
locatorInfo.transitCdnKey && locatorInfo.transitCdnNumber != null
transitCdnKey && transitCdnNumber != null
? {
cdnKey: locatorInfo.transitCdnKey,
cdnNumber: locatorInfo.transitCdnNumber,
uploadTimestamp:
locatorInfo.transitTierUploadTimestamp?.toNumber(),
cdnKey: transitCdnKey,
cdnNumber: transitCdnNumber,
uploadTimestamp: transitTierUploadTimestamp?.toNumber(),
}
: undefined,
},
};
}
return { filePointer, backupJob };
},
};
}
function maybeFixupAttachment(attachment: AttachmentType): AttachmentType {
// Fixup attachment which has plaintextHash but no key
if (
isValidPlaintextHash(attachment.plaintextHash) &&
!isValidAttachmentKey(attachment.key)
) {
const fixedUpAttachment = { ...attachment };
fixedUpAttachment.key = Bytes.toBase64(generateAttachmentKeys());
// Delete all info dependent on key
delete fixedUpAttachment.cdnKey;
delete fixedUpAttachment.cdnNumber;
delete fixedUpAttachment.uploadTimestamp;
delete fixedUpAttachment.digest;
delete fixedUpAttachment.backupCdnNumber;
strictAssert(
hasRequiredInformationForRemoteBackup(fixedUpAttachment),
'should be backupable with new key'
);
return fixedUpAttachment;
}
return attachment;
}
function getLocatorInfoForAttachment({
attachment: _rawAttachment,
isLocalBackup,
attachment,
backupOptions,
isOnDisk,
backupTierInfo,
}: {
attachment: AttachmentType;
isLocalBackup: boolean;
backupOptions: BackupExportOptions;
isOnDisk: boolean;
backupTierInfo: BackupCdnInfoType;
}): Backups.FilePointer.LocatorInfo {
const locatorInfo = new Backups.FilePointer.LocatorInfo();
const attachment = { ..._rawAttachment };
if (attachment.error) {
return locatorInfo;
}
const isLocalBackup = backupOptions.type === 'local-encrypted';
{
const isBackupable = hasRequiredInformationForBackup(attachment);
const isDownloadableFromTransitTier =
hasRequiredInformationToDownloadFromTransitTier(attachment);
const shouldBeLocallyBackedUp =
isLocalBackup &&
isOnDisk &&
hasRequiredInformationForLocalBackup(attachment);
if (!isBackupable && !isDownloadableFromTransitTier) {
// TODO: DESKTOP-8914
if (
isValidPlaintextHash(attachment.plaintextHash) &&
!isValidAttachmentKey(attachment.key)
) {
attachment.key = Bytes.toBase64(generateAttachmentKeys());
// Delete all info dependent on key
delete attachment.cdnKey;
delete attachment.cdnNumber;
delete attachment.uploadTimestamp;
delete attachment.digest;
delete attachment.backupCdnNumber;
strictAssert(
hasRequiredInformationForBackup(attachment),
'should be backupable with new key'
);
}
}
}
const isBackupable = hasRequiredInformationForBackup(attachment);
const isDownloadableFromTransitTier =
hasRequiredInformationToDownloadFromTransitTier(attachment);
if (!isBackupable && !isDownloadableFromTransitTier) {
if (
!shouldBeLocallyBackedUp &&
!isDownloadableFromTransitTier &&
!hasRequiredInformationForRemoteBackup(attachment)
) {
return locatorInfo;
}
locatorInfo.size = attachment.size;
locatorInfo.key = Bytes.fromBase64(attachment.key);
if (isValidAttachmentKey(attachment.key)) {
locatorInfo.key = Bytes.fromBase64(attachment.key);
}
if (isValidPlaintextHash(attachment.plaintextHash)) {
locatorInfo.plaintextHash = Bytes.fromHex(attachment.plaintextHash);
} else if (isValidDigest(attachment.digest)) {
locatorInfo.encryptedDigest = Bytes.fromBase64(attachment.digest);
}
if (isDownloadableFromTransitTier) {
locatorInfo.transitCdnKey = attachment.cdnKey;
@@ -386,25 +400,14 @@ function getLocatorInfoForAttachment({
);
}
if (isBackupable) {
locatorInfo.plaintextHash = Bytes.fromHex(attachment.plaintextHash);
// TODO: DESKTOP-8887
if (attachment.backupCdnNumber != null) {
locatorInfo.mediaTierCdnNumber = attachment.backupCdnNumber;
}
} else {
locatorInfo.encryptedDigest = Bytes.fromBase64(attachment.digest);
if (shouldBeLocallyBackedUp) {
locatorInfo.localKey = Bytes.fromBase64(attachment.localKey);
}
// TODO: DESKTOP-8904
if (isLocalBackup && isBackupable) {
const attachmentExistsLocally =
attachment.path != null &&
existsSync(getAbsoluteAttachmentPath(attachment.path));
if (attachmentExistsLocally && attachment.localKey) {
locatorInfo.localKey = Bytes.fromBase64(attachment.localKey);
}
if (backupTierInfo.isInBackupTier && backupTierInfo.cdnNumber != null) {
locatorInfo.mediaTierCdnNumber = backupTierInfo.cdnNumber;
} else if (backupOptions.type === 'cross-client-integration-test') {
locatorInfo.mediaTierCdnNumber = attachment.backupCdnNumber;
}
return locatorInfo;

View File

@@ -4,7 +4,11 @@
import { DataReader } from '../../../sql/Client.preload.js';
import * as Bytes from '../../../Bytes.std.js';
import { getBackupMediaRootKey } from '../crypto.preload.js';
import type { BackupableAttachmentType } from '../../../types/Attachment.std.js';
import type {
BackupableAttachmentType,
AttachmentReadyForLocalBackup,
} from '../../../types/Attachment.std.js';
import { sha256 } from '../../../Crypto.node.js';
export function getMediaIdFromMediaName(mediaName: string): {
string: string;
@@ -56,6 +60,25 @@ export function getMediaName({
return Bytes.toHex(Bytes.concatenate([plaintextHash, key]));
}
export function getLocalBackupFileNameForAttachment(
attachment: AttachmentReadyForLocalBackup
): string {
return getLocalBackupFileName({
plaintextHash: Bytes.fromHex(attachment.plaintextHash),
localKey: Bytes.fromBase64(attachment.localKey),
});
}
export function getLocalBackupFileName({
plaintextHash,
localKey,
}: {
plaintextHash: Uint8Array;
localKey: Uint8Array;
}): string {
return Bytes.toHex(sha256(Bytes.concatenate([plaintextHash, localKey])));
}
export function getMediaNameForAttachmentThumbnail(
fullsizeMediaName: string
): `${string}_thumbnail` {

View File

@@ -26,7 +26,7 @@ import type {
QuotedMessageType,
} from '../../model-types.d.ts';
import {
hasRequiredInformationForBackup,
hasRequiredInformationForRemoteBackup,
isVoiceMessage,
} from '../../util/Attachment.std.js';
import type { AttachmentType } from '../../types/Attachment.std.js';
@@ -128,7 +128,7 @@ describe('backup/attachments', () => {
attachment: AttachmentType
): AttachmentType {
const base = omit(attachment, NON_ROUNDTRIPPED_FIELDS);
if (hasRequiredInformationForBackup(attachment)) {
if (hasRequiredInformationForRemoteBackup(attachment)) {
delete base.digest;
} else {
delete base.plaintextHash;
@@ -793,7 +793,7 @@ describe('backup/attachments', () => {
strictAssert(key, 'thumbnail key was created');
strictAssert(plaintextHash, 'quote plaintextHash was roundtripped');
strictAssert(
hasRequiredInformationForBackup(thumbnail),
hasRequiredInformationForRemoteBackup(thumbnail),
'has key and plaintextHash'
);
assert.deepStrictEqual(thumbnail, {

View File

@@ -6,6 +6,7 @@ import * as sinon from 'sinon';
import { BackupLevel } from '@signalapp/libsignal-client/zkgroup.js';
import { randomBytes } from 'node:crypto';
import { join } from 'node:path';
import { emptyDir, ensureFile } from 'fs-extra';
import { Backups } from '../../protobuf/index.std.js';
@@ -22,6 +23,9 @@ import type { GetBackupCdnInfoType } from '../../services/backups/util/mediaId.p
import { strictAssert } from '../../util/assert.std.js';
import { isValidAttachmentKey } from '../../types/Crypto.std.js';
import { itemStorage } from '../../textsecure/Storage.preload.js';
import { getAbsoluteAttachmentPath } from '../../util/migrations.preload.js';
import { getPath } from '../../../app/attachments.node.js';
import { sha256 } from '../../Crypto.node.js';
describe('convertFilePointerToAttachment', () => {
const commonFilePointerProps = {
@@ -52,6 +56,7 @@ describe('convertFilePointerToAttachment', () => {
...commonFilePointerProps,
locatorInfo: {},
}),
{ type: 'remote' },
{ _createName: () => 'downloadPath' }
);
@@ -65,6 +70,7 @@ describe('convertFilePointerToAttachment', () => {
it('processes filepointer with missing locatorInfo', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer(commonFilePointerProps),
{ type: 'remote' },
{ _createName: () => 'downloadPath' }
);
@@ -90,6 +96,7 @@ describe('convertFilePointerToAttachment', () => {
mediaTierCdnNumber: 43,
},
}),
{ type: 'remote' },
{ _createName: () => 'downloadPath' }
);
@@ -123,17 +130,19 @@ describe('convertFilePointerToAttachment', () => {
localKey: Bytes.fromString('localKey'),
},
}),
{ type: 'local-encrypted', localBackupSnapshotDir: '/root/backups' },
{
_createName: () => 'downloadPath',
localBackupSnapshotDir: '/root/backups',
}
);
const mediaName = Bytes.toHex(
Bytes.concatenate([
Bytes.fromString('plaintextHash'),
Bytes.fromString('key'),
])
sha256(
Bytes.concatenate([
Bytes.fromString('plaintextHash'),
Bytes.fromString('localKey'),
])
)
);
assert.deepStrictEqual(result, {
...commonAttachmentProps,
@@ -207,7 +216,7 @@ const notInBackupCdn: GetBackupCdnInfoType = async () => {
describe('getFilePointerForAttachment', () => {
let sandbox: sinon.SinonSandbox;
beforeEach(() => {
beforeEach(async () => {
sandbox = sinon.createSandbox();
sandbox.stub(itemStorage, 'get').callsFake(key => {
if (key === 'masterKey') {
@@ -218,16 +227,21 @@ describe('getFilePointerForAttachment', () => {
}
return undefined;
});
await ensureFile(getAbsoluteAttachmentPath(defaultAttachment.path));
});
afterEach(() => {
afterEach(async () => {
sandbox.restore();
await emptyDir(getPath(window.SignalContext.config.userDataPath));
});
it('if missing key, generates a new one and removes transit info & digest', async () => {
const { filePointer } = await getFilePointerForAttachment({
attachment: { ...defaultAttachment, key: undefined },
backupLevel: BackupLevel.Paid,
backupOptions: {
type: 'remote',
level: BackupLevel.Paid,
},
getBackupCdnInfo: notInBackupCdn,
messageReceivedAt: 100,
});
@@ -254,7 +268,10 @@ describe('getFilePointerForAttachment', () => {
assert.deepEqual(
await getFilePointerForAttachment({
attachment: { ...defaultAttachment, plaintextHash: undefined },
backupLevel: BackupLevel.Paid,
backupOptions: {
type: 'remote',
level: BackupLevel.Paid,
},
getBackupCdnInfo: notInBackupCdn,
messageReceivedAt: 100,
}),
@@ -278,7 +295,10 @@ describe('getFilePointerForAttachment', () => {
assert.deepEqual(
await getFilePointerForAttachment({
attachment: defaultAttachment,
backupLevel: BackupLevel.Free,
backupOptions: {
type: 'remote',
level: BackupLevel.Free,
},
getBackupCdnInfo: notInBackupCdn,
messageReceivedAt: 100,
}),
@@ -292,7 +312,6 @@ describe('getFilePointerForAttachment', () => {
transitCdnKey: 'cdnKey',
transitCdnNumber: 2,
transitTierUploadTimestamp: Long.fromNumber(1234),
mediaTierCdnNumber: 42,
}),
}),
backupJob: undefined,
@@ -304,7 +323,10 @@ describe('getFilePointerForAttachment', () => {
assert.deepEqual(
await getFilePointerForAttachment({
attachment: { ...defaultAttachment, digest: undefined },
backupLevel: BackupLevel.Free,
backupOptions: {
type: 'remote',
level: BackupLevel.Free,
},
getBackupCdnInfo: notInBackupCdn,
messageReceivedAt: 100,
}),
@@ -318,7 +340,6 @@ describe('getFilePointerForAttachment', () => {
transitCdnKey: 'cdnKey',
transitCdnNumber: 2,
transitTierUploadTimestamp: Long.fromNumber(1234),
mediaTierCdnNumber: 42,
}),
}),
backupJob: undefined,
@@ -330,7 +351,10 @@ describe('getFilePointerForAttachment', () => {
assert.deepEqual(
await getFilePointerForAttachment({
attachment: { ...defaultAttachment, cdnKey: undefined },
backupLevel: BackupLevel.Free,
backupOptions: {
type: 'remote',
level: BackupLevel.Free,
},
getBackupCdnInfo: notInBackupCdn,
messageReceivedAt: 100,
}),
@@ -341,7 +365,6 @@ describe('getFilePointerForAttachment', () => {
plaintextHash: Bytes.fromHex(defaultAttachment.plaintextHash),
key: Bytes.fromBase64(defaultAttachment.key),
size: 100,
mediaTierCdnNumber: 42,
}),
}),
backupJob: undefined,
@@ -352,7 +375,10 @@ describe('getFilePointerForAttachment', () => {
assert.deepEqual(
await getFilePointerForAttachment({
attachment: defaultAttachment,
backupLevel: BackupLevel.Paid,
backupOptions: {
type: 'remote',
level: BackupLevel.Paid,
},
getBackupCdnInfo: notInBackupCdn,
messageReceivedAt: 100,
}),
@@ -366,7 +392,6 @@ describe('getFilePointerForAttachment', () => {
transitCdnKey: 'cdnKey',
transitCdnNumber: 2,
transitTierUploadTimestamp: Long.fromNumber(1234),
mediaTierCdnNumber: 42,
}),
}),
backupJob: {
@@ -390,14 +415,16 @@ describe('getFilePointerForAttachment', () => {
}
);
});
it('if local backup includes local backup job', async () => {
it('does not include backup job if file does not exist', async () => {
assert.deepEqual(
await getFilePointerForAttachment({
attachment: defaultAttachment,
backupLevel: BackupLevel.Paid,
attachment: { ...defaultAttachment, path: 'not/here' },
backupOptions: {
type: 'remote',
level: BackupLevel.Paid,
},
getBackupCdnInfo: notInBackupCdn,
messageReceivedAt: 100,
isLocalBackup: true,
}),
{
filePointer: new FilePointer({
@@ -409,19 +436,82 @@ describe('getFilePointerForAttachment', () => {
transitCdnKey: 'cdnKey',
transitCdnNumber: 2,
transitTierUploadTimestamp: Long.fromNumber(1234),
mediaTierCdnNumber: 42,
}),
}),
backupJob: {
data: {
localKey: defaultAttachment.localKey,
path: defaultAttachment.path,
size: 100,
},
mediaName: defaultMediaName,
type: 'local',
},
backupJob: undefined,
}
);
});
describe('local backups', () => {
const defaultLocalMediaName = Bytes.toHex(
sha256(
Bytes.concatenate([
Bytes.fromHex(defaultAttachment.plaintextHash),
Bytes.fromBase64(defaultAttachment.localKey),
])
)
);
it('generates local backup locatorInfo and a local backup job', async () => {
assert.deepEqual(
await getFilePointerForAttachment({
attachment: defaultAttachment,
backupOptions: {
type: 'local-encrypted',
localBackupSnapshotDir: '/root/backups',
},
getBackupCdnInfo: notInBackupCdn,
messageReceivedAt: 100,
}),
{
filePointer: new FilePointer({
...defaultFilePointer,
locatorInfo: new LocatorInfo({
plaintextHash: Bytes.fromHex(defaultAttachment.plaintextHash),
localKey: Bytes.fromBase64(defaultAttachment.localKey),
key: Bytes.fromBase64(defaultAttachment.key),
size: 100,
transitCdnKey: 'cdnKey',
transitCdnNumber: 2,
transitTierUploadTimestamp: Long.fromNumber(1234),
}),
}),
backupJob: {
data: {
path: defaultAttachment.path,
},
mediaName: defaultLocalMediaName,
type: 'local',
},
}
);
});
it('if file does not exist, does not include localKey or backup job', async () => {
assert.deepEqual(
await getFilePointerForAttachment({
attachment: { ...defaultAttachment, path: 'no/file/here' },
backupOptions: {
type: 'local-encrypted',
localBackupSnapshotDir: '/root/backups',
},
getBackupCdnInfo: notInBackupCdn,
messageReceivedAt: 100,
}),
{
filePointer: new FilePointer({
...defaultFilePointer,
locatorInfo: new LocatorInfo({
plaintextHash: Bytes.fromHex(defaultAttachment.plaintextHash),
key: Bytes.fromBase64(defaultAttachment.key),
size: 100,
transitCdnKey: 'cdnKey',
transitCdnNumber: 2,
transitTierUploadTimestamp: Long.fromNumber(1234),
}),
}),
backupJob: undefined,
}
);
});
});
});

View File

@@ -228,13 +228,19 @@ export async function asymmetricRoundtripHarness(
postSaveUpdates,
});
await backupsService.exportToDisk(targetOutputFile, options.backupLevel);
await backupsService.exportToDisk(targetOutputFile, {
type: 'remote',
level: options.backupLevel,
});
await updateConvoIdToTitle();
await clearData();
await backupsService.importBackup(() => createReadStream(targetOutputFile));
await backupsService.importBackup(
() => createReadStream(targetOutputFile),
{ type: 'remote' }
);
const messagesFromDatabase = await DataReader._getAllMessages();

View File

@@ -14,10 +14,7 @@ import { assert } from 'chai';
import { clearData } from './helpers.preload.js';
import { loadAllAndReinitializeRedux } from '../../services/allLoaders.preload.js';
import {
backupsService,
BackupType,
} from '../../services/backups/index.preload.js';
import { backupsService } from '../../services/backups/index.preload.js';
import { initialize as initializeExpiringMessageService } from '../../services/expiringMessagesDeletion.preload.js';
import { MemoryStream } from '../../util/MemoryStream.node.js';
@@ -56,13 +53,13 @@ describe('backup/integration', () => {
const expectedBuffer = await readFile(fullPath);
await backupsService.importBackup(() => Readable.from([expectedBuffer]), {
backupType: BackupType.TestOnlyPlaintext,
type: 'cross-client-integration-test',
});
const { data: exported } = await backupsService.exportBackupData(
BackupLevel.Paid,
BackupType.TestOnlyPlaintext
);
const { data: exported } = await backupsService.exportBackupData({
type: 'cross-client-integration-test',
level: BackupLevel.Paid,
});
const actualStream = new MemoryStream(Buffer.from(exported));
const expectedStream = new MemoryStream(expectedBuffer);

View File

@@ -16,7 +16,7 @@ import {
getAbsoluteDownloadsPath,
getAbsoluteAttachmentPath,
} from '../util/migrations.preload.js';
import { hasRequiredInformationForBackup } from '../util/Attachment.std.js';
import { hasRequiredInformationForRemoteBackup } from '../util/Attachment.std.js';
import {
AttachmentSizeError,
type AttachmentType,
@@ -200,7 +200,7 @@ export async function downloadAttachment(
}
if (mediaTier === MediaTier.BACKUP) {
strictAssert(
hasRequiredInformationForBackup(attachment),
hasRequiredInformationForRemoteBackup(attachment),
`${logId}: attachment missing critical information for backup tier`
);
}

View File

@@ -233,6 +233,10 @@ export type BackupableAttachmentType = WithRequiredProperties<
AttachmentType,
'plaintextHash' | 'key'
>;
export type AttachmentReadyForLocalBackup = WithRequiredProperties<
AttachmentType,
'plaintextHash' | 'localKey' | 'path'
>;
export type AttachmentDownloadableFromTransitTier = WithRequiredProperties<
AttachmentType,

View File

@@ -48,8 +48,6 @@ export type CoreAttachmentLocalBackupJobType = {
mediaName: string;
data: {
path: string | null;
size: number;
localKey: string;
};
backupsBaseDir: string;
};

View File

@@ -12,6 +12,7 @@ import type {
BackupableAttachmentType,
AttachmentDownloadableFromTransitTier,
LocallySavedAttachment,
AttachmentReadyForLocalBackup,
} from '../types/Attachment.std.js';
import type { LoggerType } from '../types/Logging.std.js';
import { createLogger } from '../logging/log.std.js';
@@ -847,7 +848,7 @@ export function getCachedAttachmentBySignature<T>(
return undefined;
}
export function hasRequiredInformationForBackup(
export function hasRequiredInformationForRemoteBackup(
attachment: AttachmentType
): attachment is BackupableAttachmentType {
return (
@@ -856,13 +857,23 @@ export function hasRequiredInformationForBackup(
);
}
export function hasRequiredInformationForLocalBackup(
attachment: AttachmentType
): attachment is AttachmentReadyForLocalBackup {
return (
isValidAttachmentKey(attachment.localKey) &&
isValidPlaintextHash(attachment.plaintextHash) &&
Boolean(attachment.path)
);
}
export function wasImportedFromLocalBackup(
attachment: AttachmentType
): attachment is BackupableAttachmentType {
return (
hasRequiredInformationForBackup(attachment) &&
Boolean(attachment.localBackupPath) &&
isValidAttachmentKey(attachment.localKey)
isValidPlaintextHash(attachment.plaintextHash) &&
isValidAttachmentKey(attachment.localKey) &&
Boolean(attachment.localBackupPath)
);
}
@@ -900,7 +911,7 @@ export function shouldAttachmentEndUpInRemoteBackup({
attachment: AttachmentType;
hasMediaBackups: boolean;
}): boolean {
return hasMediaBackups && hasRequiredInformationForBackup(attachment);
return hasMediaBackups && hasRequiredInformationForRemoteBackup(attachment);
}
export function isDownloadable(attachment: AttachmentType): boolean {

View File

@@ -2,7 +2,7 @@
// SPDX-License-Identifier: AGPL-3.0-only
import { ErrorCode, LibSignalErrorBase } from '@signalapp/libsignal-client';
import {
hasRequiredInformationForBackup,
hasRequiredInformationForRemoteBackup,
wasImportedFromLocalBackup,
} from './Attachment.std.js';
import {
@@ -58,7 +58,7 @@ export async function downloadAttachment({
variant !== AttachmentVariant.Default ? `[${variant}]` : '';
const logId = `${_logId}${variantForLogging}`;
const isBackupable = hasRequiredInformationForBackup(attachment);
const isBackupable = hasRequiredInformationForRemoteBackup(attachment);
const mightBeOnBackupTierNow = isBackupable && hasMediaBackups;
const mightBeOnBackupTierInTheFuture = isBackupable;