mirror of
https://github.com/signalapp/Signal-Desktop.git
synced 2026-02-15 07:28:59 +00:00
Refactor backup import/export options
This commit is contained in:
@@ -38,7 +38,7 @@ import {
|
||||
shouldAttachmentEndUpInRemoteBackup,
|
||||
getUndownloadedAttachmentSignature,
|
||||
isIncremental,
|
||||
hasRequiredInformationForBackup,
|
||||
hasRequiredInformationForRemoteBackup,
|
||||
} from '../util/Attachment.std.js';
|
||||
import type { ReadonlyMessageAttributesType } from '../model-types.d.ts';
|
||||
import { backupsService } from '../services/backups/index.preload.js';
|
||||
@@ -310,7 +310,7 @@ export class AttachmentDownloadManager extends JobManager<CoreAttachmentDownload
|
||||
|
||||
if (
|
||||
source === AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA &&
|
||||
!hasRequiredInformationForBackup(attachment)
|
||||
!hasRequiredInformationForRemoteBackup(attachment)
|
||||
) {
|
||||
source = AttachmentDownloadSource.BACKUP_IMPORT_NO_MEDIA;
|
||||
}
|
||||
|
||||
@@ -3,8 +3,12 @@
|
||||
/* eslint-disable max-classes-per-file */
|
||||
|
||||
import { existsSync } from 'node:fs';
|
||||
import { PassThrough } from 'node:stream';
|
||||
import { constants as FS_CONSTANTS, copyFile, mkdir } from 'node:fs/promises';
|
||||
import {
|
||||
constants as FS_CONSTANTS,
|
||||
copyFile,
|
||||
mkdir,
|
||||
rename,
|
||||
} from 'node:fs/promises';
|
||||
|
||||
import * as durations from '../util/durations/index.std.js';
|
||||
import { createLogger } from '../logging/log.std.js';
|
||||
@@ -14,6 +18,7 @@ import { redactGenericText } from '../util/privacy.node.js';
|
||||
import {
|
||||
getAbsoluteAttachmentPath,
|
||||
getAbsoluteAttachmentPath as doGetAbsoluteAttachmentPath,
|
||||
getAbsoluteTempPath,
|
||||
} from '../util/migrations.preload.js';
|
||||
import {
|
||||
JobManager,
|
||||
@@ -36,6 +41,7 @@ import {
|
||||
getLocalBackupDirectoryForMediaName,
|
||||
getLocalBackupPathForMediaName,
|
||||
} from '../services/backups/util/localBackup.node.js';
|
||||
import { createName } from '../util/attachmentPath.node.js';
|
||||
|
||||
const log = createLogger('AttachmentLocalBackupManager');
|
||||
|
||||
@@ -219,7 +225,7 @@ async function runAttachmentBackupJobInner(
|
||||
log.info(`${logId}: starting`);
|
||||
|
||||
const { backupsBaseDir, mediaName } = job;
|
||||
const { localKey, path, size } = job.data;
|
||||
const { path } = job.data;
|
||||
|
||||
if (!path) {
|
||||
throw new AttachmentPermanentlyMissingError('No path property');
|
||||
@@ -230,45 +236,26 @@ async function runAttachmentBackupJobInner(
|
||||
throw new AttachmentPermanentlyMissingError('No file at provided path');
|
||||
}
|
||||
|
||||
if (!localKey) {
|
||||
throw new Error('No localKey property, required for test decryption');
|
||||
}
|
||||
|
||||
const localBackupFileDir = getLocalBackupDirectoryForMediaName({
|
||||
backupsBaseDir,
|
||||
mediaName,
|
||||
});
|
||||
await mkdir(localBackupFileDir, { recursive: true });
|
||||
|
||||
const localBackupFilePath = getLocalBackupPathForMediaName({
|
||||
const destinationLocalBackupFilePath = getLocalBackupPathForMediaName({
|
||||
backupsBaseDir,
|
||||
mediaName,
|
||||
});
|
||||
|
||||
// TODO: Add check in local FS to prevent double backup
|
||||
|
||||
// File is already encrypted with localKey, so we just have to copy it to the backup dir
|
||||
const attachmentPath = getAbsoluteAttachmentPath(path);
|
||||
const sourceAttachmentPath = getAbsoluteAttachmentPath(path);
|
||||
const tempPath = getAbsoluteTempPath(createName());
|
||||
|
||||
// A unique constraint on the DB table should enforce that only one job is writing to
|
||||
// the same mediaName at a time, but just to be safe, we copy to temp file and rename to
|
||||
// ensure the atomicity of the copy operation
|
||||
|
||||
// Set COPYFILE_FICLONE for Copy on Write (OS dependent, gracefully falls back to copy)
|
||||
await copyFile(
|
||||
attachmentPath,
|
||||
localBackupFilePath,
|
||||
FS_CONSTANTS.COPYFILE_FICLONE
|
||||
);
|
||||
|
||||
// TODO: Optimize this check -- it can be expensive to test decrypt on every export
|
||||
log.info(`${logId}: Verifying file in local backup`);
|
||||
const sink = new PassThrough();
|
||||
sink.resume();
|
||||
await decryptAttachmentV2ToSink(
|
||||
{
|
||||
ciphertextPath: localBackupFilePath,
|
||||
idForLogging: 'AttachmentLocalBackupManager',
|
||||
keysBase64: localKey,
|
||||
size,
|
||||
type: 'local',
|
||||
},
|
||||
sink
|
||||
);
|
||||
await copyFile(sourceAttachmentPath, tempPath, FS_CONSTANTS.COPYFILE_FICLONE);
|
||||
await rename(tempPath, destinationLocalBackupFilePath);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import Long from 'long';
|
||||
import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client';
|
||||
import type { BackupLevel } from '@signalapp/libsignal-client/zkgroup.js';
|
||||
import { dirname } from 'node:path';
|
||||
import pMap from 'p-map';
|
||||
import pTimeout from 'p-timeout';
|
||||
@@ -117,8 +116,11 @@ import {
|
||||
} from '../../types/CallDisposition.std.js';
|
||||
import { isAciString } from '../../util/isAciString.std.js';
|
||||
import { hslToRGBInt } from '../../util/hslToRGB.std.js';
|
||||
import type { AboutMe, LocalChatStyle } from './types.std.js';
|
||||
import { BackupType } from './types.std.js';
|
||||
import type {
|
||||
AboutMe,
|
||||
BackupExportOptions,
|
||||
LocalChatStyle,
|
||||
} from './types.std.js';
|
||||
import { messageHasPaymentEvent } from '../../messages/payments.std.js';
|
||||
import {
|
||||
numberToAddressType,
|
||||
@@ -129,7 +131,8 @@ import type { AttachmentType } from '../../types/Attachment.std.js';
|
||||
import {
|
||||
isGIF,
|
||||
isDownloaded,
|
||||
hasRequiredInformationForBackup,
|
||||
hasRequiredInformationForLocalBackup,
|
||||
hasRequiredInformationForRemoteBackup,
|
||||
} from '../../util/Attachment.std.js';
|
||||
import { getFilePointerForAttachment } from './util/filePointers.preload.js';
|
||||
import { getBackupMediaRootKey } from './crypto.preload.js';
|
||||
@@ -141,6 +144,7 @@ import { AttachmentBackupManager } from '../../jobs/AttachmentBackupManager.prel
|
||||
import { AttachmentLocalBackupManager } from '../../jobs/AttachmentLocalBackupManager.preload.js';
|
||||
import {
|
||||
getBackupCdnInfo,
|
||||
getLocalBackupFileNameForAttachment,
|
||||
getMediaNameForAttachment,
|
||||
} from './util/mediaId.preload.js';
|
||||
import { calculateExpirationTimestamp } from '../../util/expirationTimer.std.js';
|
||||
@@ -210,8 +214,6 @@ type GetRecipientIdOptionsType =
|
||||
type ToChatItemOptionsType = Readonly<{
|
||||
aboutMe: AboutMe;
|
||||
callHistoryByCallId: Record<string, CallHistoryDetails>;
|
||||
backupLevel: BackupLevel;
|
||||
isLocalBackup: boolean;
|
||||
}>;
|
||||
|
||||
type NonBubbleOptionsType = Pick<
|
||||
@@ -287,18 +289,11 @@ export class BackupExportStream extends Readable {
|
||||
// array.
|
||||
#customColorIdByUuid = new Map<string, Long>();
|
||||
|
||||
constructor(private readonly backupType: BackupType) {
|
||||
constructor(private readonly options: Readonly<BackupExportOptions>) {
|
||||
super();
|
||||
}
|
||||
|
||||
public run(
|
||||
backupLevel: BackupLevel,
|
||||
localBackupSnapshotDir: string | undefined = undefined
|
||||
): void {
|
||||
const localBackupsBaseDir = localBackupSnapshotDir
|
||||
? dirname(localBackupSnapshotDir)
|
||||
: undefined;
|
||||
const isLocalBackup = localBackupsBaseDir != null;
|
||||
public run(): void {
|
||||
drop(
|
||||
(async () => {
|
||||
log.info('BackupExportStream: starting...');
|
||||
@@ -309,37 +304,44 @@ export class BackupExportStream extends Readable {
|
||||
|
||||
await pauseWriteAccess();
|
||||
try {
|
||||
await this.#unsafeRun(backupLevel, isLocalBackup);
|
||||
await this.#unsafeRun();
|
||||
} catch (error) {
|
||||
this.emit('error', error);
|
||||
} finally {
|
||||
await resumeWriteAccess();
|
||||
// TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction
|
||||
const { type } = this.options;
|
||||
switch (type) {
|
||||
case 'local-encrypted':
|
||||
{
|
||||
log.info(
|
||||
`BackupExportStream: Adding ${this.#attachmentBackupJobs.length} jobs for AttachmentLocalBackupManager`
|
||||
);
|
||||
const backupsBaseDir = dirname(
|
||||
this.options.localBackupSnapshotDir
|
||||
);
|
||||
|
||||
if (isLocalBackup) {
|
||||
log.info(
|
||||
`BackupExportStream: Adding ${this.#attachmentBackupJobs.length} jobs for AttachmentLocalBackupManager`
|
||||
);
|
||||
AttachmentLocalBackupManager.clearAllJobs();
|
||||
await Promise.all(
|
||||
this.#attachmentBackupJobs.map(job => {
|
||||
if (job.type !== 'local') {
|
||||
log.error(
|
||||
"BackupExportStream: Can't enqueue remote backup jobs during local backup, skipping"
|
||||
);
|
||||
return Promise.resolve();
|
||||
}
|
||||
AttachmentLocalBackupManager.clearAllJobs();
|
||||
await Promise.all(
|
||||
this.#attachmentBackupJobs.map(job => {
|
||||
if (job.type !== 'local') {
|
||||
log.error(
|
||||
"BackupExportStream: Can't enqueue remote backup jobs during local backup, skipping"
|
||||
);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return AttachmentLocalBackupManager.addJob({
|
||||
...job,
|
||||
backupsBaseDir: localBackupsBaseDir,
|
||||
});
|
||||
})
|
||||
);
|
||||
drop(AttachmentLocalBackupManager.start());
|
||||
} else {
|
||||
// TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction
|
||||
await DataWriter.clearAllAttachmentBackupJobs();
|
||||
if (this.backupType !== BackupType.TestOnlyPlaintext) {
|
||||
return AttachmentLocalBackupManager.addJob({
|
||||
...job,
|
||||
backupsBaseDir,
|
||||
});
|
||||
})
|
||||
);
|
||||
drop(AttachmentLocalBackupManager.start());
|
||||
}
|
||||
break;
|
||||
case 'remote':
|
||||
await DataWriter.clearAllAttachmentBackupJobs();
|
||||
await Promise.all(
|
||||
this.#attachmentBackupJobs.map(job => {
|
||||
if (job.type === 'local') {
|
||||
@@ -355,9 +357,13 @@ export class BackupExportStream extends Readable {
|
||||
})
|
||||
);
|
||||
drop(AttachmentBackupManager.start());
|
||||
}
|
||||
break;
|
||||
case 'cross-client-integration-test':
|
||||
break;
|
||||
default:
|
||||
// eslint-disable-next-line no-unsafe-finally
|
||||
throw missingCaseError(type);
|
||||
}
|
||||
|
||||
log.info('BackupExportStream: finished');
|
||||
}
|
||||
})()
|
||||
@@ -372,10 +378,7 @@ export class BackupExportStream extends Readable {
|
||||
return this.#stats;
|
||||
}
|
||||
|
||||
async #unsafeRun(
|
||||
backupLevel: BackupLevel,
|
||||
isLocalBackup: boolean
|
||||
): Promise<void> {
|
||||
async #unsafeRun(): Promise<void> {
|
||||
this.#ourConversation =
|
||||
window.ConversationController.getOurConversationOrThrow().attributes;
|
||||
this.push(
|
||||
@@ -752,8 +755,6 @@ export class BackupExportStream extends Readable {
|
||||
const chatItem = await this.#toChatItem(message, {
|
||||
aboutMe,
|
||||
callHistoryByCallId,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
});
|
||||
|
||||
if (chatItem === undefined) {
|
||||
@@ -1214,12 +1215,7 @@ export class BackupExportStream extends Readable {
|
||||
|
||||
async #toChatItem(
|
||||
message: MessageAttributesType,
|
||||
{
|
||||
aboutMe,
|
||||
callHistoryByCallId,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
}: ToChatItemOptionsType
|
||||
{ aboutMe, callHistoryByCallId }: ToChatItemOptionsType
|
||||
): Promise<Backups.IChatItem | undefined> {
|
||||
const conversation = window.ConversationController.get(
|
||||
message.conversationId
|
||||
@@ -1378,8 +1374,6 @@ export class BackupExportStream extends Readable {
|
||||
if (isTapToView(message)) {
|
||||
result.viewOnceMessage = await this.#toViewOnceMessage({
|
||||
message,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
});
|
||||
} else if (message.deletedForEveryone) {
|
||||
result.remoteDeletedMessage = {};
|
||||
@@ -1440,8 +1434,6 @@ export class BackupExportStream extends Readable {
|
||||
avatar: contactDetails.avatar?.avatar
|
||||
? await this.#processAttachment({
|
||||
attachment: contactDetails.avatar.avatar,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
messageReceivedAt: message.received_at,
|
||||
})
|
||||
: undefined,
|
||||
@@ -1462,8 +1454,6 @@ export class BackupExportStream extends Readable {
|
||||
stickerProto.data = sticker.data
|
||||
? await this.#processAttachment({
|
||||
attachment: sticker.data,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
messageReceivedAt: message.received_at,
|
||||
})
|
||||
: undefined;
|
||||
@@ -1506,29 +1496,15 @@ export class BackupExportStream extends Readable {
|
||||
} else if (message.storyReplyContext) {
|
||||
result.directStoryReplyMessage = await this.#toDirectStoryReplyMessage({
|
||||
message,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
});
|
||||
|
||||
result.revisions = await this.#toChatItemRevisions(
|
||||
result,
|
||||
message,
|
||||
backupLevel,
|
||||
isLocalBackup
|
||||
);
|
||||
result.revisions = await this.#toChatItemRevisions(result, message);
|
||||
} else {
|
||||
result.standardMessage = await this.#toStandardMessage({
|
||||
message,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
});
|
||||
|
||||
result.revisions = await this.#toChatItemRevisions(
|
||||
result,
|
||||
message,
|
||||
backupLevel,
|
||||
isLocalBackup
|
||||
);
|
||||
result.revisions = await this.#toChatItemRevisions(result, message);
|
||||
}
|
||||
|
||||
if (isOutgoing) {
|
||||
@@ -2446,12 +2422,8 @@ export class BackupExportStream extends Readable {
|
||||
|
||||
async #toQuote({
|
||||
message,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
}: {
|
||||
message: Pick<MessageAttributesType, 'quote' | 'received_at' | 'body'>;
|
||||
backupLevel: BackupLevel;
|
||||
isLocalBackup: boolean;
|
||||
}): Promise<Backups.IQuote | null> {
|
||||
const { quote } = message;
|
||||
if (!quote) {
|
||||
@@ -2513,9 +2485,7 @@ export class BackupExportStream extends Readable {
|
||||
thumbnail: attachment.thumbnail
|
||||
? await this.#processMessageAttachment({
|
||||
attachment: attachment.thumbnail,
|
||||
backupLevel,
|
||||
message,
|
||||
isLocalBackup,
|
||||
})
|
||||
: undefined,
|
||||
};
|
||||
@@ -2572,21 +2542,15 @@ export class BackupExportStream extends Readable {
|
||||
|
||||
async #processMessageAttachment({
|
||||
attachment,
|
||||
backupLevel,
|
||||
message,
|
||||
isLocalBackup,
|
||||
}: {
|
||||
attachment: AttachmentType;
|
||||
backupLevel: BackupLevel;
|
||||
message: Pick<MessageAttributesType, 'quote' | 'received_at' | 'body'>;
|
||||
isLocalBackup: boolean;
|
||||
}): Promise<Backups.MessageAttachment> {
|
||||
const { clientUuid } = attachment;
|
||||
const filePointer = await this.#processAttachment({
|
||||
attachment,
|
||||
backupLevel,
|
||||
messageReceivedAt: message.received_at,
|
||||
isLocalBackup,
|
||||
});
|
||||
|
||||
return new Backups.MessageAttachment({
|
||||
@@ -2599,26 +2563,28 @@ export class BackupExportStream extends Readable {
|
||||
|
||||
async #processAttachment({
|
||||
attachment,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
messageReceivedAt,
|
||||
}: {
|
||||
attachment: AttachmentType;
|
||||
backupLevel: BackupLevel;
|
||||
isLocalBackup: boolean;
|
||||
messageReceivedAt: number;
|
||||
}): Promise<Backups.FilePointer> {
|
||||
const { filePointer, backupJob } = await getFilePointerForAttachment({
|
||||
attachment,
|
||||
isLocalBackup,
|
||||
backupLevel,
|
||||
backupOptions: this.options,
|
||||
messageReceivedAt,
|
||||
getBackupCdnInfo,
|
||||
});
|
||||
|
||||
if (hasRequiredInformationForBackup(attachment)) {
|
||||
const mediaName = getMediaNameForAttachment(attachment);
|
||||
let mediaName: string | undefined;
|
||||
if (this.options.type === 'local-encrypted') {
|
||||
if (hasRequiredInformationForLocalBackup(attachment)) {
|
||||
mediaName = getLocalBackupFileNameForAttachment(attachment);
|
||||
}
|
||||
} else if (hasRequiredInformationForRemoteBackup(attachment)) {
|
||||
mediaName = getMediaNameForAttachment(attachment);
|
||||
}
|
||||
|
||||
if (mediaName) {
|
||||
// Re-use existing locatorInfo and backup job if we've already seen this file
|
||||
const existingFilePointer = this.#mediaNamesToFilePointers.get(mediaName);
|
||||
|
||||
@@ -2632,7 +2598,6 @@ export class BackupExportStream extends Readable {
|
||||
if (filePointer.locatorInfo) {
|
||||
this.#mediaNamesToFilePointers.set(mediaName, filePointer);
|
||||
}
|
||||
|
||||
if (backupJob) {
|
||||
this.#attachmentBackupJobs.push(backupJob);
|
||||
}
|
||||
@@ -2814,8 +2779,6 @@ export class BackupExportStream extends Readable {
|
||||
|
||||
async #toStandardMessage({
|
||||
message,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
}: {
|
||||
message: Pick<
|
||||
MessageAttributesType,
|
||||
@@ -2829,8 +2792,6 @@ export class BackupExportStream extends Readable {
|
||||
| 'received_at'
|
||||
| 'timestamp'
|
||||
>;
|
||||
backupLevel: BackupLevel;
|
||||
isLocalBackup: boolean;
|
||||
}): Promise<Backups.IStandardMessage> {
|
||||
if (
|
||||
message.body &&
|
||||
@@ -2842,17 +2803,13 @@ export class BackupExportStream extends Readable {
|
||||
return {
|
||||
quote: await this.#toQuote({
|
||||
message,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
}),
|
||||
attachments: message.attachments?.length
|
||||
? await Promise.all(
|
||||
message.attachments.map(attachment => {
|
||||
return this.#processMessageAttachment({
|
||||
attachment,
|
||||
backupLevel,
|
||||
message,
|
||||
isLocalBackup,
|
||||
});
|
||||
})
|
||||
)
|
||||
@@ -2863,8 +2820,6 @@ export class BackupExportStream extends Readable {
|
||||
message.bodyAttachment && !isDownloaded(message.bodyAttachment)
|
||||
? await this.#processAttachment({
|
||||
attachment: message.bodyAttachment,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
messageReceivedAt: message.received_at,
|
||||
})
|
||||
: undefined,
|
||||
@@ -2890,8 +2845,6 @@ export class BackupExportStream extends Readable {
|
||||
image: preview.image
|
||||
? await this.#processAttachment({
|
||||
attachment: preview.image,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
messageReceivedAt: message.received_at,
|
||||
})
|
||||
: undefined,
|
||||
@@ -2905,8 +2858,6 @@ export class BackupExportStream extends Readable {
|
||||
|
||||
async #toDirectStoryReplyMessage({
|
||||
message,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
}: {
|
||||
message: Pick<
|
||||
MessageAttributesType,
|
||||
@@ -2917,8 +2868,6 @@ export class BackupExportStream extends Readable {
|
||||
| 'received_at'
|
||||
| 'reactions'
|
||||
>;
|
||||
backupLevel: BackupLevel;
|
||||
isLocalBackup: boolean;
|
||||
}): Promise<Backups.IDirectStoryReplyMessage> {
|
||||
const result = new Backups.DirectStoryReplyMessage({
|
||||
reactions: this.#getMessageReactions(message),
|
||||
@@ -2931,8 +2880,6 @@ export class BackupExportStream extends Readable {
|
||||
longText: message.bodyAttachment
|
||||
? await this.#processAttachment({
|
||||
attachment: message.bodyAttachment,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
messageReceivedAt: message.received_at,
|
||||
})
|
||||
: undefined,
|
||||
@@ -2952,15 +2899,11 @@ export class BackupExportStream extends Readable {
|
||||
|
||||
async #toViewOnceMessage({
|
||||
message,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
}: {
|
||||
message: Pick<
|
||||
MessageAttributesType,
|
||||
'attachments' | 'received_at' | 'reactions'
|
||||
>;
|
||||
backupLevel: BackupLevel;
|
||||
isLocalBackup: boolean;
|
||||
}): Promise<Backups.IViewOnceMessage> {
|
||||
const attachment = message.attachments?.at(0);
|
||||
// Integration tests use the 'link-and-sync' version of export, which will include
|
||||
@@ -2972,9 +2915,7 @@ export class BackupExportStream extends Readable {
|
||||
? null
|
||||
: await this.#processMessageAttachment({
|
||||
attachment,
|
||||
backupLevel,
|
||||
message,
|
||||
isLocalBackup,
|
||||
}),
|
||||
reactions: this.#getMessageReactions(message),
|
||||
};
|
||||
@@ -2982,9 +2923,7 @@ export class BackupExportStream extends Readable {
|
||||
|
||||
async #toChatItemRevisions(
|
||||
parent: Backups.IChatItem,
|
||||
message: MessageAttributesType,
|
||||
backupLevel: BackupLevel,
|
||||
isLocalBackup: boolean
|
||||
message: MessageAttributesType
|
||||
): Promise<Array<Backups.IChatItem> | undefined> {
|
||||
const { editHistory } = message;
|
||||
if (editHistory == null) {
|
||||
@@ -3022,14 +2961,10 @@ export class BackupExportStream extends Readable {
|
||||
result.directStoryReplyMessage =
|
||||
await this.#toDirectStoryReplyMessage({
|
||||
message: history,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
});
|
||||
} else {
|
||||
result.standardMessage = await this.#toStandardMessage({
|
||||
message: history,
|
||||
backupLevel,
|
||||
isLocalBackup,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
|
||||
@@ -92,8 +92,11 @@ import { signalProtocolStore } from '../../SignalProtocolStore.preload.js';
|
||||
import * as Bytes from '../../Bytes.std.js';
|
||||
import { BACKUP_VERSION, WALLPAPER_TO_BUBBLE_COLOR } from './constants.std.js';
|
||||
import { UnsupportedBackupVersion } from './errors.std.js';
|
||||
import type { AboutMe, LocalChatStyle } from './types.std.js';
|
||||
import { BackupType } from './types.std.js';
|
||||
import type {
|
||||
AboutMe,
|
||||
BackupImportOptions,
|
||||
LocalChatStyle,
|
||||
} from './types.std.js';
|
||||
import { getBackupMediaRootKey } from './crypto.preload.js';
|
||||
import type { GroupV2ChangeDetailType } from '../../types/groups.std.js';
|
||||
import { queueAttachmentDownloads } from '../../util/queueAttachmentDownloads.preload.js';
|
||||
@@ -274,22 +277,18 @@ export class BackupImportStream extends Writable {
|
||||
#frameErrorCount: number = 0;
|
||||
#backupTier: BackupLevel | undefined;
|
||||
|
||||
private constructor(
|
||||
private readonly backupType: BackupType,
|
||||
private readonly localBackupSnapshotDir: string | undefined
|
||||
) {
|
||||
private constructor(private readonly options: BackupImportOptions) {
|
||||
super({ objectMode: true });
|
||||
}
|
||||
|
||||
public static async create(
|
||||
backupType = BackupType.Ciphertext,
|
||||
localBackupSnapshotDir: string | undefined = undefined
|
||||
options: BackupImportOptions
|
||||
): Promise<BackupImportStream> {
|
||||
await AttachmentDownloadManager.stop();
|
||||
await DataWriter.removeAllBackupAttachmentDownloadJobs();
|
||||
await resetBackupMediaDownloadStats();
|
||||
|
||||
return new BackupImportStream(backupType, localBackupSnapshotDir);
|
||||
return new BackupImportStream(options);
|
||||
}
|
||||
|
||||
override async _write(
|
||||
@@ -424,7 +423,7 @@ export class BackupImportStream extends Writable {
|
||||
await pMap(
|
||||
[...this.#pendingGroupAvatars.entries()],
|
||||
async ([conversationId, newAvatarUrl]) => {
|
||||
if (this.backupType === BackupType.TestOnlyPlaintext) {
|
||||
if (this.options.type === 'cross-client-integration-test') {
|
||||
return;
|
||||
}
|
||||
await groupAvatarJobQueue.add({ conversationId, newAvatarUrl });
|
||||
@@ -446,7 +445,7 @@ export class BackupImportStream extends Writable {
|
||||
);
|
||||
|
||||
if (
|
||||
this.backupType !== BackupType.TestOnlyPlaintext &&
|
||||
this.options.type !== 'cross-client-integration-test' &&
|
||||
!isTestEnvironment(getEnvironment())
|
||||
) {
|
||||
await startBackupMediaDownload();
|
||||
@@ -1906,17 +1905,14 @@ export class BackupImportStream extends Writable {
|
||||
bodyRanges: this.#fromBodyRanges(data.text),
|
||||
})),
|
||||
bodyAttachment: data.longText
|
||||
? convertFilePointerToAttachment(
|
||||
data.longText,
|
||||
this.#getFilePointerOptions()
|
||||
)
|
||||
? convertFilePointerToAttachment(data.longText, this.options)
|
||||
: undefined,
|
||||
attachments: data.attachments?.length
|
||||
? data.attachments
|
||||
.map(attachment =>
|
||||
convertBackupMessageAttachmentToAttachment(
|
||||
attachment,
|
||||
this.#getFilePointerOptions()
|
||||
this.options
|
||||
)
|
||||
)
|
||||
.filter(isNotNil)
|
||||
@@ -1962,10 +1958,7 @@ export class BackupImportStream extends Writable {
|
||||
description: dropNull(preview.description),
|
||||
date: getCheckedTimestampOrUndefinedFromLong(preview.date),
|
||||
image: preview.image
|
||||
? convertFilePointerToAttachment(
|
||||
preview.image,
|
||||
this.#getFilePointerOptions()
|
||||
)
|
||||
? convertFilePointerToAttachment(preview.image, this.options)
|
||||
: undefined,
|
||||
};
|
||||
})
|
||||
@@ -1984,7 +1977,7 @@ export class BackupImportStream extends Writable {
|
||||
? [
|
||||
convertBackupMessageAttachmentToAttachment(
|
||||
attachment,
|
||||
this.#getFilePointerOptions()
|
||||
this.options
|
||||
),
|
||||
].filter(isNotNil)
|
||||
: undefined,
|
||||
@@ -2023,10 +2016,7 @@ export class BackupImportStream extends Writable {
|
||||
result.body = textReply.text?.body ?? undefined;
|
||||
result.bodyRanges = this.#fromBodyRanges(textReply.text);
|
||||
result.bodyAttachment = textReply.longText
|
||||
? convertFilePointerToAttachment(
|
||||
textReply.longText,
|
||||
this.#getFilePointerOptions()
|
||||
)
|
||||
? convertFilePointerToAttachment(textReply.longText, this.options)
|
||||
: undefined;
|
||||
} else if (emoji) {
|
||||
result.storyReaction = {
|
||||
@@ -2056,10 +2046,7 @@ export class BackupImportStream extends Writable {
|
||||
body: textReply.text?.body ?? undefined,
|
||||
bodyRanges: this.#fromBodyRanges(textReply.text),
|
||||
bodyAttachment: textReply.longText
|
||||
? convertFilePointerToAttachment(
|
||||
textReply.longText,
|
||||
this.#getFilePointerOptions()
|
||||
)
|
||||
? convertFilePointerToAttachment(textReply.longText, this.options)
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
@@ -2182,10 +2169,7 @@ export class BackupImportStream extends Writable {
|
||||
? stringToMIMEType(contentType)
|
||||
: APPLICATION_OCTET_STREAM,
|
||||
thumbnail: thumbnail?.pointer
|
||||
? convertFilePointerToAttachment(
|
||||
thumbnail.pointer,
|
||||
this.#getFilePointerOptions()
|
||||
)
|
||||
? convertFilePointerToAttachment(thumbnail.pointer, this.options)
|
||||
: undefined,
|
||||
};
|
||||
}) ?? [],
|
||||
@@ -2347,7 +2331,7 @@ export class BackupImportStream extends Writable {
|
||||
? {
|
||||
avatar: convertFilePointerToAttachment(
|
||||
avatar,
|
||||
this.#getFilePointerOptions()
|
||||
this.options
|
||||
),
|
||||
isProfile: false,
|
||||
}
|
||||
@@ -2396,10 +2380,7 @@ export class BackupImportStream extends Writable {
|
||||
packKey: Bytes.toBase64(packKey),
|
||||
stickerId,
|
||||
data: data
|
||||
? convertFilePointerToAttachment(
|
||||
data,
|
||||
this.#getFilePointerOptions()
|
||||
)
|
||||
? convertFilePointerToAttachment(data, this.options)
|
||||
: undefined,
|
||||
},
|
||||
reactions: this.#fromReactions(chatItem.stickerMessage.reactions),
|
||||
@@ -3800,16 +3781,8 @@ export class BackupImportStream extends Writable {
|
||||
};
|
||||
}
|
||||
|
||||
#getFilePointerOptions() {
|
||||
if (this.localBackupSnapshotDir != null) {
|
||||
return { localBackupSnapshotDir: this.localBackupSnapshotDir };
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
#isLocalBackup() {
|
||||
return this.localBackupSnapshotDir != null;
|
||||
return this.options.type === 'local-encrypted';
|
||||
}
|
||||
|
||||
#isMediaEnabledBackup() {
|
||||
|
||||
@@ -69,7 +69,7 @@ import {
|
||||
validateBackupStream,
|
||||
ValidationType,
|
||||
} from './validator.preload.js';
|
||||
import { BackupType } from './types.std.js';
|
||||
import type { BackupExportOptions, BackupImportOptions } from './types.std.js';
|
||||
import {
|
||||
BackupInstallerError,
|
||||
BackupDownloadFailedError,
|
||||
@@ -103,8 +103,6 @@ const { isEqual, noop } = lodash;
|
||||
|
||||
const log = createLogger('backupsService');
|
||||
|
||||
export { BackupType };
|
||||
|
||||
const IV_LENGTH = 16;
|
||||
|
||||
const BACKUP_REFRESH_INTERVAL = 24 * HOUR;
|
||||
@@ -128,13 +126,6 @@ type DoDownloadOptionsType = Readonly<{
|
||||
) => void;
|
||||
}>;
|
||||
|
||||
export type ImportOptionsType = Readonly<{
|
||||
backupType?: BackupType;
|
||||
localBackupSnapshotDir?: string;
|
||||
ephemeralKey?: Uint8Array;
|
||||
onProgress?: (currentBytes: number, totalBytes: number) => void;
|
||||
}>;
|
||||
|
||||
export type ExportResultType = Readonly<{
|
||||
totalBytes: number;
|
||||
duration: number;
|
||||
@@ -318,7 +309,10 @@ export class BackupsService {
|
||||
log.info(`exportBackup: starting, backup level: ${backupLevel}...`);
|
||||
|
||||
try {
|
||||
const { totalBytes } = await this.exportToDisk(filePath, backupLevel);
|
||||
const { totalBytes } = await this.exportToDisk(filePath, {
|
||||
type: 'remote',
|
||||
level: backupLevel,
|
||||
});
|
||||
|
||||
await this.api.upload(filePath, totalBytes);
|
||||
} finally {
|
||||
@@ -331,8 +325,7 @@ export class BackupsService {
|
||||
}
|
||||
|
||||
public async exportLocalBackup(
|
||||
backupsBaseDir: string | undefined = undefined,
|
||||
backupLevel: BackupLevel = BackupLevel.Free
|
||||
backupsBaseDir: string | undefined = undefined
|
||||
): Promise<LocalBackupExportResultType> {
|
||||
strictAssert(isLocalBackupsEnabled(), 'Local backups must be enabled');
|
||||
|
||||
@@ -347,12 +340,10 @@ export class BackupsService {
|
||||
|
||||
log.info('exportLocalBackup: starting');
|
||||
|
||||
const exportResult = await this.exportToDisk(
|
||||
mainProtoPath,
|
||||
backupLevel,
|
||||
BackupType.Ciphertext,
|
||||
snapshotDir
|
||||
);
|
||||
const exportResult = await this.exportToDisk(mainProtoPath, {
|
||||
type: 'local-encrypted',
|
||||
localBackupSnapshotDir: snapshotDir,
|
||||
});
|
||||
|
||||
log.info('exportLocalBackup: writing metadata');
|
||||
const metadataArgs = {
|
||||
@@ -405,6 +396,7 @@ export class BackupsService {
|
||||
|
||||
const backupFile = join(this.#localBackupSnapshotDir, 'main');
|
||||
await this.importFromDisk(backupFile, {
|
||||
type: 'local-encrypted',
|
||||
localBackupSnapshotDir: this.#localBackupSnapshotDir,
|
||||
});
|
||||
|
||||
@@ -421,14 +413,13 @@ export class BackupsService {
|
||||
|
||||
// Test harness
|
||||
public async exportBackupData(
|
||||
backupLevel: BackupLevel = BackupLevel.Free,
|
||||
backupType = BackupType.Ciphertext
|
||||
options: BackupExportOptions
|
||||
): Promise<{ data: Uint8Array } & ExportResultType> {
|
||||
const sink = new PassThrough();
|
||||
|
||||
const chunks = new Array<Uint8Array>();
|
||||
sink.on('data', chunk => chunks.push(chunk));
|
||||
const result = await this.#exportBackup(sink, backupLevel, backupType);
|
||||
const result = await this.#exportBackup(sink, options);
|
||||
|
||||
return {
|
||||
...result,
|
||||
@@ -438,18 +429,14 @@ export class BackupsService {
|
||||
|
||||
public async exportToDisk(
|
||||
path: string,
|
||||
backupLevel: BackupLevel = BackupLevel.Free,
|
||||
backupType = BackupType.Ciphertext,
|
||||
localBackupSnapshotDir: string | undefined = undefined
|
||||
options: BackupExportOptions
|
||||
): Promise<ExportResultType> {
|
||||
const exportResult = await this.#exportBackup(
|
||||
createWriteStream(path),
|
||||
backupLevel,
|
||||
backupType,
|
||||
localBackupSnapshotDir
|
||||
options
|
||||
);
|
||||
|
||||
if (backupType === BackupType.Ciphertext) {
|
||||
if (options.type !== 'cross-client-integration-test') {
|
||||
await validateBackup(
|
||||
() => new FileStream(path),
|
||||
exportResult.totalBytes,
|
||||
@@ -462,9 +449,7 @@ export class BackupsService {
|
||||
return exportResult;
|
||||
}
|
||||
|
||||
public async _internalExportLocalBackup(
|
||||
backupLevel: BackupLevel = BackupLevel.Free
|
||||
): Promise<ValidationResultType> {
|
||||
public async _internalExportLocalBackup(): Promise<ValidationResultType> {
|
||||
try {
|
||||
const { canceled, dirPath: backupsBaseDir } = await ipcRenderer.invoke(
|
||||
'show-open-folder-dialog'
|
||||
@@ -473,7 +458,7 @@ export class BackupsService {
|
||||
return { error: 'Backups directory not selected' };
|
||||
}
|
||||
|
||||
const result = await this.exportLocalBackup(backupsBaseDir, backupLevel);
|
||||
const result = await this.exportLocalBackup(backupsBaseDir);
|
||||
return { result };
|
||||
} catch (error) {
|
||||
return { error: Errors.toLogFormat(error) };
|
||||
@@ -496,16 +481,16 @@ export class BackupsService {
|
||||
}
|
||||
|
||||
// Test harness
|
||||
public async _internalValidate(
|
||||
backupLevel: BackupLevel = BackupLevel.Free,
|
||||
backupType = BackupType.Ciphertext
|
||||
): Promise<ValidationResultType> {
|
||||
public async _internalValidate(): Promise<ValidationResultType> {
|
||||
try {
|
||||
const start = Date.now();
|
||||
|
||||
const recordStream = new BackupExportStream(backupType);
|
||||
const recordStream = new BackupExportStream({
|
||||
type: 'remote',
|
||||
level: BackupLevel.Free,
|
||||
});
|
||||
|
||||
recordStream.run(backupLevel);
|
||||
recordStream.run();
|
||||
|
||||
const totalBytes = await validateBackupStream(recordStream);
|
||||
|
||||
@@ -521,7 +506,10 @@ export class BackupsService {
|
||||
|
||||
// Test harness
|
||||
public async exportWithDialog(): Promise<void> {
|
||||
const { data } = await this.exportBackupData();
|
||||
const { data } = await this.exportBackupData({
|
||||
type: 'remote',
|
||||
level: BackupLevel.Free,
|
||||
});
|
||||
|
||||
await saveAttachmentToDisk({
|
||||
name: 'backup.bin',
|
||||
@@ -531,7 +519,7 @@ export class BackupsService {
|
||||
|
||||
public async importFromDisk(
|
||||
backupFile: string,
|
||||
options?: ImportOptionsType
|
||||
options: BackupImportOptions
|
||||
): Promise<void> {
|
||||
return this.importBackup(() => createReadStream(backupFile), options);
|
||||
}
|
||||
@@ -562,18 +550,13 @@ export class BackupsService {
|
||||
|
||||
public async importBackup(
|
||||
createBackupStream: () => Readable,
|
||||
{
|
||||
backupType = BackupType.Ciphertext,
|
||||
ephemeralKey,
|
||||
onProgress,
|
||||
localBackupSnapshotDir = undefined,
|
||||
}: ImportOptionsType = {}
|
||||
options: BackupImportOptions
|
||||
): Promise<void> {
|
||||
strictAssert(!this.#isRunning, 'BackupService is already running');
|
||||
|
||||
window.IPC.startTrackingQueryStats();
|
||||
|
||||
log.info(`importBackup: starting ${backupType}...`);
|
||||
log.info(`importBackup: starting ${options.type}...`);
|
||||
this.#isRunning = 'import';
|
||||
const importStart = Date.now();
|
||||
|
||||
@@ -588,13 +571,10 @@ export class BackupsService {
|
||||
|
||||
window.ConversationController.setReadOnly(true);
|
||||
|
||||
const importStream = await BackupImportStream.create(
|
||||
backupType,
|
||||
localBackupSnapshotDir
|
||||
);
|
||||
if (backupType === BackupType.Ciphertext) {
|
||||
const importStream = await BackupImportStream.create(options);
|
||||
if (options.type === 'remote' || options.type === 'local-encrypted') {
|
||||
const { aesKey, macKey } = getKeyMaterial(
|
||||
ephemeralKey ? new BackupKey(ephemeralKey) : undefined
|
||||
options.ephemeralKey ? new BackupKey(options.ephemeralKey) : undefined
|
||||
);
|
||||
|
||||
// First pass - don't decrypt, only verify mac
|
||||
@@ -621,7 +601,7 @@ export class BackupsService {
|
||||
throw new BackupImportCanceledError();
|
||||
}
|
||||
|
||||
onProgress?.(0, totalBytes);
|
||||
options.onProgress?.(0, totalBytes);
|
||||
|
||||
strictAssert(theirMac != null, 'importBackup: Missing MAC');
|
||||
strictAssert(
|
||||
@@ -638,7 +618,7 @@ export class BackupsService {
|
||||
let currentBytes = 0;
|
||||
progressReporter.on('data', chunk => {
|
||||
currentBytes += chunk.byteLength;
|
||||
onProgress?.(currentBytes, totalBytes);
|
||||
options.onProgress?.(currentBytes, totalBytes);
|
||||
});
|
||||
|
||||
await pipeline(
|
||||
@@ -656,13 +636,13 @@ export class BackupsService {
|
||||
constantTimeEqual(hmac.digest(), theirMac),
|
||||
'importBackup: Bad MAC, second pass'
|
||||
);
|
||||
} else if (backupType === BackupType.TestOnlyPlaintext) {
|
||||
} else if (options.type === 'cross-client-integration-test') {
|
||||
strictAssert(
|
||||
isTestOrMockEnvironment(),
|
||||
'Plaintext backups can be imported only in test harness'
|
||||
);
|
||||
strictAssert(
|
||||
ephemeralKey == null,
|
||||
options.ephemeralKey == null,
|
||||
'Plaintext backups cannot have ephemeral key'
|
||||
);
|
||||
await pipeline(
|
||||
@@ -671,7 +651,7 @@ export class BackupsService {
|
||||
importStream
|
||||
);
|
||||
} else {
|
||||
throw missingCaseError(backupType);
|
||||
throw missingCaseError(options.type);
|
||||
}
|
||||
|
||||
log.info('importBackup: finished...');
|
||||
@@ -876,6 +856,7 @@ export class BackupsService {
|
||||
await itemStorage.remove('password');
|
||||
|
||||
await this.importFromDisk(downloadPath, {
|
||||
type: 'remote',
|
||||
ephemeralKey,
|
||||
onProgress: (currentBytes, totalBytes) => {
|
||||
onProgress?.(
|
||||
@@ -913,9 +894,7 @@ export class BackupsService {
|
||||
|
||||
async #exportBackup(
|
||||
sink: Writable,
|
||||
backupLevel: BackupLevel = BackupLevel.Free,
|
||||
backupType = BackupType.Ciphertext,
|
||||
localBackupSnapshotDir: string | undefined = undefined
|
||||
options: BackupExportOptions
|
||||
): Promise<ExportResultType> {
|
||||
strictAssert(!this.#isRunning, 'BackupService is already running');
|
||||
|
||||
@@ -925,7 +904,7 @@ export class BackupsService {
|
||||
const start = Date.now();
|
||||
try {
|
||||
// TODO (DESKTOP-7168): Update mock-server to support this endpoint
|
||||
if (window.SignalCI || backupType === BackupType.TestOnlyPlaintext) {
|
||||
if (window.SignalCI || options.type === 'cross-client-integration-test') {
|
||||
strictAssert(
|
||||
isTestOrMockEnvironment(),
|
||||
'Plaintext backups can be exported only in test harness'
|
||||
@@ -938,47 +917,52 @@ export class BackupsService {
|
||||
}
|
||||
|
||||
const { aesKey, macKey } = getKeyMaterial();
|
||||
const recordStream = new BackupExportStream(backupType);
|
||||
const recordStream = new BackupExportStream(options);
|
||||
|
||||
recordStream.run(backupLevel, localBackupSnapshotDir);
|
||||
recordStream.run();
|
||||
|
||||
const iv = randomBytes(IV_LENGTH);
|
||||
|
||||
let totalBytes = 0;
|
||||
|
||||
if (backupType === BackupType.Ciphertext) {
|
||||
await pipeline(
|
||||
recordStream,
|
||||
createGzip(),
|
||||
appendPaddingStream(),
|
||||
createCipheriv(CipherType.AES256CBC, aesKey, iv),
|
||||
prependStream(iv),
|
||||
appendMacStream(macKey),
|
||||
measureSize({
|
||||
onComplete: size => {
|
||||
totalBytes = size;
|
||||
},
|
||||
}),
|
||||
sink
|
||||
);
|
||||
} else if (backupType === BackupType.TestOnlyPlaintext) {
|
||||
strictAssert(
|
||||
isTestOrMockEnvironment(),
|
||||
'Plaintext backups can be exported only in test harness'
|
||||
);
|
||||
await pipeline(recordStream, sink);
|
||||
} else {
|
||||
throw missingCaseError(backupType);
|
||||
const { type } = options;
|
||||
switch (type) {
|
||||
case 'remote':
|
||||
case 'local-encrypted':
|
||||
await pipeline(
|
||||
recordStream,
|
||||
createGzip(),
|
||||
appendPaddingStream(),
|
||||
createCipheriv(CipherType.AES256CBC, aesKey, iv),
|
||||
prependStream(iv),
|
||||
appendMacStream(macKey),
|
||||
measureSize({
|
||||
onComplete: size => {
|
||||
totalBytes = size;
|
||||
},
|
||||
}),
|
||||
sink
|
||||
);
|
||||
break;
|
||||
case 'cross-client-integration-test':
|
||||
strictAssert(
|
||||
isTestOrMockEnvironment(),
|
||||
'Plaintext backups can be exported only in test harness'
|
||||
);
|
||||
await pipeline(recordStream, sink);
|
||||
break;
|
||||
default:
|
||||
throw missingCaseError(type);
|
||||
}
|
||||
|
||||
if (localBackupSnapshotDir) {
|
||||
if (type === 'local-encrypted') {
|
||||
log.info('exportBackup: writing local backup files list');
|
||||
const filesWritten = await writeLocalBackupFilesList({
|
||||
snapshotDir: localBackupSnapshotDir,
|
||||
snapshotDir: options.localBackupSnapshotDir,
|
||||
mediaNamesIterator: recordStream.getMediaNamesIterator(),
|
||||
});
|
||||
const filesRead = await readLocalBackupFilesList(
|
||||
localBackupSnapshotDir
|
||||
options.localBackupSnapshotDir
|
||||
);
|
||||
strictAssert(
|
||||
isEqual(filesWritten, filesRead),
|
||||
|
||||
@@ -25,10 +25,22 @@ export type AboutMe = {
|
||||
e164?: string;
|
||||
};
|
||||
|
||||
export enum BackupType {
|
||||
Ciphertext = 'Ciphertext',
|
||||
TestOnlyPlaintext = 'TestOnlyPlaintext',
|
||||
}
|
||||
export type BackupExportOptions =
|
||||
| { type: 'remote' | 'cross-client-integration-test'; level: BackupLevel }
|
||||
| {
|
||||
type: 'local-encrypted';
|
||||
localBackupSnapshotDir: string;
|
||||
};
|
||||
export type BackupImportOptions = (
|
||||
| { type: 'remote' | 'cross-client-integration-test' }
|
||||
| {
|
||||
type: 'local-encrypted';
|
||||
localBackupSnapshotDir: string;
|
||||
}
|
||||
) & {
|
||||
ephemeralKey?: Uint8Array;
|
||||
onProgress?: (currentBytes: number, totalBytes: number) => void;
|
||||
};
|
||||
|
||||
export type LocalChatStyle = Readonly<{
|
||||
wallpaperPhotoPointer: Uint8Array | undefined;
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
// Copyright 2024 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
import { existsSync } from 'node:fs';
|
||||
import { BackupLevel } from '@signalapp/libsignal-client/zkgroup.js';
|
||||
|
||||
import {
|
||||
APPLICATION_OCTET_STREAM,
|
||||
stringToMIMEType,
|
||||
} from '../../../types/MIME.std.js';
|
||||
import { createLogger } from '../../../logging/log.std.js';
|
||||
import type { AttachmentType } from '../../../types/Attachment.std.js';
|
||||
import { getAbsoluteAttachmentPath } from '../../../util/migrations.preload.js';
|
||||
import { doesAttachmentExist } from '../../../util/migrations.preload.js';
|
||||
import {
|
||||
hasRequiredInformationForBackup,
|
||||
hasRequiredInformationForLocalBackup,
|
||||
hasRequiredInformationForRemoteBackup,
|
||||
hasRequiredInformationToDownloadFromTransitTier,
|
||||
} from '../../../util/Attachment.std.js';
|
||||
import { Backups, SignalService } from '../../../protobuf/index.std.js';
|
||||
@@ -29,6 +28,10 @@ import {
|
||||
type GetBackupCdnInfoType,
|
||||
getMediaIdFromMediaName,
|
||||
getMediaName,
|
||||
getMediaNameForAttachment,
|
||||
type BackupCdnInfoType,
|
||||
getLocalBackupFileNameForAttachment,
|
||||
getLocalBackupFileName,
|
||||
} from './mediaId.preload.js';
|
||||
import { missingCaseError } from '../../../util/missingCaseError.std.js';
|
||||
import { bytesToUuid } from '../../../util/uuidToBytes.std.js';
|
||||
@@ -37,21 +40,16 @@ import { generateAttachmentKeys } from '../../../AttachmentCrypto.node.js';
|
||||
import { getAttachmentLocalBackupPathFromSnapshotDir } from './localBackup.node.js';
|
||||
import {
|
||||
isValidAttachmentKey,
|
||||
isValidDigest,
|
||||
isValidPlaintextHash,
|
||||
} from '../../../types/Crypto.std.js';
|
||||
import type { BackupExportOptions, BackupImportOptions } from '../types.std.js';
|
||||
import { isTestOrMockEnvironment } from '../../../environment.std.js';
|
||||
|
||||
const log = createLogger('filePointers');
|
||||
|
||||
type ConvertFilePointerToAttachmentOptions = {
|
||||
// Only for testing
|
||||
_createName: (suffix?: string) => string;
|
||||
localBackupSnapshotDir: string | undefined;
|
||||
};
|
||||
|
||||
export function convertFilePointerToAttachment(
|
||||
filePointer: Backups.FilePointer,
|
||||
options: Partial<ConvertFilePointerToAttachmentOptions> = {}
|
||||
options: BackupImportOptions,
|
||||
testDependencies?: { _createName: (suffix?: string) => string }
|
||||
): AttachmentType {
|
||||
const {
|
||||
contentType,
|
||||
@@ -64,7 +62,7 @@ export function convertFilePointerToAttachment(
|
||||
incrementalMacChunkSize,
|
||||
locatorInfo,
|
||||
} = filePointer;
|
||||
const doCreateName = options._createName ?? createName;
|
||||
const doCreateName = testDependencies?._createName ?? createName;
|
||||
|
||||
const commonProps: AttachmentType = {
|
||||
size: 0,
|
||||
@@ -124,24 +122,16 @@ export function convertFilePointerToAttachment(
|
||||
}
|
||||
|
||||
let localBackupPath: string | undefined;
|
||||
if (Bytes.isNotEmpty(localKey)) {
|
||||
const { localBackupSnapshotDir } = options;
|
||||
|
||||
strictAssert(
|
||||
localBackupSnapshotDir,
|
||||
'localBackupSnapshotDir is required for filePointer.localLocator'
|
||||
if (
|
||||
options.type === 'local-encrypted' &&
|
||||
Bytes.isNotEmpty(localKey) &&
|
||||
Bytes.isNotEmpty(plaintextHash)
|
||||
) {
|
||||
const localMediaName = getLocalBackupFileName({ plaintextHash, localKey });
|
||||
localBackupPath = getAttachmentLocalBackupPathFromSnapshotDir(
|
||||
localMediaName,
|
||||
options.localBackupSnapshotDir
|
||||
);
|
||||
|
||||
if (mediaName) {
|
||||
localBackupPath = getAttachmentLocalBackupPathFromSnapshotDir(
|
||||
mediaName,
|
||||
localBackupSnapshotDir
|
||||
);
|
||||
} else {
|
||||
log.error(
|
||||
'convertFilePointerToAttachment: localKey but no plaintextHash'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -172,7 +162,7 @@ export function convertFilePointerToAttachment(
|
||||
|
||||
export function convertBackupMessageAttachmentToAttachment(
|
||||
messageAttachment: Backups.IMessageAttachment,
|
||||
options: Partial<ConvertFilePointerToAttachmentOptions> = {}
|
||||
options: BackupImportOptions
|
||||
): AttachmentType | null {
|
||||
const { clientUuid } = messageAttachment;
|
||||
|
||||
@@ -207,21 +197,21 @@ export function convertBackupMessageAttachmentToAttachment(
|
||||
}
|
||||
|
||||
export async function getFilePointerForAttachment({
|
||||
attachment,
|
||||
attachment: rawAttachment,
|
||||
getBackupCdnInfo,
|
||||
backupLevel,
|
||||
backupOptions,
|
||||
messageReceivedAt,
|
||||
isLocalBackup = false,
|
||||
}: {
|
||||
attachment: Readonly<AttachmentType>;
|
||||
getBackupCdnInfo: GetBackupCdnInfoType;
|
||||
backupLevel: BackupLevel;
|
||||
backupOptions: BackupExportOptions;
|
||||
messageReceivedAt: number;
|
||||
isLocalBackup?: boolean;
|
||||
}): Promise<{
|
||||
filePointer: Backups.FilePointer;
|
||||
backupJob?: CoreAttachmentBackupJobType | PartialAttachmentLocalBackupJobType;
|
||||
}> {
|
||||
const attachment = maybeFixupAttachment(rawAttachment);
|
||||
|
||||
const filePointer = new Backups.FilePointer({
|
||||
contentType: attachment.contentType,
|
||||
fileName: attachment.fileName,
|
||||
@@ -241,67 +231,79 @@ export async function getFilePointerForAttachment({
|
||||
}
|
||||
}
|
||||
|
||||
const locatorInfo = getLocatorInfoForAttachment({
|
||||
const isAttachmentOnDisk =
|
||||
attachment.path != null && (await doesAttachmentExist(attachment.path));
|
||||
|
||||
const remoteMediaName = hasRequiredInformationForRemoteBackup(attachment)
|
||||
? getMediaNameForAttachment(attachment)
|
||||
: undefined;
|
||||
|
||||
const remoteMediaId = remoteMediaName
|
||||
? getMediaIdFromMediaName(remoteMediaName)
|
||||
: undefined;
|
||||
|
||||
const remoteBackupStatus: BackupCdnInfoType = remoteMediaId
|
||||
? await getBackupCdnInfo(remoteMediaId.string)
|
||||
: { isInBackupTier: false };
|
||||
|
||||
const isLocalBackup = backupOptions.type === 'local-encrypted';
|
||||
filePointer.locatorInfo = getLocatorInfoForAttachment({
|
||||
attachment,
|
||||
isLocalBackup,
|
||||
backupOptions,
|
||||
isOnDisk: isAttachmentOnDisk,
|
||||
backupTierInfo: remoteBackupStatus,
|
||||
});
|
||||
|
||||
if (locatorInfo) {
|
||||
filePointer.locatorInfo = locatorInfo;
|
||||
}
|
||||
|
||||
let backupJob:
|
||||
| CoreAttachmentBackupJobType
|
||||
| PartialAttachmentLocalBackupJobType
|
||||
| undefined;
|
||||
|
||||
if (backupLevel !== BackupLevel.Paid && !isLocalBackup) {
|
||||
return { filePointer, backupJob: undefined };
|
||||
}
|
||||
|
||||
if (!Bytes.isNotEmpty(locatorInfo.plaintextHash)) {
|
||||
return { filePointer, backupJob: undefined };
|
||||
}
|
||||
|
||||
const mediaName = getMediaName({
|
||||
plaintextHash: locatorInfo.plaintextHash,
|
||||
key: locatorInfo.key,
|
||||
});
|
||||
|
||||
const backupInfo = await getBackupCdnInfo(
|
||||
getMediaIdFromMediaName(mediaName).string
|
||||
);
|
||||
|
||||
if (backupInfo.isInBackupTier) {
|
||||
if (locatorInfo.mediaTierCdnNumber !== backupInfo.cdnNumber) {
|
||||
log.warn(
|
||||
'backupCdnNumber on attachment differs from cdnNumber from list endpoint'
|
||||
);
|
||||
// Prefer the one from the list endpoint
|
||||
locatorInfo.mediaTierCdnNumber = backupInfo.cdnNumber;
|
||||
}
|
||||
return { filePointer, backupJob: undefined };
|
||||
}
|
||||
|
||||
const { path, localKey, version, size } = attachment;
|
||||
|
||||
if (!path || !isValidAttachmentKey(localKey)) {
|
||||
return { filePointer, backupJob: undefined };
|
||||
}
|
||||
|
||||
if (isLocalBackup) {
|
||||
backupJob = {
|
||||
mediaName,
|
||||
type: 'local',
|
||||
data: {
|
||||
path,
|
||||
size,
|
||||
localKey,
|
||||
},
|
||||
if (
|
||||
isAttachmentOnDisk &&
|
||||
hasRequiredInformationForLocalBackup(attachment)
|
||||
) {
|
||||
return {
|
||||
filePointer,
|
||||
backupJob: {
|
||||
mediaName: getLocalBackupFileNameForAttachment(attachment),
|
||||
type: 'local',
|
||||
data: {
|
||||
path: attachment.path,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
filePointer,
|
||||
backupJob: undefined,
|
||||
};
|
||||
} else {
|
||||
backupJob = {
|
||||
mediaName,
|
||||
}
|
||||
|
||||
if (backupOptions.level !== BackupLevel.Paid) {
|
||||
return { filePointer, backupJob: undefined };
|
||||
}
|
||||
|
||||
if (remoteBackupStatus.isInBackupTier) {
|
||||
return { filePointer, backupJob: undefined };
|
||||
}
|
||||
|
||||
if (!isAttachmentOnDisk) {
|
||||
return { filePointer, backupJob: undefined };
|
||||
}
|
||||
|
||||
if (!remoteMediaName) {
|
||||
return { filePointer, backupJob: undefined };
|
||||
}
|
||||
|
||||
const { path, localKey, key, version } = attachment;
|
||||
|
||||
strictAssert(path, 'Path must exist for attachment on disk');
|
||||
strictAssert(key, 'Key must exist for remote backupable attachment');
|
||||
|
||||
const { transitCdnKey, transitCdnNumber, transitTierUploadTimestamp } =
|
||||
filePointer.locatorInfo;
|
||||
|
||||
return {
|
||||
filePointer,
|
||||
backupJob: {
|
||||
mediaName: remoteMediaName,
|
||||
receivedAt: messageReceivedAt,
|
||||
type: 'standard',
|
||||
data: {
|
||||
@@ -309,74 +311,86 @@ export async function getFilePointerForAttachment({
|
||||
localKey,
|
||||
version,
|
||||
contentType: attachment.contentType,
|
||||
keys: Bytes.toBase64(locatorInfo.key),
|
||||
size: locatorInfo.size,
|
||||
keys: key,
|
||||
size: attachment.size,
|
||||
transitCdnInfo:
|
||||
locatorInfo.transitCdnKey && locatorInfo.transitCdnNumber != null
|
||||
transitCdnKey && transitCdnNumber != null
|
||||
? {
|
||||
cdnKey: locatorInfo.transitCdnKey,
|
||||
cdnNumber: locatorInfo.transitCdnNumber,
|
||||
uploadTimestamp:
|
||||
locatorInfo.transitTierUploadTimestamp?.toNumber(),
|
||||
cdnKey: transitCdnKey,
|
||||
cdnNumber: transitCdnNumber,
|
||||
uploadTimestamp: transitTierUploadTimestamp?.toNumber(),
|
||||
}
|
||||
: undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return { filePointer, backupJob };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function maybeFixupAttachment(attachment: AttachmentType): AttachmentType {
|
||||
// Fixup attachment which has plaintextHash but no key
|
||||
if (
|
||||
isValidPlaintextHash(attachment.plaintextHash) &&
|
||||
!isValidAttachmentKey(attachment.key)
|
||||
) {
|
||||
const fixedUpAttachment = { ...attachment };
|
||||
fixedUpAttachment.key = Bytes.toBase64(generateAttachmentKeys());
|
||||
// Delete all info dependent on key
|
||||
delete fixedUpAttachment.cdnKey;
|
||||
delete fixedUpAttachment.cdnNumber;
|
||||
delete fixedUpAttachment.uploadTimestamp;
|
||||
delete fixedUpAttachment.digest;
|
||||
delete fixedUpAttachment.backupCdnNumber;
|
||||
|
||||
strictAssert(
|
||||
hasRequiredInformationForRemoteBackup(fixedUpAttachment),
|
||||
'should be backupable with new key'
|
||||
);
|
||||
return fixedUpAttachment;
|
||||
}
|
||||
return attachment;
|
||||
}
|
||||
function getLocatorInfoForAttachment({
|
||||
attachment: _rawAttachment,
|
||||
isLocalBackup,
|
||||
attachment,
|
||||
backupOptions,
|
||||
isOnDisk,
|
||||
backupTierInfo,
|
||||
}: {
|
||||
attachment: AttachmentType;
|
||||
isLocalBackup: boolean;
|
||||
backupOptions: BackupExportOptions;
|
||||
isOnDisk: boolean;
|
||||
backupTierInfo: BackupCdnInfoType;
|
||||
}): Backups.FilePointer.LocatorInfo {
|
||||
const locatorInfo = new Backups.FilePointer.LocatorInfo();
|
||||
const attachment = { ..._rawAttachment };
|
||||
|
||||
if (attachment.error) {
|
||||
return locatorInfo;
|
||||
}
|
||||
const isLocalBackup = backupOptions.type === 'local-encrypted';
|
||||
|
||||
{
|
||||
const isBackupable = hasRequiredInformationForBackup(attachment);
|
||||
const isDownloadableFromTransitTier =
|
||||
hasRequiredInformationToDownloadFromTransitTier(attachment);
|
||||
const shouldBeLocallyBackedUp =
|
||||
isLocalBackup &&
|
||||
isOnDisk &&
|
||||
hasRequiredInformationForLocalBackup(attachment);
|
||||
|
||||
if (!isBackupable && !isDownloadableFromTransitTier) {
|
||||
// TODO: DESKTOP-8914
|
||||
if (
|
||||
isValidPlaintextHash(attachment.plaintextHash) &&
|
||||
!isValidAttachmentKey(attachment.key)
|
||||
) {
|
||||
attachment.key = Bytes.toBase64(generateAttachmentKeys());
|
||||
// Delete all info dependent on key
|
||||
delete attachment.cdnKey;
|
||||
delete attachment.cdnNumber;
|
||||
delete attachment.uploadTimestamp;
|
||||
delete attachment.digest;
|
||||
delete attachment.backupCdnNumber;
|
||||
|
||||
strictAssert(
|
||||
hasRequiredInformationForBackup(attachment),
|
||||
'should be backupable with new key'
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
const isBackupable = hasRequiredInformationForBackup(attachment);
|
||||
const isDownloadableFromTransitTier =
|
||||
hasRequiredInformationToDownloadFromTransitTier(attachment);
|
||||
|
||||
if (!isBackupable && !isDownloadableFromTransitTier) {
|
||||
if (
|
||||
!shouldBeLocallyBackedUp &&
|
||||
!isDownloadableFromTransitTier &&
|
||||
!hasRequiredInformationForRemoteBackup(attachment)
|
||||
) {
|
||||
return locatorInfo;
|
||||
}
|
||||
|
||||
locatorInfo.size = attachment.size;
|
||||
locatorInfo.key = Bytes.fromBase64(attachment.key);
|
||||
|
||||
if (isValidAttachmentKey(attachment.key)) {
|
||||
locatorInfo.key = Bytes.fromBase64(attachment.key);
|
||||
}
|
||||
|
||||
if (isValidPlaintextHash(attachment.plaintextHash)) {
|
||||
locatorInfo.plaintextHash = Bytes.fromHex(attachment.plaintextHash);
|
||||
} else if (isValidDigest(attachment.digest)) {
|
||||
locatorInfo.encryptedDigest = Bytes.fromBase64(attachment.digest);
|
||||
}
|
||||
|
||||
if (isDownloadableFromTransitTier) {
|
||||
locatorInfo.transitCdnKey = attachment.cdnKey;
|
||||
@@ -386,25 +400,14 @@ function getLocatorInfoForAttachment({
|
||||
);
|
||||
}
|
||||
|
||||
if (isBackupable) {
|
||||
locatorInfo.plaintextHash = Bytes.fromHex(attachment.plaintextHash);
|
||||
// TODO: DESKTOP-8887
|
||||
if (attachment.backupCdnNumber != null) {
|
||||
locatorInfo.mediaTierCdnNumber = attachment.backupCdnNumber;
|
||||
}
|
||||
} else {
|
||||
locatorInfo.encryptedDigest = Bytes.fromBase64(attachment.digest);
|
||||
if (shouldBeLocallyBackedUp) {
|
||||
locatorInfo.localKey = Bytes.fromBase64(attachment.localKey);
|
||||
}
|
||||
|
||||
// TODO: DESKTOP-8904
|
||||
if (isLocalBackup && isBackupable) {
|
||||
const attachmentExistsLocally =
|
||||
attachment.path != null &&
|
||||
existsSync(getAbsoluteAttachmentPath(attachment.path));
|
||||
|
||||
if (attachmentExistsLocally && attachment.localKey) {
|
||||
locatorInfo.localKey = Bytes.fromBase64(attachment.localKey);
|
||||
}
|
||||
if (backupTierInfo.isInBackupTier && backupTierInfo.cdnNumber != null) {
|
||||
locatorInfo.mediaTierCdnNumber = backupTierInfo.cdnNumber;
|
||||
} else if (backupOptions.type === 'cross-client-integration-test') {
|
||||
locatorInfo.mediaTierCdnNumber = attachment.backupCdnNumber;
|
||||
}
|
||||
|
||||
return locatorInfo;
|
||||
|
||||
@@ -4,7 +4,11 @@
|
||||
import { DataReader } from '../../../sql/Client.preload.js';
|
||||
import * as Bytes from '../../../Bytes.std.js';
|
||||
import { getBackupMediaRootKey } from '../crypto.preload.js';
|
||||
import type { BackupableAttachmentType } from '../../../types/Attachment.std.js';
|
||||
import type {
|
||||
BackupableAttachmentType,
|
||||
AttachmentReadyForLocalBackup,
|
||||
} from '../../../types/Attachment.std.js';
|
||||
import { sha256 } from '../../../Crypto.node.js';
|
||||
|
||||
export function getMediaIdFromMediaName(mediaName: string): {
|
||||
string: string;
|
||||
@@ -56,6 +60,25 @@ export function getMediaName({
|
||||
return Bytes.toHex(Bytes.concatenate([plaintextHash, key]));
|
||||
}
|
||||
|
||||
export function getLocalBackupFileNameForAttachment(
|
||||
attachment: AttachmentReadyForLocalBackup
|
||||
): string {
|
||||
return getLocalBackupFileName({
|
||||
plaintextHash: Bytes.fromHex(attachment.plaintextHash),
|
||||
localKey: Bytes.fromBase64(attachment.localKey),
|
||||
});
|
||||
}
|
||||
|
||||
export function getLocalBackupFileName({
|
||||
plaintextHash,
|
||||
localKey,
|
||||
}: {
|
||||
plaintextHash: Uint8Array;
|
||||
localKey: Uint8Array;
|
||||
}): string {
|
||||
return Bytes.toHex(sha256(Bytes.concatenate([plaintextHash, localKey])));
|
||||
}
|
||||
|
||||
export function getMediaNameForAttachmentThumbnail(
|
||||
fullsizeMediaName: string
|
||||
): `${string}_thumbnail` {
|
||||
|
||||
@@ -26,7 +26,7 @@ import type {
|
||||
QuotedMessageType,
|
||||
} from '../../model-types.d.ts';
|
||||
import {
|
||||
hasRequiredInformationForBackup,
|
||||
hasRequiredInformationForRemoteBackup,
|
||||
isVoiceMessage,
|
||||
} from '../../util/Attachment.std.js';
|
||||
import type { AttachmentType } from '../../types/Attachment.std.js';
|
||||
@@ -128,7 +128,7 @@ describe('backup/attachments', () => {
|
||||
attachment: AttachmentType
|
||||
): AttachmentType {
|
||||
const base = omit(attachment, NON_ROUNDTRIPPED_FIELDS);
|
||||
if (hasRequiredInformationForBackup(attachment)) {
|
||||
if (hasRequiredInformationForRemoteBackup(attachment)) {
|
||||
delete base.digest;
|
||||
} else {
|
||||
delete base.plaintextHash;
|
||||
@@ -793,7 +793,7 @@ describe('backup/attachments', () => {
|
||||
strictAssert(key, 'thumbnail key was created');
|
||||
strictAssert(plaintextHash, 'quote plaintextHash was roundtripped');
|
||||
strictAssert(
|
||||
hasRequiredInformationForBackup(thumbnail),
|
||||
hasRequiredInformationForRemoteBackup(thumbnail),
|
||||
'has key and plaintextHash'
|
||||
);
|
||||
assert.deepStrictEqual(thumbnail, {
|
||||
|
||||
@@ -6,6 +6,7 @@ import * as sinon from 'sinon';
|
||||
import { BackupLevel } from '@signalapp/libsignal-client/zkgroup.js';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { join } from 'node:path';
|
||||
import { emptyDir, ensureFile } from 'fs-extra';
|
||||
|
||||
import { Backups } from '../../protobuf/index.std.js';
|
||||
|
||||
@@ -22,6 +23,9 @@ import type { GetBackupCdnInfoType } from '../../services/backups/util/mediaId.p
|
||||
import { strictAssert } from '../../util/assert.std.js';
|
||||
import { isValidAttachmentKey } from '../../types/Crypto.std.js';
|
||||
import { itemStorage } from '../../textsecure/Storage.preload.js';
|
||||
import { getAbsoluteAttachmentPath } from '../../util/migrations.preload.js';
|
||||
import { getPath } from '../../../app/attachments.node.js';
|
||||
import { sha256 } from '../../Crypto.node.js';
|
||||
|
||||
describe('convertFilePointerToAttachment', () => {
|
||||
const commonFilePointerProps = {
|
||||
@@ -52,6 +56,7 @@ describe('convertFilePointerToAttachment', () => {
|
||||
...commonFilePointerProps,
|
||||
locatorInfo: {},
|
||||
}),
|
||||
{ type: 'remote' },
|
||||
{ _createName: () => 'downloadPath' }
|
||||
);
|
||||
|
||||
@@ -65,6 +70,7 @@ describe('convertFilePointerToAttachment', () => {
|
||||
it('processes filepointer with missing locatorInfo', () => {
|
||||
const result = convertFilePointerToAttachment(
|
||||
new Backups.FilePointer(commonFilePointerProps),
|
||||
{ type: 'remote' },
|
||||
{ _createName: () => 'downloadPath' }
|
||||
);
|
||||
|
||||
@@ -90,6 +96,7 @@ describe('convertFilePointerToAttachment', () => {
|
||||
mediaTierCdnNumber: 43,
|
||||
},
|
||||
}),
|
||||
{ type: 'remote' },
|
||||
{ _createName: () => 'downloadPath' }
|
||||
);
|
||||
|
||||
@@ -123,17 +130,19 @@ describe('convertFilePointerToAttachment', () => {
|
||||
localKey: Bytes.fromString('localKey'),
|
||||
},
|
||||
}),
|
||||
{ type: 'local-encrypted', localBackupSnapshotDir: '/root/backups' },
|
||||
{
|
||||
_createName: () => 'downloadPath',
|
||||
localBackupSnapshotDir: '/root/backups',
|
||||
}
|
||||
);
|
||||
|
||||
const mediaName = Bytes.toHex(
|
||||
Bytes.concatenate([
|
||||
Bytes.fromString('plaintextHash'),
|
||||
Bytes.fromString('key'),
|
||||
])
|
||||
sha256(
|
||||
Bytes.concatenate([
|
||||
Bytes.fromString('plaintextHash'),
|
||||
Bytes.fromString('localKey'),
|
||||
])
|
||||
)
|
||||
);
|
||||
assert.deepStrictEqual(result, {
|
||||
...commonAttachmentProps,
|
||||
@@ -207,7 +216,7 @@ const notInBackupCdn: GetBackupCdnInfoType = async () => {
|
||||
describe('getFilePointerForAttachment', () => {
|
||||
let sandbox: sinon.SinonSandbox;
|
||||
|
||||
beforeEach(() => {
|
||||
beforeEach(async () => {
|
||||
sandbox = sinon.createSandbox();
|
||||
sandbox.stub(itemStorage, 'get').callsFake(key => {
|
||||
if (key === 'masterKey') {
|
||||
@@ -218,16 +227,21 @@ describe('getFilePointerForAttachment', () => {
|
||||
}
|
||||
return undefined;
|
||||
});
|
||||
await ensureFile(getAbsoluteAttachmentPath(defaultAttachment.path));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
afterEach(async () => {
|
||||
sandbox.restore();
|
||||
await emptyDir(getPath(window.SignalContext.config.userDataPath));
|
||||
});
|
||||
|
||||
it('if missing key, generates a new one and removes transit info & digest', async () => {
|
||||
const { filePointer } = await getFilePointerForAttachment({
|
||||
attachment: { ...defaultAttachment, key: undefined },
|
||||
backupLevel: BackupLevel.Paid,
|
||||
backupOptions: {
|
||||
type: 'remote',
|
||||
level: BackupLevel.Paid,
|
||||
},
|
||||
getBackupCdnInfo: notInBackupCdn,
|
||||
messageReceivedAt: 100,
|
||||
});
|
||||
@@ -254,7 +268,10 @@ describe('getFilePointerForAttachment', () => {
|
||||
assert.deepEqual(
|
||||
await getFilePointerForAttachment({
|
||||
attachment: { ...defaultAttachment, plaintextHash: undefined },
|
||||
backupLevel: BackupLevel.Paid,
|
||||
backupOptions: {
|
||||
type: 'remote',
|
||||
level: BackupLevel.Paid,
|
||||
},
|
||||
getBackupCdnInfo: notInBackupCdn,
|
||||
messageReceivedAt: 100,
|
||||
}),
|
||||
@@ -278,7 +295,10 @@ describe('getFilePointerForAttachment', () => {
|
||||
assert.deepEqual(
|
||||
await getFilePointerForAttachment({
|
||||
attachment: defaultAttachment,
|
||||
backupLevel: BackupLevel.Free,
|
||||
backupOptions: {
|
||||
type: 'remote',
|
||||
level: BackupLevel.Free,
|
||||
},
|
||||
getBackupCdnInfo: notInBackupCdn,
|
||||
messageReceivedAt: 100,
|
||||
}),
|
||||
@@ -292,7 +312,6 @@ describe('getFilePointerForAttachment', () => {
|
||||
transitCdnKey: 'cdnKey',
|
||||
transitCdnNumber: 2,
|
||||
transitTierUploadTimestamp: Long.fromNumber(1234),
|
||||
mediaTierCdnNumber: 42,
|
||||
}),
|
||||
}),
|
||||
backupJob: undefined,
|
||||
@@ -304,7 +323,10 @@ describe('getFilePointerForAttachment', () => {
|
||||
assert.deepEqual(
|
||||
await getFilePointerForAttachment({
|
||||
attachment: { ...defaultAttachment, digest: undefined },
|
||||
backupLevel: BackupLevel.Free,
|
||||
backupOptions: {
|
||||
type: 'remote',
|
||||
level: BackupLevel.Free,
|
||||
},
|
||||
getBackupCdnInfo: notInBackupCdn,
|
||||
messageReceivedAt: 100,
|
||||
}),
|
||||
@@ -318,7 +340,6 @@ describe('getFilePointerForAttachment', () => {
|
||||
transitCdnKey: 'cdnKey',
|
||||
transitCdnNumber: 2,
|
||||
transitTierUploadTimestamp: Long.fromNumber(1234),
|
||||
mediaTierCdnNumber: 42,
|
||||
}),
|
||||
}),
|
||||
backupJob: undefined,
|
||||
@@ -330,7 +351,10 @@ describe('getFilePointerForAttachment', () => {
|
||||
assert.deepEqual(
|
||||
await getFilePointerForAttachment({
|
||||
attachment: { ...defaultAttachment, cdnKey: undefined },
|
||||
backupLevel: BackupLevel.Free,
|
||||
backupOptions: {
|
||||
type: 'remote',
|
||||
level: BackupLevel.Free,
|
||||
},
|
||||
getBackupCdnInfo: notInBackupCdn,
|
||||
messageReceivedAt: 100,
|
||||
}),
|
||||
@@ -341,7 +365,6 @@ describe('getFilePointerForAttachment', () => {
|
||||
plaintextHash: Bytes.fromHex(defaultAttachment.plaintextHash),
|
||||
key: Bytes.fromBase64(defaultAttachment.key),
|
||||
size: 100,
|
||||
mediaTierCdnNumber: 42,
|
||||
}),
|
||||
}),
|
||||
backupJob: undefined,
|
||||
@@ -352,7 +375,10 @@ describe('getFilePointerForAttachment', () => {
|
||||
assert.deepEqual(
|
||||
await getFilePointerForAttachment({
|
||||
attachment: defaultAttachment,
|
||||
backupLevel: BackupLevel.Paid,
|
||||
backupOptions: {
|
||||
type: 'remote',
|
||||
level: BackupLevel.Paid,
|
||||
},
|
||||
getBackupCdnInfo: notInBackupCdn,
|
||||
messageReceivedAt: 100,
|
||||
}),
|
||||
@@ -366,7 +392,6 @@ describe('getFilePointerForAttachment', () => {
|
||||
transitCdnKey: 'cdnKey',
|
||||
transitCdnNumber: 2,
|
||||
transitTierUploadTimestamp: Long.fromNumber(1234),
|
||||
mediaTierCdnNumber: 42,
|
||||
}),
|
||||
}),
|
||||
backupJob: {
|
||||
@@ -390,14 +415,16 @@ describe('getFilePointerForAttachment', () => {
|
||||
}
|
||||
);
|
||||
});
|
||||
it('if local backup includes local backup job', async () => {
|
||||
it('does not include backup job if file does not exist', async () => {
|
||||
assert.deepEqual(
|
||||
await getFilePointerForAttachment({
|
||||
attachment: defaultAttachment,
|
||||
backupLevel: BackupLevel.Paid,
|
||||
attachment: { ...defaultAttachment, path: 'not/here' },
|
||||
backupOptions: {
|
||||
type: 'remote',
|
||||
level: BackupLevel.Paid,
|
||||
},
|
||||
getBackupCdnInfo: notInBackupCdn,
|
||||
messageReceivedAt: 100,
|
||||
isLocalBackup: true,
|
||||
}),
|
||||
{
|
||||
filePointer: new FilePointer({
|
||||
@@ -409,19 +436,82 @@ describe('getFilePointerForAttachment', () => {
|
||||
transitCdnKey: 'cdnKey',
|
||||
transitCdnNumber: 2,
|
||||
transitTierUploadTimestamp: Long.fromNumber(1234),
|
||||
mediaTierCdnNumber: 42,
|
||||
}),
|
||||
}),
|
||||
backupJob: {
|
||||
data: {
|
||||
localKey: defaultAttachment.localKey,
|
||||
path: defaultAttachment.path,
|
||||
size: 100,
|
||||
},
|
||||
mediaName: defaultMediaName,
|
||||
type: 'local',
|
||||
},
|
||||
backupJob: undefined,
|
||||
}
|
||||
);
|
||||
});
|
||||
describe('local backups', () => {
|
||||
const defaultLocalMediaName = Bytes.toHex(
|
||||
sha256(
|
||||
Bytes.concatenate([
|
||||
Bytes.fromHex(defaultAttachment.plaintextHash),
|
||||
Bytes.fromBase64(defaultAttachment.localKey),
|
||||
])
|
||||
)
|
||||
);
|
||||
|
||||
it('generates local backup locatorInfo and a local backup job', async () => {
|
||||
assert.deepEqual(
|
||||
await getFilePointerForAttachment({
|
||||
attachment: defaultAttachment,
|
||||
backupOptions: {
|
||||
type: 'local-encrypted',
|
||||
localBackupSnapshotDir: '/root/backups',
|
||||
},
|
||||
getBackupCdnInfo: notInBackupCdn,
|
||||
messageReceivedAt: 100,
|
||||
}),
|
||||
{
|
||||
filePointer: new FilePointer({
|
||||
...defaultFilePointer,
|
||||
locatorInfo: new LocatorInfo({
|
||||
plaintextHash: Bytes.fromHex(defaultAttachment.plaintextHash),
|
||||
localKey: Bytes.fromBase64(defaultAttachment.localKey),
|
||||
key: Bytes.fromBase64(defaultAttachment.key),
|
||||
size: 100,
|
||||
transitCdnKey: 'cdnKey',
|
||||
transitCdnNumber: 2,
|
||||
transitTierUploadTimestamp: Long.fromNumber(1234),
|
||||
}),
|
||||
}),
|
||||
backupJob: {
|
||||
data: {
|
||||
path: defaultAttachment.path,
|
||||
},
|
||||
mediaName: defaultLocalMediaName,
|
||||
type: 'local',
|
||||
},
|
||||
}
|
||||
);
|
||||
});
|
||||
it('if file does not exist, does not include localKey or backup job', async () => {
|
||||
assert.deepEqual(
|
||||
await getFilePointerForAttachment({
|
||||
attachment: { ...defaultAttachment, path: 'no/file/here' },
|
||||
backupOptions: {
|
||||
type: 'local-encrypted',
|
||||
localBackupSnapshotDir: '/root/backups',
|
||||
},
|
||||
getBackupCdnInfo: notInBackupCdn,
|
||||
messageReceivedAt: 100,
|
||||
}),
|
||||
{
|
||||
filePointer: new FilePointer({
|
||||
...defaultFilePointer,
|
||||
locatorInfo: new LocatorInfo({
|
||||
plaintextHash: Bytes.fromHex(defaultAttachment.plaintextHash),
|
||||
key: Bytes.fromBase64(defaultAttachment.key),
|
||||
size: 100,
|
||||
transitCdnKey: 'cdnKey',
|
||||
transitCdnNumber: 2,
|
||||
transitTierUploadTimestamp: Long.fromNumber(1234),
|
||||
}),
|
||||
}),
|
||||
backupJob: undefined,
|
||||
}
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -228,13 +228,19 @@ export async function asymmetricRoundtripHarness(
|
||||
postSaveUpdates,
|
||||
});
|
||||
|
||||
await backupsService.exportToDisk(targetOutputFile, options.backupLevel);
|
||||
await backupsService.exportToDisk(targetOutputFile, {
|
||||
type: 'remote',
|
||||
level: options.backupLevel,
|
||||
});
|
||||
|
||||
await updateConvoIdToTitle();
|
||||
|
||||
await clearData();
|
||||
|
||||
await backupsService.importBackup(() => createReadStream(targetOutputFile));
|
||||
await backupsService.importBackup(
|
||||
() => createReadStream(targetOutputFile),
|
||||
{ type: 'remote' }
|
||||
);
|
||||
|
||||
const messagesFromDatabase = await DataReader._getAllMessages();
|
||||
|
||||
|
||||
@@ -14,10 +14,7 @@ import { assert } from 'chai';
|
||||
|
||||
import { clearData } from './helpers.preload.js';
|
||||
import { loadAllAndReinitializeRedux } from '../../services/allLoaders.preload.js';
|
||||
import {
|
||||
backupsService,
|
||||
BackupType,
|
||||
} from '../../services/backups/index.preload.js';
|
||||
import { backupsService } from '../../services/backups/index.preload.js';
|
||||
import { initialize as initializeExpiringMessageService } from '../../services/expiringMessagesDeletion.preload.js';
|
||||
import { MemoryStream } from '../../util/MemoryStream.node.js';
|
||||
|
||||
@@ -56,13 +53,13 @@ describe('backup/integration', () => {
|
||||
const expectedBuffer = await readFile(fullPath);
|
||||
|
||||
await backupsService.importBackup(() => Readable.from([expectedBuffer]), {
|
||||
backupType: BackupType.TestOnlyPlaintext,
|
||||
type: 'cross-client-integration-test',
|
||||
});
|
||||
|
||||
const { data: exported } = await backupsService.exportBackupData(
|
||||
BackupLevel.Paid,
|
||||
BackupType.TestOnlyPlaintext
|
||||
);
|
||||
const { data: exported } = await backupsService.exportBackupData({
|
||||
type: 'cross-client-integration-test',
|
||||
level: BackupLevel.Paid,
|
||||
});
|
||||
|
||||
const actualStream = new MemoryStream(Buffer.from(exported));
|
||||
const expectedStream = new MemoryStream(expectedBuffer);
|
||||
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
getAbsoluteDownloadsPath,
|
||||
getAbsoluteAttachmentPath,
|
||||
} from '../util/migrations.preload.js';
|
||||
import { hasRequiredInformationForBackup } from '../util/Attachment.std.js';
|
||||
import { hasRequiredInformationForRemoteBackup } from '../util/Attachment.std.js';
|
||||
import {
|
||||
AttachmentSizeError,
|
||||
type AttachmentType,
|
||||
@@ -200,7 +200,7 @@ export async function downloadAttachment(
|
||||
}
|
||||
if (mediaTier === MediaTier.BACKUP) {
|
||||
strictAssert(
|
||||
hasRequiredInformationForBackup(attachment),
|
||||
hasRequiredInformationForRemoteBackup(attachment),
|
||||
`${logId}: attachment missing critical information for backup tier`
|
||||
);
|
||||
}
|
||||
|
||||
@@ -233,6 +233,10 @@ export type BackupableAttachmentType = WithRequiredProperties<
|
||||
AttachmentType,
|
||||
'plaintextHash' | 'key'
|
||||
>;
|
||||
export type AttachmentReadyForLocalBackup = WithRequiredProperties<
|
||||
AttachmentType,
|
||||
'plaintextHash' | 'localKey' | 'path'
|
||||
>;
|
||||
|
||||
export type AttachmentDownloadableFromTransitTier = WithRequiredProperties<
|
||||
AttachmentType,
|
||||
|
||||
@@ -48,8 +48,6 @@ export type CoreAttachmentLocalBackupJobType = {
|
||||
mediaName: string;
|
||||
data: {
|
||||
path: string | null;
|
||||
size: number;
|
||||
localKey: string;
|
||||
};
|
||||
backupsBaseDir: string;
|
||||
};
|
||||
|
||||
@@ -12,6 +12,7 @@ import type {
|
||||
BackupableAttachmentType,
|
||||
AttachmentDownloadableFromTransitTier,
|
||||
LocallySavedAttachment,
|
||||
AttachmentReadyForLocalBackup,
|
||||
} from '../types/Attachment.std.js';
|
||||
import type { LoggerType } from '../types/Logging.std.js';
|
||||
import { createLogger } from '../logging/log.std.js';
|
||||
@@ -847,7 +848,7 @@ export function getCachedAttachmentBySignature<T>(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function hasRequiredInformationForBackup(
|
||||
export function hasRequiredInformationForRemoteBackup(
|
||||
attachment: AttachmentType
|
||||
): attachment is BackupableAttachmentType {
|
||||
return (
|
||||
@@ -856,13 +857,23 @@ export function hasRequiredInformationForBackup(
|
||||
);
|
||||
}
|
||||
|
||||
export function hasRequiredInformationForLocalBackup(
|
||||
attachment: AttachmentType
|
||||
): attachment is AttachmentReadyForLocalBackup {
|
||||
return (
|
||||
isValidAttachmentKey(attachment.localKey) &&
|
||||
isValidPlaintextHash(attachment.plaintextHash) &&
|
||||
Boolean(attachment.path)
|
||||
);
|
||||
}
|
||||
|
||||
export function wasImportedFromLocalBackup(
|
||||
attachment: AttachmentType
|
||||
): attachment is BackupableAttachmentType {
|
||||
return (
|
||||
hasRequiredInformationForBackup(attachment) &&
|
||||
Boolean(attachment.localBackupPath) &&
|
||||
isValidAttachmentKey(attachment.localKey)
|
||||
isValidPlaintextHash(attachment.plaintextHash) &&
|
||||
isValidAttachmentKey(attachment.localKey) &&
|
||||
Boolean(attachment.localBackupPath)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -900,7 +911,7 @@ export function shouldAttachmentEndUpInRemoteBackup({
|
||||
attachment: AttachmentType;
|
||||
hasMediaBackups: boolean;
|
||||
}): boolean {
|
||||
return hasMediaBackups && hasRequiredInformationForBackup(attachment);
|
||||
return hasMediaBackups && hasRequiredInformationForRemoteBackup(attachment);
|
||||
}
|
||||
|
||||
export function isDownloadable(attachment: AttachmentType): boolean {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
import { ErrorCode, LibSignalErrorBase } from '@signalapp/libsignal-client';
|
||||
import {
|
||||
hasRequiredInformationForBackup,
|
||||
hasRequiredInformationForRemoteBackup,
|
||||
wasImportedFromLocalBackup,
|
||||
} from './Attachment.std.js';
|
||||
import {
|
||||
@@ -58,7 +58,7 @@ export async function downloadAttachment({
|
||||
variant !== AttachmentVariant.Default ? `[${variant}]` : '';
|
||||
const logId = `${_logId}${variantForLogging}`;
|
||||
|
||||
const isBackupable = hasRequiredInformationForBackup(attachment);
|
||||
const isBackupable = hasRequiredInformationForRemoteBackup(attachment);
|
||||
|
||||
const mightBeOnBackupTierNow = isBackupable && hasMediaBackups;
|
||||
const mightBeOnBackupTierInTheFuture = isBackupable;
|
||||
|
||||
Reference in New Issue
Block a user