Signal/Desktop
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Desktop.git synced 2025-12-24 12:19:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Backups: remove legacy locators

Browse Source 
Co-authored-by: trevor-signal <131492920+trevor-signal@users.noreply.github.com>
This commit is contained in:
automated-signal
2025-08-04 09:15:12 -05:00
committed by GitHub
parent c9a3979259
commit 268b63ef60
6 changed files with 136 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
protos/Backups.proto
View File

@@ -692,73 +692,13 @@ message MessageAttachment {
}
message FilePointer {
// References attachments in the backup (media) storage tier.
// DEPRECATED; use LocatorInfo instead if available.
message BackupLocator {
string mediaName = 1;
// If present, the cdn number of the succesful upload.
// If empty/0, may still have been uploaded, and clients
// can discover the cdn number via the list endpoint.
optional uint32 cdnNumber = 2;
bytes key = 3;
bytes digest = 4;
uint32 size = 5;
// Fallback in case backup tier upload failed.
optional string transitCdnKey = 6;
optional uint32 transitCdnNumber = 7;
}
// References attachments in the transit storage tier.
// May be downloaded or not when the backup is generated;
// primarily for free-tier users who cannot copy the
// attachments to the backup (media) storage tier.
// DEPRECATED; use LocatorInfo instead if available.
message AttachmentLocator {
string cdnKey = 1;
uint32 cdnNumber = 2;
optional uint64 uploadTimestamp = 3;
bytes key = 4;
bytes digest = 5;
uint32 size = 6;
}
// References attachments that are invalid in such a way where download
// cannot be attempted. Could range from missing digests to missing
// CDN keys or anything else that makes download attempts impossible.
// This serves as a 'tombstone' so that the UX can show that an attachment
// did exist, but for whatever reason it's not retrievable.
// DEPRECATED; use LocatorInfo instead if available.
message InvalidAttachmentLocator {
}
// References attachments in a local encrypted backup.
// Importers should first attempt to read the file from the local backup,
// and on failure fallback to backup and transit cdn if possible.
// DEPRECATED; use LocatorInfo instead if available.
message LocalLocator {
string mediaName = 1;
// Separate key used to encrypt this file for the local backup.
// Generally required. Missing field indicates attachment was not
// available locally when the backup was generated, but remote
// backup or transit info was available.
optional bytes localKey = 2;
bytes remoteKey = 3;
bytes remoteDigest = 4;
uint32 size = 5;
optional uint32 backupCdnNumber = 6;
optional string transitCdnKey = 7;
optional uint32 transitCdnNumber = 8;
}
message LocatorInfo {
// Must be non-empty if transitCdnKey or plaintextHash are set/nonempty.
// Otherwise must be empty.
bytes key = 1;
// From the sender of the attachment (incl. ourselves)
// Will be reserved once all clients start reading integrityCheck
bytes legacyDigest = 2;
reserved /*legacyDigest*/ 2;
oneof integrityCheck {
// Set if file was at one point downloaded and its plaintextHash was calculated
@@ -786,11 +726,7 @@ message FilePointer {
// has not rotated since last upload; even if currently free tier.
optional uint32 mediaTierCdnNumber = 7;
// Nonempty any time the attachment was downloaded and its
// digest validated, whether free tier or paid subscription.
// Will be reserved once all clients start reading integrityCheck,
// when mediaName will be derived from the plaintextHash and encryption key
string legacyMediaName = 8;
reserved /*legacyMediaName*/ 8;
// Separate key used to encrypt this file for the local backup.
// Generally required for local backups.
@@ -800,14 +736,10 @@ message FilePointer {
optional bytes localKey = 9;
}
// If unset, importers should consider it to be an InvalidAttachmentLocator without throwing an error.
// DEPRECATED; use locatorInfo instead.
oneof locator {
BackupLocator backupLocator = 1;
AttachmentLocator attachmentLocator = 2;
InvalidAttachmentLocator invalidAttachmentLocator = 3;
LocalLocator localLocator = 12;
}
reserved /*backupLocator*/ 1;
reserved /*attachmentLocator*/ 2;
reserved /*invalidAttachmentLocator*/ 3;
reserved /*localLocator*/ 12;
optional string contentType = 4;
optional bytes incrementalMac = 5;

55
ts/services/backups/export.ts
View File

@@ -124,6 +124,7 @@ import {
type AttachmentType,
isGIF,
isDownloaded,
hasRequiredInformationForBackup,
} from '../../types/Attachment';
import { getFilePointerForAttachment } from './util/filePointers';
import { getBackupMediaRootKey } from './crypto';
@@ -133,7 +134,7 @@ import type {
} from '../../types/AttachmentBackup';
import { AttachmentBackupManager } from '../../jobs/AttachmentBackupManager';
import { AttachmentLocalBackupManager } from '../../jobs/AttachmentLocalBackupManager';
import { getBackupCdnInfo } from './util/mediaId';
import { getBackupCdnInfo, getMediaNameForAttachment } from './util/mediaId';
import { calculateExpirationTimestamp } from '../../util/expirationTimer';
import { ReadStatus } from '../../messages/MessageReadStatus';
import { CallLinkRestrictions } from '../../types/CallLink';
@@ -245,7 +246,10 @@ export class BackupExportStream extends Readable {
readonly #serviceIdToRecipientId = new Map<string, number>();
readonly #e164ToRecipientId = new Map<string, number>();
readonly #roomIdToRecipientId = new Map<string, number>();
readonly #mediaNamesToFilePointers = new Map<string, Backups.FilePointer>();
readonly #mediaNamesToLocatorInfos = new Map<
string,
Backups.FilePointer.ILocatorInfo
>();
readonly #stats: StatsType = {
adHocCalls: 0,
callLinks: 0,
@@ -348,7 +352,7 @@ export class BackupExportStream extends Readable {
}
public getMediaNamesIterator(): MapIterator<string> {
return this.#mediaNamesToFilePointers.keys();
return this.#mediaNamesToLocatorInfos.keys();
}
public getStats(): Readonly<StatsType> {
@@ -2595,33 +2599,26 @@ export class BackupExportStream extends Readable {
getBackupCdnInfo,
});
// TODO: DESKTOP-8887
if (isLocalBackup && filePointer.localLocator) {
// Duplicate attachment check. Local backups can only contain 1 file per mediaName,
// so if we see a duplicate mediaName then we must reuse the previous FilePointer.
const { mediaName } = filePointer.localLocator;
strictAssert(
mediaName,
'FilePointer.LocalLocator must contain mediaName'
);
const existingFilePointer = this.#mediaNamesToFilePointers.get(mediaName);
if (existingFilePointer) {
strictAssert(
existingFilePointer.localLocator,
'Local backup existing mediaName FilePointer must contain LocalLocator'
);
strictAssert(
existingFilePointer.localLocator.size === attachment.size,
'Local backup existing mediaName FilePointer size must match attachment'
);
return existingFilePointer;
if (hasRequiredInformationForBackup(attachment)) {
const mediaName = getMediaNameForAttachment(attachment);
// Re-use existing locatorInfo and backup job if we've already seen this file
const existingLocatorInfo = this.#mediaNamesToLocatorInfos.get(mediaName);
if (existingLocatorInfo) {
filePointer.locatorInfo = existingLocatorInfo;
} else {
if (filePointer.locatorInfo) {
this.#mediaNamesToLocatorInfos.set(
mediaName,
filePointer.locatorInfo
);
}
if (backupJob) {
this.#attachmentBackupJobs.push(backupJob);
}
}
this.#mediaNamesToFilePointers.set(mediaName, filePointer);
}
if (backupJob) {
this.#attachmentBackupJobs.push(backupJob);
}
return filePointer;

230
ts/services/backups/util/filePointers.ts
View File

@@ -84,201 +84,87 @@ export function convertFilePointerToAttachment(
commonProps.chunkSize = incrementalMacChunkSize;
}
if (locatorInfo) {
const {
key,
localKey,
legacyDigest,
legacyMediaName,
plaintextHash,
encryptedDigest,
size,
transitCdnKey,
transitCdnNumber,
transitTierUploadTimestamp,
mediaTierCdnNumber,
} = locatorInfo;
if (!Bytes.isNotEmpty(key)) {
return {
...commonProps,
error: true,
size: 0,
downloadPath: undefined,
};
}
const digest = Bytes.isNotEmpty(encryptedDigest)
? encryptedDigest
: legacyDigest;
let mediaName: string | undefined;
if (Bytes.isNotEmpty(plaintextHash) && Bytes.isNotEmpty(key)) {
mediaName =
getMediaName({
key,
plaintextHash,
}) ?? undefined;
} else if (legacyMediaName) {
mediaName = legacyMediaName;
}
let localBackupPath: string | undefined;
if (Bytes.isNotEmpty(localKey)) {
const { localBackupSnapshotDir } = options;
strictAssert(
localBackupSnapshotDir,
'localBackupSnapshotDir is required for filePointer.localLocator'
);
if (mediaName) {
localBackupPath = getAttachmentLocalBackupPathFromSnapshotDir(
mediaName,
localBackupSnapshotDir
);
} else {
log.error(
'convertFilePointerToAttachment: localKey but no plaintextHash'
);
}
}
if (!locatorInfo) {
return {
...commonProps,
key: Bytes.toBase64(key),
digest: Bytes.isNotEmpty(digest) ? Bytes.toBase64(digest) : undefined,
size: size ?? 0,
cdnKey: transitCdnKey ?? undefined,
cdnNumber: transitCdnNumber ?? undefined,
uploadTimestamp: transitTierUploadTimestamp
? getTimestampFromLong(transitTierUploadTimestamp)
: undefined,
plaintextHash: Bytes.isNotEmpty(plaintextHash)
? Bytes.toHex(plaintextHash)
: undefined,
localBackupPath,
// TODO: DESKTOP-8883
localKey: Bytes.isNotEmpty(localKey)
? Bytes.toBase64(localKey)
: undefined,
...(mediaName && mediaTierCdnNumber != null
? {
backupCdnNumber: mediaTierCdnNumber,
}
: {}),
};
}
return {
...commonProps,
...getAttachmentLocatorInfoFromLegacyLocators(filePointer, options),
};
}
function getAttachmentLocatorInfoFromLegacyLocators(
filePointer: Backups.FilePointer,
options: Partial<ConvertFilePointerToAttachmentOptions>
) {
const {
attachmentLocator,
backupLocator,
localLocator,
invalidAttachmentLocator,
} = filePointer;
if (invalidAttachmentLocator) {
return {
error: true,
downloadPath: undefined,
};
}
if (attachmentLocator) {
const { cdnKey, cdnNumber, key, digest, uploadTimestamp, size } =
attachmentLocator;
const {
key,
localKey,
plaintextHash,
encryptedDigest,
size,
transitCdnKey,
transitCdnNumber,
transitTierUploadTimestamp,
mediaTierCdnNumber,
} = locatorInfo;
if (!Bytes.isNotEmpty(key)) {
return {
size: size ?? 0,
cdnKey: cdnKey ?? undefined,
cdnNumber: cdnNumber ?? undefined,
key: key?.length ? Bytes.toBase64(key) : undefined,
digest: digest?.length ? Bytes.toBase64(digest) : undefined,
uploadTimestamp: uploadTimestamp
? getTimestampFromLong(uploadTimestamp)
: undefined,
...commonProps,
error: true,
downloadPath: undefined,
};
}
// These are legacy locators so the mediaName would not be correct
if (backupLocator) {
const {
mediaName,
cdnNumber,
key,
digest,
size,
transitCdnKey,
transitCdnNumber,
} = backupLocator;
return {
cdnKey: transitCdnKey ?? undefined,
cdnNumber: transitCdnNumber ?? undefined,
key: key?.length ? Bytes.toBase64(key) : undefined,
digest: digest?.length ? Bytes.toBase64(digest) : undefined,
size: size ?? 0,
...(mediaName && cdnNumber != null
? {
backupCdnNumber: cdnNumber,
}
: {}),
};
let mediaName: string | undefined;
if (Bytes.isNotEmpty(plaintextHash) && Bytes.isNotEmpty(key)) {
mediaName =
getMediaName({
key,
plaintextHash,
}) ?? undefined;
}
if (localLocator) {
const {
mediaName,
localKey,
remoteKey: key,
remoteDigest: digest,
size,
transitCdnKey,
transitCdnNumber,
} = localLocator;
let localBackupPath: string | undefined;
if (Bytes.isNotEmpty(localKey)) {
const { localBackupSnapshotDir } = options;
strictAssert(
localBackupSnapshotDir,
'localBackupSnapshotDir is required for filePointer.localLocator'
);
if (mediaName == null) {
log.error(
'convertFilePointerToAttachment: filePointer.localLocator missing mediaName!'
if (mediaName) {
localBackupPath = getAttachmentLocalBackupPathFromSnapshotDir(
mediaName,
localBackupSnapshotDir
);
} else {
log.error(
'convertFilePointerToAttachment: localKey but no plaintextHash'
);
return {
error: true,
downloadPath: undefined,
};
}
const localBackupPath = getAttachmentLocalBackupPathFromSnapshotDir(
mediaName,
localBackupSnapshotDir
);
return {
cdnKey: transitCdnKey ?? undefined,
cdnNumber: transitCdnNumber ?? undefined,
key: key?.length ? Bytes.toBase64(key) : undefined,
digest: digest?.length ? Bytes.toBase64(digest) : undefined,
size: size ?? 0,
localBackupPath,
localKey: localKey?.length ? Bytes.toBase64(localKey) : undefined,
};
}
return {
error: true,
downloadPath: undefined,
...commonProps,
key: Bytes.toBase64(key),
digest: Bytes.isNotEmpty(encryptedDigest)
? Bytes.toBase64(encryptedDigest)
: undefined,
size: size ?? 0,
cdnKey: transitCdnKey ?? undefined,
cdnNumber: transitCdnNumber ?? undefined,
uploadTimestamp: transitTierUploadTimestamp
? getTimestampFromLong(transitTierUploadTimestamp)
: undefined,
plaintextHash: Bytes.isNotEmpty(plaintextHash)
? Bytes.toHex(plaintextHash)
: undefined,
localBackupPath,
// TODO: DESKTOP-8883
localKey: Bytes.isNotEmpty(localKey) ? Bytes.toBase64(localKey) : undefined,
...(mediaName && mediaTierCdnNumber != null
? {
backupCdnNumber: mediaTierCdnNumber,
}
: {}),
};
}

37
ts/test-electron/backup/attachments_test.ts
View File

@@ -103,6 +103,7 @@ describe('backup/attachments', () => {
size: 100,
contentType: IMAGE_JPEG,
path: `/path/to/file${index}.png`,
caption: `caption${index}`,
localKey: Bytes.toBase64(generateAttachmentKeys()),
uploadTimestamp: index,
thumbnail: {
@@ -401,6 +402,42 @@ describe('backup/attachments', () => {
{ backupLevel: BackupLevel.Paid }
);
});
it('deduplicates attachments on export based on mediaName', async () => {
const attachment1 = composeAttachment(1);
const attachment2 = {
...attachment1,
contentType: IMAGE_WEBP,
caption: 'attachment2caption',
cdnKey: 'attachment2cdnkey',
cdnNumber: 25,
};
await asymmetricRoundtripHarness(
[
composeMessage(1, {
attachments: [attachment1],
}),
composeMessage(2, {
attachments: [attachment2],
}),
],
[
composeMessage(1, {
attachments: [expectedRoundtrippedFields(attachment1)],
}),
composeMessage(2, {
attachments: [
expectedRoundtrippedFields({
...attachment2,
cdnKey: attachment1.cdnKey,
cdnNumber: attachment1.cdnNumber,
}),
],
}),
],
{ backupLevel: BackupLevel.Paid }
);
});
it('roundtrips voice message attachments', async () => {
const attachment = composeAttachment(1);
attachment.contentType = AUDIO_MP3;

204
ts/test-electron/backup/filePointer_test.ts
View File

@@ -13,7 +13,7 @@ import {
getFilePointerForAttachment,
convertFilePointerToAttachment,
} from '../../services/backups/util/filePointers';
import { APPLICATION_OCTET_STREAM, IMAGE_PNG } from '../../types/MIME';
import { IMAGE_PNG } from '../../types/MIME';
import * as Bytes from '../../Bytes';
import { type AttachmentType } from '../../types/Attachment';
import { MASTER_KEY, MEDIA_ROOT_KEY } from './helpers';
@@ -44,109 +44,6 @@ describe('convertFilePointerToAttachment', () => {
chunkSize: 1000,
} as const;
const key = generateKeys();
const digest = randomBytes(32);
describe('legacy locators', () => {
it('processes filepointer with attachmentLocator', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
...commonFilePointerProps,
attachmentLocator: new Backups.FilePointer.AttachmentLocator({
size: 128,
cdnKey: 'cdnKey',
cdnNumber: 2,
key,
digest,
uploadTimestamp: Long.fromNumber(1970),
}),
}),
{ _createName: () => 'downloadPath' }
);
assert.deepStrictEqual(result, {
...commonAttachmentProps,
size: 128,
cdnKey: 'cdnKey',
cdnNumber: 2,
key: Bytes.toBase64(key),
digest: Bytes.toBase64(digest),
uploadTimestamp: 1970,
downloadPath: 'downloadPath',
});
});
it('processes filepointer with backupLocator and missing fields', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
...commonFilePointerProps,
backupLocator: new Backups.FilePointer.BackupLocator({
mediaName: 'mediaName',
cdnNumber: 3,
size: 128,
key,
digest,
transitCdnKey: 'transitCdnKey',
transitCdnNumber: 2,
}),
}),
{ _createName: () => 'downloadPath' }
);
assert.deepStrictEqual(result, {
...commonAttachmentProps,
size: 128,
cdnKey: 'transitCdnKey',
cdnNumber: 2,
key: Bytes.toBase64(key),
digest: Bytes.toBase64(digest),
backupCdnNumber: 3,
downloadPath: 'downloadPath',
});
});
it('processes filepointer with invalidAttachmentLocator', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
...commonFilePointerProps,
invalidAttachmentLocator:
new Backups.FilePointer.InvalidAttachmentLocator(),
})
);
assert.deepStrictEqual(result, {
...commonAttachmentProps,
size: 0,
error: true,
downloadPath: undefined,
});
});
it('accepts missing / null fields and adds defaults to contentType and size', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
backupLocator: new Backups.FilePointer.BackupLocator(),
}),
{ _createName: () => 'downloadPath' }
);
assert.deepStrictEqual(result, {
contentType: APPLICATION_OCTET_STREAM,
size: 0,
downloadPath: 'downloadPath',
width: undefined,
height: undefined,
blurHash: undefined,
fileName: undefined,
caption: undefined,
cdnKey: undefined,
cdnNumber: undefined,
key: undefined,
digest: undefined,
incrementalMac: undefined,
chunkSize: undefined,
});
});
});
describe('locatorInfo', () => {
it('processes filepointer with empty locatorInfo', () => {
const result = convertFilePointerToAttachment(
@@ -164,103 +61,20 @@ describe('convertFilePointerToAttachment', () => {
downloadPath: undefined,
});
});
describe('legacyDigest/legacyMediaName', () => {
it('processes locatorInfo with transit only info & legacy digest', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
...commonFilePointerProps,
locatorInfo: {
transitCdnKey: 'cdnKey',
transitCdnNumber: 42,
size: 128,
transitTierUploadTimestamp: Long.fromNumber(12345),
key: Bytes.fromString('key'),
legacyDigest: Bytes.fromString('legacyDigest'),
},
}),
{ _createName: () => 'downloadPath' }
);
assert.deepStrictEqual(result, {
...commonAttachmentProps,
size: 128,
cdnKey: 'cdnKey',
cdnNumber: 42,
downloadPath: 'downloadPath',
key: Bytes.toBase64(Bytes.fromString('key')),
digest: Bytes.toBase64(Bytes.fromString('legacyDigest')),
uploadTimestamp: 12345,
plaintextHash: undefined,
localBackupPath: undefined,
localKey: undefined,
});
});
it('processes locatorInfo with legacy digest and legacyMediaName', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
...commonFilePointerProps,
locatorInfo: {
transitCdnKey: 'cdnKey',
transitCdnNumber: 42,
size: 128,
transitTierUploadTimestamp: Long.fromNumber(12345),
key: Bytes.fromString('key'),
legacyDigest: Bytes.fromString('legacyDigest'),
legacyMediaName: 'legacyMediaName',
mediaTierCdnNumber: 43,
},
}),
{ _createName: () => 'downloadPath' }
);
assert.deepStrictEqual(result, {
...commonAttachmentProps,
size: 128,
cdnKey: 'cdnKey',
cdnNumber: 42,
downloadPath: 'downloadPath',
key: Bytes.toBase64(Bytes.fromString('key')),
digest: Bytes.toBase64(Bytes.fromString('legacyDigest')),
uploadTimestamp: 12345,
backupCdnNumber: 43,
plaintextHash: undefined,
localBackupPath: undefined,
localKey: undefined,
});
});
});
it('processes locatorInfo with new and legacy digests and prefers new one', () => {
it('processes filepointer with missing locatorInfo', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
...commonFilePointerProps,
locatorInfo: {
transitCdnKey: 'cdnKey',
transitCdnNumber: 42,
size: 128,
transitTierUploadTimestamp: Long.fromNumber(12345),
key: Bytes.fromString('key'),
legacyDigest: Bytes.fromString('legacyDigest'),
encryptedDigest: Bytes.fromString('encryptedDigest'),
},
}),
new Backups.FilePointer(commonFilePointerProps),
{ _createName: () => 'downloadPath' }
);
assert.deepStrictEqual(result, {
...commonAttachmentProps,
size: 128,
cdnKey: 'cdnKey',
cdnNumber: 42,
downloadPath: 'downloadPath',
key: Bytes.toBase64(Bytes.fromString('key')),
digest: Bytes.toBase64(Bytes.fromString('encryptedDigest')),
uploadTimestamp: 12345,
plaintextHash: undefined,
localBackupPath: undefined,
localKey: undefined,
size: 0,
error: true,
downloadPath: undefined,
});
});
it('processes locatorInfo with plaintextHash', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
@@ -272,8 +86,6 @@ describe('convertFilePointerToAttachment', () => {
transitTierUploadTimestamp: Long.fromNumber(12345),
key: Bytes.fromString('key'),
plaintextHash: Bytes.fromString('plaintextHash'),
legacyDigest: Bytes.fromString('legacyDigest'),
legacyMediaName: 'legacyMediaName',
mediaTierCdnNumber: 43,
},
}),
@@ -286,8 +98,8 @@ describe('convertFilePointerToAttachment', () => {
cdnKey: 'cdnKey',
cdnNumber: 42,
downloadPath: 'downloadPath',
digest: undefined,
key: Bytes.toBase64(Bytes.fromString('key')),
digest: Bytes.toBase64(Bytes.fromString('legacyDigest')),
uploadTimestamp: 12345,
backupCdnNumber: 43,
plaintextHash: Bytes.toHex(Bytes.fromString('plaintextHash')),

2
ts/textsecure/Provisioner.ts
View File

@@ -417,7 +417,7 @@ export class Provisioner {
.toAppUrl({
uuid,
pubKey: Bytes.toBase64(cipher.getPublicKey().serialize()),
capabilities: isLinkAndSyncEnabled() ? ['backup3', 'backup4'] : [],
capabilities: isLinkAndSyncEnabled() ? ['backup4'] : [],
})
.toString();