mirror of
https://github.com/signalapp/Signal-Desktop.git
synced 2026-10-02 09:50:04 +01:00
Use libsignal API for backup chunking
This commit is contained in:
Fedor Indutny
+63
-8
@@ -11535,7 +11535,7 @@ Signal Desktop makes use of the following open source projects.
|
||||
|
||||
libsignal makes use of the following open source projects.
|
||||
|
||||
## spqr 1.5.3, partial-default-derive 0.1.0, partial-default 0.1.0
|
||||
## spqr 1.6.0, partial-default-derive 0.1.0, partial-default 0.1.0
|
||||
|
||||
```
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
@@ -14717,7 +14717,7 @@ THIS SOFTWARE.
|
||||
|
||||
```
|
||||
|
||||
## rustls-webpki 0.103.13
|
||||
## rustls-webpki 0.103.15
|
||||
|
||||
```
|
||||
Except as otherwise noted, this project is licensed under the following
|
||||
@@ -15712,7 +15712,7 @@ The above copyright notice and this permission notice shall be included in all c
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
```
|
||||
|
||||
## rustls-native-certs 0.8.4, rustls 0.23.41
|
||||
## rustls-native-certs 0.8.4, rustls 0.23.45
|
||||
|
||||
```
|
||||
Copyright (c) 2016 Joseph Birr-Pixton <jpixton@gmail.com>
|
||||
@@ -17286,7 +17286,7 @@ DEALINGS IN THE SOFTWARE.
|
||||
|
||||
```
|
||||
|
||||
## downcast-rs 1.2.1
|
||||
## downcast-rs 2.0.2
|
||||
|
||||
```
|
||||
Copyright (c) 2020 Ashish Myles and contributors
|
||||
@@ -17591,7 +17591,7 @@ SOFTWARE.
|
||||
|
||||
```
|
||||
|
||||
## mediasan-common 0.5.3, mp4san-derive 0.5.3, mp4san 0.5.3, webpsan 0.5.3
|
||||
## mediasan-common 0.5.4, mp4san-derive 0.5.4, mp4san 0.5.4, webpsan 0.5.4
|
||||
|
||||
```
|
||||
Copyright (c) 2022 jessa0
|
||||
@@ -18009,6 +18009,37 @@ DEALINGS IN THE SOFTWARE.
|
||||
|
||||
```
|
||||
|
||||
## rand_distr 0.5.1
|
||||
|
||||
```
|
||||
Copyright 2018 Developers of the Rand project
|
||||
|
||||
Permission is hereby granted, free of charge, to any
|
||||
person obtaining a copy of this software and associated
|
||||
documentation files (the "Software"), to deal in the
|
||||
Software without restriction, including without
|
||||
limitation the rights to use, copy, modify, merge,
|
||||
publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software
|
||||
is furnished to do so, subject to the following
|
||||
conditions:
|
||||
|
||||
The above copyright notice and this permission notice
|
||||
shall be included in all copies or substantial portions
|
||||
of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
|
||||
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
|
||||
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
||||
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
||||
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
|
||||
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
```
|
||||
|
||||
## rand 0.10.2, rand 0.9.4, rand_chacha 0.9.0, rand_core 0.6.4, rand_core 0.9.5
|
||||
|
||||
```
|
||||
@@ -18442,7 +18473,7 @@ SOFTWARE.
|
||||
|
||||
```
|
||||
|
||||
## convert_case 0.10.0, convert_case 0.4.0
|
||||
## convert_case 0.10.0
|
||||
|
||||
```
|
||||
MIT License
|
||||
@@ -18631,6 +18662,30 @@ SOFTWARE.
|
||||
|
||||
```
|
||||
|
||||
## libm 0.2.16
|
||||
|
||||
```
|
||||
MIT License
|
||||
|
||||
Copyright (c) <year> <copyright holders>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial
|
||||
portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
|
||||
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
|
||||
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
```
|
||||
|
||||
## tokio-stream 0.1.18, tokio-util 0.7.18, tokio 1.52.3
|
||||
|
||||
```
|
||||
@@ -18769,7 +18824,7 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
```
|
||||
|
||||
## adler2 2.0.1, anyhow 1.0.103, async-trait 0.1.89, atomic-waker 1.1.2, auto_enums 0.8.9, curve25519-dalek-derive 0.1.1, derive_utils 0.15.1, displaydoc 0.2.6, dyn-clone 1.0.20, fastrand 2.4.1, home 0.5.12, itoa 1.0.18, linkme-impl 0.3.36, linkme 0.3.36, linux-raw-sys 0.12.1, linux-raw-sys 0.4.15, minimal-lexical 0.2.1, num_enum 0.7.6, num_enum_derive 0.7.6, once_cell 1.21.4, paste 1.0.15, pin-project-internal 1.1.13, pin-project-lite 0.2.17, pin-project 1.1.13, prettyplease 0.2.37, proc-macro-crate 3.5.0, proc-macro2 1.0.106, quote 1.0.46, ref-cast-impl 1.0.25, ref-cast 1.0.25, rustix 0.38.44, rustix 1.1.4, rustversion 1.0.23, semver 1.0.28, send_wrapper 0.6.0, serde 1.0.228, serde_core 1.0.228, serde_derive 1.0.228, serde_json 1.0.150, syn-mid 0.6.1, syn 1.0.109, syn 2.0.118, syn 3.0.3, thiserror-impl 1.0.69, thiserror-impl 2.0.18, thiserror 1.0.69, thiserror 2.0.18, unicode-ident 1.0.24, zmij 1.0.21
|
||||
## adler2 2.0.1, anyhow 1.0.103, async-trait 0.1.89, atomic-waker 1.1.2, auto_enums 0.8.9, curve25519-dalek-derive 0.1.1, derive_utils 0.15.1, displaydoc 0.2.6, dyn-clone 1.0.20, fastrand 2.4.1, home 0.5.12, itoa 1.0.18, linkme-impl 0.3.36, linkme 0.3.36, linux-raw-sys 0.12.1, linux-raw-sys 0.4.15, minimal-lexical 0.2.1, num_enum 0.7.6, num_enum_derive 0.7.6, once_cell 1.21.4, paste 1.0.15, pin-project-internal 1.1.13, pin-project-lite 0.2.17, pin-project 1.1.13, prettyplease 0.2.37, proc-macro-crate 3.5.0, proc-macro2 1.0.106, quote 1.0.46, ref-cast-impl 1.0.25, ref-cast 1.0.25, rustix 0.38.44, rustix 1.1.4, rustversion 1.0.23, semver 1.0.28, send_wrapper 0.6.0, serde 1.0.228, serde_core 1.0.228, serde_derive 1.0.228, serde_json 1.0.150, syn-mid 0.6.1, syn 2.0.118, syn 3.0.3, thiserror-impl 1.0.69, thiserror-impl 2.0.18, thiserror 1.0.69, thiserror 2.0.18, unicode-ident 1.0.24, zmij 1.0.21
|
||||
|
||||
```
|
||||
Permission is hereby granted, free of charge, to any
|
||||
@@ -19039,7 +19094,7 @@ SOFTWARE.
|
||||
|
||||
```
|
||||
|
||||
## derive_more-impl 2.1.1, derive_more 0.99.20, derive_more 2.1.1
|
||||
## derive_more-impl 2.1.1, derive_more 2.1.1
|
||||
|
||||
```
|
||||
The MIT License (MIT)
|
||||
|
||||
+1
-1
@@ -123,7 +123,7 @@
|
||||
"dependencies": {
|
||||
"@indutny/mac-screen-share": "1.2.5",
|
||||
"@indutny/simple-windows-notifications": "2.0.21",
|
||||
"@signalapp/libsignal-client": "0.101.2",
|
||||
"@signalapp/libsignal-client": "0.103.0",
|
||||
"@signalapp/mute-state-change": "workspace:*",
|
||||
"@signalapp/ringrtc": "2.72.0",
|
||||
"@signalapp/sqlcipher": "4.1.0",
|
||||
|
||||
@@ -120,6 +120,26 @@ type GrpcResponse<Endpoint extends keyof typeof $services> = Parameters<
|
||||
(typeof $services)[Endpoint]['Response']['encode']
|
||||
>[0];
|
||||
|
||||
function convertSendMessageType(
|
||||
type: org.signal.chat.messages.SendMessageType,
|
||||
): Proto.Envelope.Type {
|
||||
const { SendMessageType } = org.signal.chat.messages;
|
||||
switch (type) {
|
||||
case SendMessageType.UNSPECIFIED:
|
||||
return Proto.Envelope.Type.UNKNOWN;
|
||||
case SendMessageType.DOUBLE_RATCHET:
|
||||
return Proto.Envelope.Type.DOUBLE_RATCHET;
|
||||
case SendMessageType.PREKEY_MESSAGE:
|
||||
return Proto.Envelope.Type.PREKEY_MESSAGE;
|
||||
case SendMessageType.PLAINTEXT_CONTENT:
|
||||
return Proto.Envelope.Type.PLAINTEXT_CONTENT;
|
||||
case SendMessageType.UNIDENTIFIED_SENDER:
|
||||
return Proto.Envelope.Type.UNIDENTIFIED_SENDER;
|
||||
default:
|
||||
throw new Error(`Unsupported SendMessageType: ${type}`);
|
||||
}
|
||||
}
|
||||
|
||||
export const createHandler = (server: Server): RequestHandler => {
|
||||
function grpcRoute<Endpoint extends keyof typeof $services>(
|
||||
endpoint: Endpoint,
|
||||
@@ -329,6 +349,60 @@ export const createHandler = (server: Server): RequestHandler => {
|
||||
onMultiRecipientMessage,
|
||||
);
|
||||
|
||||
const onSendSyncMessage = authenticatedGrpcRoute(
|
||||
'org.signal.chat.messages.Messages/SendSyncMessage',
|
||||
async ({ messages: bundle }, device) => {
|
||||
if (!bundle) {
|
||||
throw new Error('Missing bundle');
|
||||
}
|
||||
const { timestamp, messages } = bundle;
|
||||
|
||||
const prepared = await server.prepareMultiDeviceMessage(
|
||||
device,
|
||||
device.aci, // Sending to ourselves
|
||||
Array.from(messages.entries()).map(([deviceId, message]) => {
|
||||
if (!message) {
|
||||
throw new Error('Missing message');
|
||||
}
|
||||
|
||||
return {
|
||||
type: convertSendMessageType(message.type),
|
||||
destinationDeviceId: deviceId as DeviceId,
|
||||
destinationRegistrationId: message.registrationId as RegistrationId,
|
||||
// TODO(indutny): pass buffer
|
||||
content: Buffer.from(message.payload).toString('base64'),
|
||||
};
|
||||
}),
|
||||
timestamp,
|
||||
);
|
||||
|
||||
switch (prepared.status) {
|
||||
case 'ok':
|
||||
await server.handlePreparedMultiDeviceMessage(
|
||||
device,
|
||||
prepared.targetServiceId,
|
||||
prepared.result,
|
||||
);
|
||||
return {
|
||||
response: {
|
||||
success: {},
|
||||
},
|
||||
};
|
||||
case 'unknown':
|
||||
return {
|
||||
response: {
|
||||
destinationNotFound: {},
|
||||
},
|
||||
};
|
||||
case 'incomplete':
|
||||
throw new Error('TODO(indutny)');
|
||||
case 'stale':
|
||||
throw new Error('TODO(indutny)');
|
||||
}
|
||||
throw new Error('TODO(indutny)');
|
||||
},
|
||||
);
|
||||
|
||||
const onLookupUsernameHash = grpcRoute(
|
||||
'org.signal.chat.account.AccountsAnonymous/LookupUsernameHash',
|
||||
async ({ usernameHash }) => {
|
||||
@@ -747,6 +821,7 @@ export const createHandler = (server: Server): RequestHandler => {
|
||||
// gRPC
|
||||
onSendMultiRecipientMessage,
|
||||
onSendMultiRecipientStory,
|
||||
onSendSyncMessage,
|
||||
onLookupUsernameHash,
|
||||
onLookupUsernameLink,
|
||||
onGetUploadForm,
|
||||
|
||||
Generated
+5
-5
@@ -256,8 +256,8 @@ importers:
|
||||
specifier: 2.0.21
|
||||
version: 2.0.21
|
||||
'@signalapp/libsignal-client':
|
||||
specifier: 0.101.2
|
||||
version: 0.101.2
|
||||
specifier: 0.103.0
|
||||
version: 0.103.0
|
||||
'@signalapp/mute-state-change':
|
||||
specifier: workspace:*
|
||||
version: link:packages/mute-state-change
|
||||
@@ -4741,8 +4741,8 @@ packages:
|
||||
'@signalapp/libsignal-client@0.100.0':
|
||||
resolution: {integrity: sha512-pS67moiIxs3h0kcO81Y9FrGbquh5xo2jyG9RG/+YyDEL6Bcz33IR5fCTdfpnYJR4pd3CoutJNbyUnieMGCpI4Q==, tarball: https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.100.0.tgz}
|
||||
|
||||
'@signalapp/libsignal-client@0.101.2':
|
||||
resolution: {integrity: sha512-DZ6HnuMD6R3E4nQpa7ly7kDxRL0z9QcVR9l1m25ofUwrICYug0KPRM8ucRNar8b65nrhxNJAk9OeF20SwYVvGg==, tarball: https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.101.2.tgz}
|
||||
'@signalapp/libsignal-client@0.103.0':
|
||||
resolution: {integrity: sha512-eTPT0ebDsoriIjeMjuB882H51UpFXnWDIu43VX0uBSM6iryKmCa0zxabncrS6xG0KJqAc+Modjkhk0nk+w2k+A==, tarball: https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.103.0.tgz}
|
||||
|
||||
'@signalapp/minimask@1.0.1':
|
||||
resolution: {integrity: sha512-QAwo0joA60urTNbW9RIz6vLKQjy+jdVtH7cvY0wD9PVooD46MAjE40MLssp4xUJrph91n2XvtJ3pbEUDrmT2AA==, tarball: https://registry.npmjs.org/@signalapp/minimask/-/minimask-1.0.1.tgz}
|
||||
@@ -15734,7 +15734,7 @@ snapshots:
|
||||
node-gyp-build: 4.8.4
|
||||
type-fest: 4.26.1
|
||||
|
||||
'@signalapp/libsignal-client@0.101.2':
|
||||
'@signalapp/libsignal-client@0.103.0':
|
||||
dependencies:
|
||||
node-gyp-build: 4.8.4
|
||||
type-fest: 4.26.1
|
||||
|
||||
+1
-1
@@ -94,7 +94,7 @@ allowBuilds:
|
||||
'@indutny/simple-windows-notifications@2.0.16': true
|
||||
'@parcel/watcher@2.5.6': true
|
||||
'@signalapp/libsignal-client@0.100.0': true # duplicate for @signalapp/mock-server
|
||||
'@signalapp/libsignal-client@0.101.2': true
|
||||
'@signalapp/libsignal-client@0.103.0': true
|
||||
'@signalapp/ringrtc@2.72.0': true
|
||||
'@signalapp/sqlcipher@4.1.0': true
|
||||
'@signalapp/windows-ucv@1.0.1': true
|
||||
|
||||
@@ -114,6 +114,8 @@ const KnownDesktopLibsignalNetKeys = [
|
||||
'desktop.libsignalNet.grpc.AttachmentsGetUploadForm.beta',
|
||||
'desktop.libsignalNet.grpc.BackupsAnonymousGetUploadForm',
|
||||
'desktop.libsignalNet.grpc.BackupsAnonymousGetUploadForm.beta',
|
||||
'desktop.libsignalNet.grpc.KeyTransparencyQueryServiceSearchV2',
|
||||
'desktop.libsignalNet.grpc.KeyTransparencyQueryServiceSearchV2.beta',
|
||||
'desktop.libsignalNet.grpc.MessagesAnonymousSendMultiRecipientMessage.2',
|
||||
'desktop.libsignalNet.grpc.MessagesAnonymousSendMultiRecipientMessage.2.beta',
|
||||
'desktop.libsignalNet.grpc.MessagesAnonymousSendSingleRecipientMessage',
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
// Copyright 2026 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import { createGzip } from 'node:zlib';
|
||||
import { Readable } from 'node:stream';
|
||||
import { Buffer } from 'node:buffer';
|
||||
import {
|
||||
BackupJsonExporter,
|
||||
flushInterval,
|
||||
paddingSize,
|
||||
} from '@signalapp/libsignal-client/dist/MessageBackup.js';
|
||||
|
||||
import { Backups } from '../../protobuf/index.std.ts';
|
||||
import { encodeDelimited } from '../../util/encodeDelimited.std.ts';
|
||||
import { drop } from '../../util/drop.std.ts';
|
||||
import { wrapEventEmitterOnce } from '../../util/wrapEventEmitterOnce.node.ts';
|
||||
import type { LoggerType } from '../../types/Logging.std.ts';
|
||||
import { toLogFormat } from '../../types/errors.std.ts';
|
||||
|
||||
type RunResultType = Readonly<{
|
||||
info: Backups.BackupInfo.Params;
|
||||
iterable: AsyncIterable<NonNullable<Backups.Frame.Params['item']>>;
|
||||
}>;
|
||||
|
||||
// Number of frames to keep buffered
|
||||
const BUFFER_SIZE = 10000;
|
||||
|
||||
function buffered(
|
||||
iterable: RunResultType['iterable']
|
||||
): RunResultType['iterable'] {
|
||||
return Readable.from(iterable, {
|
||||
objectMode: true,
|
||||
highWaterMark: BUFFER_SIZE,
|
||||
})[Symbol.asyncIterator]();
|
||||
}
|
||||
|
||||
export async function* toDelimitedIterable({
|
||||
info,
|
||||
iterable,
|
||||
}: RunResultType): AsyncIterable<Uint8Array<ArrayBuffer>> {
|
||||
yield* encodeDelimited(Backups.BackupInfo.encode(info));
|
||||
|
||||
for await (const item of buffered(iterable)) {
|
||||
yield* encodeDelimited(Backups.Frame.encode({ item }));
|
||||
}
|
||||
}
|
||||
|
||||
function* generateFlushOffsets(): Iterator<bigint> {
|
||||
let offset = 0n;
|
||||
|
||||
// oxlint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
const interval = flushInterval(offset);
|
||||
offset += interval;
|
||||
yield offset;
|
||||
}
|
||||
}
|
||||
|
||||
export async function* toPaddedGzipIterable({
|
||||
info,
|
||||
iterable,
|
||||
}: RunResultType): AsyncIterable<Uint8Array<ArrayBuffer>> {
|
||||
// Track chat item blocks
|
||||
let largestBlock = 0n;
|
||||
let currentBlock = 0n;
|
||||
function onNewBlock() {
|
||||
if (currentBlock > largestBlock) {
|
||||
largestBlock = currentBlock;
|
||||
}
|
||||
currentBlock = 0n;
|
||||
}
|
||||
|
||||
async function* delimited(): AsyncIterable<Uint8Array<ArrayBuffer> | null> {
|
||||
yield* encodeDelimited(Backups.BackupInfo.encode(info));
|
||||
|
||||
// Flush after info frame
|
||||
yield null;
|
||||
|
||||
// ChatItem bytes to flush at
|
||||
const flushOffsets = generateFlushOffsets();
|
||||
let flushOffset: bigint = flushOffsets.next().value;
|
||||
|
||||
let chatItemsBytes = 0n;
|
||||
|
||||
for await (const item of buffered(iterable)) {
|
||||
let flushBeforeAfter = false;
|
||||
|
||||
// Flush before/after "account info"/recipients
|
||||
if (item.account != null || item.recipient != null) {
|
||||
flushBeforeAfter = true;
|
||||
}
|
||||
|
||||
if (chatItemsBytes === 0n && item.chatItem != null) {
|
||||
// Flush *before* starting export of chat items
|
||||
yield null;
|
||||
} else if (chatItemsBytes !== 0n && item.chatItem == null) {
|
||||
// We got a non-chat item frame in between chat item frames,
|
||||
// flush before/after it and reset the block size.
|
||||
flushBeforeAfter = true;
|
||||
onNewBlock();
|
||||
}
|
||||
|
||||
if (flushBeforeAfter) {
|
||||
yield null;
|
||||
}
|
||||
|
||||
if (item.chatItem == null) {
|
||||
yield* encodeDelimited(Backups.Frame.encode({ item }));
|
||||
} else {
|
||||
let frame = Buffer.concat(
|
||||
encodeDelimited(Backups.Frame.encode({ item }))
|
||||
);
|
||||
|
||||
// Split chat item frame into as many parts as need to cut at right
|
||||
// flush intervals
|
||||
while (frame.byteLength !== 0) {
|
||||
const cutOffset = flushOffset - chatItemsBytes;
|
||||
|
||||
const slice = frame.subarray(0, Number(cutOffset));
|
||||
yield slice;
|
||||
chatItemsBytes += BigInt(slice.byteLength);
|
||||
currentBlock += BigInt(slice.byteLength);
|
||||
|
||||
// We are finished
|
||||
if (frame.byteLength === slice.byteLength) {
|
||||
break;
|
||||
}
|
||||
|
||||
// The frame was cut - flush, get the next offset, reset block size
|
||||
frame = frame.subarray(slice.byteLength);
|
||||
flushOffset = flushOffsets.next().value;
|
||||
onNewBlock();
|
||||
yield null;
|
||||
}
|
||||
}
|
||||
|
||||
if (flushBeforeAfter) {
|
||||
yield null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const gzip = createGzip();
|
||||
|
||||
async function feedGzip() {
|
||||
try {
|
||||
// flushInterval() is capped at 512KiB so we can write all buffers at once
|
||||
// to save time on calling into C++
|
||||
let buffer = new Array<Uint8Array<ArrayBuffer>>();
|
||||
for await (const chunk of delimited()) {
|
||||
if (chunk != null) {
|
||||
buffer.push(chunk);
|
||||
continue;
|
||||
}
|
||||
|
||||
const written = gzip.write(Buffer.concat(buffer));
|
||||
buffer = [];
|
||||
gzip.flush();
|
||||
if (!written) {
|
||||
await wrapEventEmitterOnce(gzip, 'drain');
|
||||
}
|
||||
}
|
||||
gzip.end(Buffer.concat(buffer));
|
||||
} catch (error) {
|
||||
gzip.destroy(error);
|
||||
}
|
||||
}
|
||||
|
||||
drop(feedGzip());
|
||||
|
||||
let compressedLength = 0n;
|
||||
for await (const chunk of gzip) {
|
||||
compressedLength += BigInt(chunk.byteLength);
|
||||
yield chunk;
|
||||
}
|
||||
|
||||
const padding = paddingSize(largestBlock, compressedLength);
|
||||
const PADDING_CHUNK_SIZE = 64 * 1024;
|
||||
const empty = Buffer.alloc(PADDING_CHUNK_SIZE);
|
||||
for (let i = 0n; i < padding; i += BigInt(empty.byteLength)) {
|
||||
const remaining = padding - i;
|
||||
if (remaining >= BigInt(empty.byteLength)) {
|
||||
yield empty;
|
||||
} else {
|
||||
yield empty.subarray(0, Number(remaining));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function* toJSONIterable(
|
||||
{ info, iterable }: RunResultType,
|
||||
log?: LoggerType
|
||||
): AsyncIterable<string> {
|
||||
const { exporter, chunk: initialChunk } = BackupJsonExporter.start(
|
||||
Backups.BackupInfo.encode(info),
|
||||
{ validate: false }
|
||||
);
|
||||
|
||||
yield `${initialChunk}\n`;
|
||||
|
||||
for await (const item of buffered(iterable)) {
|
||||
const results = exporter.exportFrames(
|
||||
Buffer.concat(encodeDelimited(Backups.Frame.encode({ item })))
|
||||
);
|
||||
|
||||
for (const result of results) {
|
||||
if (result.errorMessage) {
|
||||
log?.warn(
|
||||
'frameToJson: frame had a validation error:',
|
||||
result.errorMessage
|
||||
);
|
||||
}
|
||||
if (!result.line) {
|
||||
log?.error('frameToJson: frame was filtered out by libsignal');
|
||||
} else {
|
||||
yield `${result.line}\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const result = exporter.finish();
|
||||
if (result?.errorMessage) {
|
||||
log?.warn(
|
||||
'jsonExporter.finish() returned validation error:',
|
||||
result.errorMessage
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
// We only warn because this isn't that big of a deal - the export is complete.
|
||||
// All we need from the exporter at the end is any validation errors it found.
|
||||
log?.warn('jsonExporter returned error', toLogFormat(error));
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,9 @@
|
||||
// Copyright 2023 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client';
|
||||
import { BackupJsonExporter } from '@signalapp/libsignal-client/dist/MessageBackup.js';
|
||||
import { pMapIterable } from 'p-map';
|
||||
import pTimeout from 'p-timeout';
|
||||
import { Readable } from 'node:stream';
|
||||
import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client';
|
||||
import { pMapIterable } from 'p-map';
|
||||
import lodash from 'lodash';
|
||||
import { CallLinkRootKey } from '@signalapp/ringrtc';
|
||||
import { MuteExpiration } from '@signalapp/types';
|
||||
@@ -44,9 +42,7 @@ import type {
|
||||
MessageAttributesType,
|
||||
QuotedAttachmentType,
|
||||
} from '../../model-types.d.ts';
|
||||
import { drop } from '../../util/drop.std.ts';
|
||||
import { isNotNil } from '../../util/isNotNil.std.ts';
|
||||
import { explodePromise } from '../../util/explodePromise.std.ts';
|
||||
import {
|
||||
isDirectConversation,
|
||||
isGroup,
|
||||
@@ -57,12 +53,7 @@ import {
|
||||
import { uuidToBytes } from '../../util/uuidToBytes.std.ts';
|
||||
import { strictAssert } from '../../util/assert.std.ts';
|
||||
import { getSafeLongFromTimestamp } from '../../util/timestampLongUtils.std.ts';
|
||||
import {
|
||||
DAY,
|
||||
MINUTE,
|
||||
SECOND,
|
||||
DurationInSeconds,
|
||||
} from '../../util/durations/index.std.ts';
|
||||
import { DAY, DurationInSeconds } from '../../util/durations/index.std.ts';
|
||||
import {
|
||||
PhoneNumberDiscoverability,
|
||||
parsePhoneNumberDiscoverability,
|
||||
@@ -189,7 +180,6 @@ import { ChatFolderType } from '../../types/ChatFolder.std.ts';
|
||||
import { expiresTooSoonForBackup } from './util/expiration.std.ts';
|
||||
import type { PinnedMessage } from '../../types/PinnedMessage.std.ts';
|
||||
import type { ThemeType } from '../../util/preload.preload.ts';
|
||||
import { encodeDelimited } from '../../util/encodeDelimited.std.ts';
|
||||
import { safeParseStrict } from '../../util/schemas.std.ts';
|
||||
import type { WithRequiredProperties } from '../../types/Util.std.ts';
|
||||
import type { Emoji } from '../../axo/emoji.std.ts';
|
||||
@@ -201,13 +191,6 @@ const log = createLogger('backupExport');
|
||||
// We only run 4 sql workers so going much higher doesn't help
|
||||
const MAX_CONCURRENCY = 8;
|
||||
|
||||
// We want a very generous timeout to make sure that we always resume write
|
||||
// access to the database.
|
||||
const FLUSH_TIMEOUT = 30 * MINUTE;
|
||||
|
||||
// Threshold for reporting slow flushes
|
||||
const REPORTING_THRESHOLD = SECOND;
|
||||
|
||||
const MAX_BACKUP_MESSAGE_BODY_BYTE_LENGTH = 128 * KIBIBYTE;
|
||||
const BACKUP_QUOTE_BODY_LIMIT = 2048;
|
||||
|
||||
@@ -268,12 +251,13 @@ type NonBubbleResultType = Readonly<
|
||||
}
|
||||
>;
|
||||
|
||||
type Options = Readonly<BackupExportOptions> & {
|
||||
validationRun?: boolean;
|
||||
};
|
||||
type RunResultType = Readonly<{
|
||||
info: Backups.BackupInfo.Params;
|
||||
iterable: AsyncIterable<NonNullable<Backups.Frame.Params['item']>>;
|
||||
}>;
|
||||
|
||||
export class BackupExportStream extends Readable {
|
||||
readonly #options: Options;
|
||||
export class BackupExportStream {
|
||||
readonly #options: BackupExportOptions;
|
||||
// Shared between all methods for consistency.
|
||||
readonly #now = Date.now();
|
||||
|
||||
@@ -303,92 +287,93 @@ export class BackupExportStream extends Readable {
|
||||
stickerPacks: 0,
|
||||
unknownConversationReferences: new Map<string, number>(),
|
||||
};
|
||||
#pendingRecipients: Array<Backups.Recipient.Params> = [];
|
||||
#ourConversation?: ConversationAttributesType;
|
||||
#attachmentBackupJobs: Array<
|
||||
CoreAttachmentBackupJobType | CoreAttachmentLocalBackupJobType
|
||||
> = [];
|
||||
#buffers = new Array<Uint8Array<ArrayBuffer>>();
|
||||
#nextRecipientId = 1n;
|
||||
#flushResolve: (() => void) | undefined;
|
||||
#jsonExporter: BackupJsonExporter | undefined;
|
||||
|
||||
// Map from custom color uuid to an index in accountSettings.customColors
|
||||
// array.
|
||||
readonly #customColorIdByUuid = new Map<string, bigint>();
|
||||
|
||||
constructor(options: Options) {
|
||||
super();
|
||||
constructor(options: BackupExportOptions) {
|
||||
this.#options = options;
|
||||
}
|
||||
|
||||
async #cleanupAfterError() {
|
||||
log.warn('Cleaning up after error...');
|
||||
await resumeWriteAccess();
|
||||
public run(): RunResultType {
|
||||
const info: Backups.BackupInfo.Params = {
|
||||
version: BACKUP_VERSION,
|
||||
backupTimeMs: this.#backupTimeMs,
|
||||
mediaRootBackupKey: getBackupMediaRootKey().serialize(),
|
||||
firstAppVersion: itemStorage.get('restoredBackupFirstAppVersion') ?? null,
|
||||
currentAppVersion: `Desktop ${window.getVersion()}`,
|
||||
debugInfo: null,
|
||||
};
|
||||
|
||||
const iterable = this.#run();
|
||||
|
||||
return {
|
||||
info,
|
||||
iterable,
|
||||
};
|
||||
}
|
||||
|
||||
override _destroy(
|
||||
error: Error | null,
|
||||
callback: (error?: Error | null) => void
|
||||
): void {
|
||||
if (error) {
|
||||
drop(this.#cleanupAfterError());
|
||||
async *#run(): AsyncIterable<NonNullable<Backups.Frame.Params['item']>> {
|
||||
log.info('starting...');
|
||||
await AttachmentBackupManager.stop();
|
||||
|
||||
log.info('message migration starting...');
|
||||
await migrateAllMessages();
|
||||
|
||||
await pauseWriteAccess();
|
||||
let isPaused = true;
|
||||
try {
|
||||
yield* this.#unsafeRun();
|
||||
|
||||
isPaused = false;
|
||||
await resumeWriteAccess();
|
||||
|
||||
// TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction
|
||||
const { type } = this.#options;
|
||||
switch (type) {
|
||||
case 'remote':
|
||||
log.info(
|
||||
`Enqueuing ${this.#attachmentBackupJobs.length} remote attachment backup jobs`
|
||||
);
|
||||
await DataWriter.clearAllAttachmentBackupJobs();
|
||||
await Promise.all(
|
||||
this.#attachmentBackupJobs.map(job => {
|
||||
if (job.type === 'local') {
|
||||
log.error(
|
||||
"Can't enqueue local backup jobs during remote backup, skipping"
|
||||
);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return AttachmentBackupManager.addJobAndMaybeThumbnailJob(job);
|
||||
})
|
||||
);
|
||||
this.#attachmentBackupJobs = [];
|
||||
break;
|
||||
case 'plaintext-export':
|
||||
case 'local-encrypted':
|
||||
case 'cross-client-integration-test':
|
||||
break;
|
||||
default:
|
||||
throw missingCaseError(type);
|
||||
}
|
||||
log.info('finished successfully');
|
||||
} catch (error) {
|
||||
log.error('errored', toLogFormat(error));
|
||||
throw error;
|
||||
} finally {
|
||||
if (isPaused) {
|
||||
await resumeWriteAccess();
|
||||
}
|
||||
await AttachmentBackupManager.start();
|
||||
}
|
||||
callback(error);
|
||||
}
|
||||
|
||||
public run(): void {
|
||||
drop(
|
||||
(async () => {
|
||||
log.info('starting...');
|
||||
drop(AttachmentBackupManager.stop());
|
||||
log.info('message migration starting...');
|
||||
await migrateAllMessages();
|
||||
|
||||
await pauseWriteAccess();
|
||||
try {
|
||||
await this.#unsafeRun();
|
||||
await resumeWriteAccess();
|
||||
// TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction
|
||||
const { type } = this.#options;
|
||||
switch (type) {
|
||||
case 'remote':
|
||||
log.info(
|
||||
`Enqueuing ${this.#attachmentBackupJobs.length} remote attachment backup jobs`
|
||||
);
|
||||
await DataWriter.clearAllAttachmentBackupJobs();
|
||||
await Promise.all(
|
||||
this.#attachmentBackupJobs.map(job => {
|
||||
if (job.type === 'local') {
|
||||
log.error(
|
||||
"Can't enqueue local backup jobs during remote backup, skipping"
|
||||
);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return AttachmentBackupManager.addJobAndMaybeThumbnailJob(
|
||||
job
|
||||
);
|
||||
})
|
||||
);
|
||||
this.#attachmentBackupJobs = [];
|
||||
break;
|
||||
case 'plaintext-export':
|
||||
case 'local-encrypted':
|
||||
case 'cross-client-integration-test':
|
||||
break;
|
||||
default:
|
||||
throw missingCaseError(type);
|
||||
}
|
||||
log.info('finished successfully');
|
||||
} catch (error) {
|
||||
await this.#cleanupAfterError();
|
||||
log.error('errored', toLogFormat(error));
|
||||
this.emit('error', error);
|
||||
} finally {
|
||||
drop(AttachmentBackupManager.start());
|
||||
}
|
||||
})()
|
||||
);
|
||||
}
|
||||
|
||||
public getMediaNames(): Array<string> {
|
||||
@@ -409,38 +394,15 @@ export class BackupExportStream extends Readable {
|
||||
return this.#attachmentBackupJobs;
|
||||
}
|
||||
|
||||
async #unsafeRun(): Promise<void> {
|
||||
async *#unsafeRun(): AsyncIterable<
|
||||
NonNullable<Backups.Frame.Params['item']>
|
||||
> {
|
||||
this.#ourConversation =
|
||||
window.ConversationController.getOurConversationOrThrow().attributes;
|
||||
const backupInfo: Backups.BackupInfo.Params = {
|
||||
version: BACKUP_VERSION,
|
||||
backupTimeMs: this.#backupTimeMs,
|
||||
mediaRootBackupKey: getBackupMediaRootKey().serialize(),
|
||||
firstAppVersion: itemStorage.get('restoredBackupFirstAppVersion') ?? null,
|
||||
currentAppVersion: `Desktop ${window.getVersion()}`,
|
||||
debugInfo: null,
|
||||
};
|
||||
|
||||
if (this.#options.type === 'plaintext-export') {
|
||||
const { exporter, chunk: initialChunk } = BackupJsonExporter.start(
|
||||
Backups.BackupInfo.encode(backupInfo),
|
||||
{ validate: false }
|
||||
);
|
||||
|
||||
this.#jsonExporter = exporter;
|
||||
this.push(`${initialChunk}\n`);
|
||||
} else {
|
||||
for (const chunk of encodeDelimited(
|
||||
Backups.BackupInfo.encode(backupInfo)
|
||||
)) {
|
||||
this.push(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
this.#pushFrame({
|
||||
yield {
|
||||
account: await this.#toAccountData(),
|
||||
});
|
||||
await this.#flush();
|
||||
};
|
||||
|
||||
const identityKeys = await DataReader.getAllIdentityKeys();
|
||||
const identityKeysById = new Map(
|
||||
@@ -476,22 +438,19 @@ export class BackupExportStream extends Readable {
|
||||
continue;
|
||||
}
|
||||
|
||||
this.#pushFrame({
|
||||
yield {
|
||||
recipient,
|
||||
});
|
||||
};
|
||||
|
||||
// oxlint-disable-next-line no-await-in-loop
|
||||
await this.#flush();
|
||||
this.#stats.conversations += 1;
|
||||
}
|
||||
|
||||
this.#pushFrame({
|
||||
yield {
|
||||
recipient: {
|
||||
id: this.#getNextRecipientId(),
|
||||
destination: { releaseNotes: {} },
|
||||
},
|
||||
});
|
||||
await this.#flush();
|
||||
};
|
||||
|
||||
const distributionLists =
|
||||
await DataReader.getAllStoryDistributionsWithMembers();
|
||||
@@ -514,38 +473,40 @@ export class BackupExportStream extends Readable {
|
||||
privacyMode = PrivacyMode.ONLY_WITH;
|
||||
}
|
||||
|
||||
this.#pushFrame({
|
||||
const destination = {
|
||||
distributionList: {
|
||||
distributionId: uuidToBytes(list.id),
|
||||
item: list.deletedAtTimestamp
|
||||
? {
|
||||
deletionTimestamp: BigInt(list.deletedAtTimestamp),
|
||||
}
|
||||
: {
|
||||
distributionList: {
|
||||
name: list.name,
|
||||
allowReplies: list.allowsReplies,
|
||||
privacyMode,
|
||||
memberRecipientIds: list.members
|
||||
.map(serviceId =>
|
||||
this.#getRecipientByServiceId(
|
||||
serviceId,
|
||||
'distributionList.memberRecipientIds'
|
||||
)
|
||||
)
|
||||
.filter(isNotNil),
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
yield* this.#flushPendingRecipients();
|
||||
|
||||
yield {
|
||||
recipient: {
|
||||
id: this.#getNextRecipientId(),
|
||||
destination: {
|
||||
distributionList: {
|
||||
distributionId: uuidToBytes(list.id),
|
||||
item: list.deletedAtTimestamp
|
||||
? {
|
||||
deletionTimestamp: BigInt(list.deletedAtTimestamp),
|
||||
}
|
||||
: {
|
||||
distributionList: {
|
||||
name: list.name,
|
||||
allowReplies: list.allowsReplies,
|
||||
privacyMode,
|
||||
memberRecipientIds: list.members
|
||||
.map(serviceId =>
|
||||
this.#getRecipientByServiceId(
|
||||
serviceId,
|
||||
'distributionList.memberRecipientIds'
|
||||
)
|
||||
)
|
||||
.filter(isNotNil),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
destination,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
// oxlint-disable-next-line no-await-in-loop
|
||||
await this.#flush();
|
||||
this.#stats.distributionLists += 1;
|
||||
}
|
||||
|
||||
@@ -572,7 +533,7 @@ export class BackupExportStream extends Readable {
|
||||
|
||||
this.#roomIdToRecipientId.set(roomId, id);
|
||||
|
||||
this.#pushFrame({
|
||||
yield {
|
||||
recipient: {
|
||||
id,
|
||||
destination: {
|
||||
@@ -587,25 +548,21 @@ export class BackupExportStream extends Readable {
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
// oxlint-disable-next-line no-await-in-loop
|
||||
await this.#flush();
|
||||
this.#stats.callLinks += 1;
|
||||
}
|
||||
|
||||
const stickerPacks = await getStickerPacksForBackup();
|
||||
|
||||
for (const { id, key } of stickerPacks) {
|
||||
this.#pushFrame({
|
||||
yield {
|
||||
stickerPack: {
|
||||
packId: Bytes.fromHex(id),
|
||||
packKey: Bytes.fromBase64(key),
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
// oxlint-disable-next-line no-await-in-loop
|
||||
await this.#flush();
|
||||
this.#stats.stickerPacks += 1;
|
||||
}
|
||||
|
||||
@@ -649,7 +606,7 @@ export class BackupExportStream extends Readable {
|
||||
}
|
||||
}
|
||||
|
||||
this.#pushFrame({
|
||||
yield {
|
||||
chat: {
|
||||
// We don't have to use separate identifiers
|
||||
id: recipientId,
|
||||
@@ -686,10 +643,8 @@ export class BackupExportStream extends Readable {
|
||||
autoBubbleColor: attributes.autoBubbleColor,
|
||||
}),
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
// oxlint-disable-next-line no-await-in-loop
|
||||
await this.#flush();
|
||||
this.#stats.chats += 1;
|
||||
}
|
||||
|
||||
@@ -728,17 +683,15 @@ export class BackupExportStream extends Readable {
|
||||
continue;
|
||||
}
|
||||
|
||||
this.#pushFrame({
|
||||
yield {
|
||||
adHocCall: {
|
||||
callId,
|
||||
recipientId,
|
||||
state: toAdHocCallStateProto(status),
|
||||
callTimestamp: BigInt(timestamp),
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
// oxlint-disable-next-line no-await-in-loop
|
||||
await this.#flush();
|
||||
this.#stats.adHocCalls += 1;
|
||||
}
|
||||
|
||||
@@ -813,7 +766,7 @@ export class BackupExportStream extends Readable {
|
||||
return true;
|
||||
});
|
||||
|
||||
this.#pushFrame({
|
||||
yield {
|
||||
notificationProfile: {
|
||||
id: Bytes.fromHex(id),
|
||||
name,
|
||||
@@ -828,10 +781,8 @@ export class BackupExportStream extends Readable {
|
||||
scheduleEndTime,
|
||||
scheduleDaysEnabled: toDayOfWeekArray(scheduleDaysEnabled) ?? null,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
// oxlint-disable-next-line no-await-in-loop
|
||||
await this.#flush();
|
||||
this.#stats.notificationProfiles += 1;
|
||||
}
|
||||
|
||||
@@ -848,7 +799,7 @@ export class BackupExportStream extends Readable {
|
||||
continue;
|
||||
}
|
||||
|
||||
this.#pushFrame({
|
||||
yield {
|
||||
chatFolder: {
|
||||
id: uuidToBytes(chatFolder.id),
|
||||
name: chatFolder.name,
|
||||
@@ -874,10 +825,8 @@ export class BackupExportStream extends Readable {
|
||||
})
|
||||
.filter(isNotNil),
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
// oxlint-disable-next-line no-await-in-loop
|
||||
await this.#flush();
|
||||
this.#stats.chatFolders += 1;
|
||||
}
|
||||
|
||||
@@ -900,6 +849,8 @@ export class BackupExportStream extends Readable {
|
||||
}
|
||||
);
|
||||
|
||||
// Note: chatItem MUST be the last item type to be exported in order
|
||||
// to produce correct encoding in `ts/services/backups/encoding.node.ts`
|
||||
for await (const chatItem of iter) {
|
||||
if (chatItem === undefined) {
|
||||
this.#stats.skippedMessages += 1;
|
||||
@@ -907,106 +858,35 @@ export class BackupExportStream extends Readable {
|
||||
continue;
|
||||
}
|
||||
|
||||
this.#pushFrame({
|
||||
yield* this.#flushPendingRecipients();
|
||||
|
||||
yield {
|
||||
chatItem,
|
||||
});
|
||||
};
|
||||
this.#stats.messages += 1;
|
||||
|
||||
if (
|
||||
this.#options.validationRun ||
|
||||
this.#stats.messages % FLUSH_EVERY === 0
|
||||
) {
|
||||
// flush every chatItem to expose all validation errors
|
||||
await this.#flush();
|
||||
}
|
||||
}
|
||||
|
||||
await this.#flush();
|
||||
|
||||
log.warn('final stats', {
|
||||
...this.#stats,
|
||||
attachmentBackupJobs: this.#attachmentBackupJobs.length,
|
||||
});
|
||||
|
||||
if (this.#jsonExporter) {
|
||||
try {
|
||||
const result = this.#jsonExporter.finish();
|
||||
if (result?.errorMessage) {
|
||||
log.warn(
|
||||
'jsonExporter.finish() returned validation error:',
|
||||
result.errorMessage
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
// We only warn because this isn't that big of a deal - the export is complete.
|
||||
// All we need from the exporter at the end is any validation errors it found.
|
||||
log.warn('jsonExporter returned error', toLogFormat(error));
|
||||
}
|
||||
}
|
||||
|
||||
this.push(null);
|
||||
}
|
||||
|
||||
#pushFrame(frame: Backups.Frame.Params['item']): void {
|
||||
const encodedFrame = Backups.Frame.encode({ item: frame });
|
||||
if (this.#options.type === 'plaintext-export') {
|
||||
const delimitedFrame = Buffer.concat(encodeDelimited(encodedFrame));
|
||||
strictAssert(
|
||||
this.#jsonExporter != null,
|
||||
'jsonExported must be initialized'
|
||||
);
|
||||
|
||||
const results = this.#jsonExporter.exportFrames(delimitedFrame);
|
||||
for (const result of results) {
|
||||
if (result.errorMessage) {
|
||||
log.warn(
|
||||
'frameToJson: frame had a validation error:',
|
||||
result.errorMessage
|
||||
);
|
||||
}
|
||||
if (!result.line) {
|
||||
log.error('frameToJson: frame was filtered out by libsignal');
|
||||
} else {
|
||||
this.#buffers.push(Buffer.from(`${result.line}\n`));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
this.#buffers.push(...encodeDelimited(encodedFrame));
|
||||
}
|
||||
}
|
||||
|
||||
async #flush(): Promise<void> {
|
||||
const chunk = Bytes.concatenate(this.#buffers);
|
||||
this.#buffers = [];
|
||||
|
||||
// Below watermark, no pausing required
|
||||
if (this.push(chunk)) {
|
||||
async *#flushPendingRecipients(): AsyncIterable<
|
||||
NonNullable<Backups.Frame.Params['item']>
|
||||
> {
|
||||
// We might have created ad-hoc recipients for service ids/e164s without
|
||||
// matching conversation while processing the message. Flush them out.
|
||||
const pendingRecipients = this.#pendingRecipients;
|
||||
if (pendingRecipients.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { promise, resolve } = explodePromise<void>();
|
||||
strictAssert(this.#flushResolve === undefined, 'flush already pending');
|
||||
this.#flushResolve = resolve;
|
||||
|
||||
const start = Date.now();
|
||||
log.info('flush paused due to pushback');
|
||||
try {
|
||||
await pTimeout(promise, {
|
||||
milliseconds: FLUSH_TIMEOUT,
|
||||
});
|
||||
} finally {
|
||||
const duration = Date.now() - start;
|
||||
if (duration > REPORTING_THRESHOLD) {
|
||||
log.info(`flush resumed after ${duration}ms`);
|
||||
}
|
||||
this.#flushResolve = undefined;
|
||||
this.#pendingRecipients = [];
|
||||
for (const recipient of pendingRecipients) {
|
||||
yield { recipient };
|
||||
}
|
||||
}
|
||||
|
||||
override _read(): void {
|
||||
this.#flushResolve?.();
|
||||
}
|
||||
|
||||
async #toAccountData(): Promise<Backups.AccountData.Params> {
|
||||
const me = window.ConversationController.getOurConversationOrThrow();
|
||||
|
||||
@@ -1233,7 +1113,7 @@ export class BackupExportStream extends Readable {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
this.#pushFrame({ recipient });
|
||||
this.#pendingRecipients.push(recipient);
|
||||
strictAssert(recipient.id != null, 'recipient.id must exist');
|
||||
return recipient.id;
|
||||
}
|
||||
@@ -1254,7 +1134,7 @@ export class BackupExportStream extends Readable {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
this.#pushFrame({ recipient });
|
||||
this.#pendingRecipients.push(recipient);
|
||||
strictAssert(recipient.id != null, 'recipient.id must exist');
|
||||
return recipient.id;
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import { createReadStream, createWriteStream } from 'node:fs';
|
||||
import { mkdir, rm, stat, unlink, writeFile } from 'node:fs/promises';
|
||||
import fsExtra from 'fs-extra';
|
||||
import { basename, join } from 'node:path';
|
||||
import { createGzip, createGunzip } from 'node:zlib';
|
||||
import { createGunzip } from 'node:zlib';
|
||||
import { createCipheriv, createHmac, randomBytes } from 'node:crypto';
|
||||
import lodash from 'lodash';
|
||||
import { BackupLevel } from '@signalapp/libsignal-client/zkgroup.js';
|
||||
@@ -26,7 +26,6 @@ import { getAbsoluteDownloadsPath } from '../../util/migrations.preload.ts';
|
||||
import { waitForAllBatchers } from '../../util/batcher.std.ts';
|
||||
import { flushAllWaitBatchers } from '../../util/waitBatcher.std.ts';
|
||||
import { DelimitedStream } from '../../util/DelimitedStream.node.ts';
|
||||
import { appendPaddingStream } from '../../util/logPadding.node.ts';
|
||||
import { prependStream } from '../../util/prependStream.node.ts';
|
||||
import { appendMacStream } from '../../util/appendMacStream.node.ts';
|
||||
import { getMacAndUpdateHmac } from '../../util/getMacAndUpdateHmac.node.ts';
|
||||
@@ -54,6 +53,11 @@ import { isTestOrMockEnvironment } from '../../environment.std.ts';
|
||||
import { runStorageServiceSyncJob } from '../storage.preload.ts';
|
||||
import { BackupExportStream } from './export.preload.ts';
|
||||
import { BackupImportStream } from './import.preload.ts';
|
||||
import {
|
||||
toPaddedGzipIterable,
|
||||
toDelimitedIterable,
|
||||
toJSONIterable,
|
||||
} from './encoding.node.ts';
|
||||
import {
|
||||
getBackupId,
|
||||
getKeyMaterial,
|
||||
@@ -63,7 +67,7 @@ import { BackupCredentials } from './credentials.preload.ts';
|
||||
import { BackupAPI } from './api.preload.ts';
|
||||
import {
|
||||
validateBackup,
|
||||
validateBackupStream,
|
||||
validateBackupIterator,
|
||||
ValidationType,
|
||||
} from './validator.preload.ts';
|
||||
import type {
|
||||
@@ -747,14 +751,11 @@ export class BackupsService {
|
||||
const start = Date.now();
|
||||
|
||||
window.IPC.startTrackingQueryStats();
|
||||
const recordStream = new BackupExportStream({
|
||||
...exportOptions,
|
||||
validationRun: true,
|
||||
});
|
||||
const exportStream = new BackupExportStream(exportOptions);
|
||||
|
||||
recordStream.run();
|
||||
const { info, iterable } = exportStream.run();
|
||||
|
||||
const totalBytes = await validateBackupStream(recordStream);
|
||||
const totalBytes = await validateBackupIterator(info, iterable);
|
||||
window.IPC.stopTrackingQueryStats({
|
||||
epochName: 'Internal Validate Backup',
|
||||
});
|
||||
@@ -764,10 +765,10 @@ export class BackupsService {
|
||||
log.info('internal validation: succeeded');
|
||||
return {
|
||||
result: {
|
||||
attachmentBackupJobs: recordStream.getAttachmentBackupJobs(),
|
||||
mediaNames: recordStream.getMediaNames(),
|
||||
attachmentBackupJobs: exportStream.getAttachmentBackupJobs(),
|
||||
mediaNames: exportStream.getMediaNames(),
|
||||
duration,
|
||||
stats: recordStream.getStats(),
|
||||
stats: exportStream.getStats(),
|
||||
totalBytes,
|
||||
},
|
||||
};
|
||||
@@ -1202,9 +1203,9 @@ export class BackupsService {
|
||||
}
|
||||
|
||||
const { aesKey, macKey } = getKeyMaterial();
|
||||
const recordStream = new BackupExportStream(options);
|
||||
const exportStream = new BackupExportStream(options);
|
||||
|
||||
recordStream.run();
|
||||
const runOutput = exportStream.run();
|
||||
|
||||
const iv = randomBytes(IV_LENGTH);
|
||||
|
||||
@@ -1215,9 +1216,7 @@ export class BackupsService {
|
||||
case 'remote':
|
||||
case 'local-encrypted':
|
||||
await pipeline(
|
||||
recordStream,
|
||||
createGzip(),
|
||||
appendPaddingStream(),
|
||||
toPaddedGzipIterable(runOutput),
|
||||
createCipheriv(CipherType.AES256CBC, aesKey, iv),
|
||||
prependStream(iv),
|
||||
appendMacStream(macKey),
|
||||
@@ -1236,7 +1235,7 @@ export class BackupsService {
|
||||
'exportBackup: Plaintext backups can be exported only in test harness'
|
||||
);
|
||||
await pipeline(
|
||||
recordStream,
|
||||
toDelimitedIterable(runOutput),
|
||||
measureSize({
|
||||
onComplete: size => {
|
||||
totalBytes = size;
|
||||
@@ -1248,7 +1247,7 @@ export class BackupsService {
|
||||
break;
|
||||
case 'plaintext-export':
|
||||
await pipeline(
|
||||
recordStream,
|
||||
toJSONIterable(runOutput, log),
|
||||
measureSize({
|
||||
onComplete: size => {
|
||||
totalBytes = size;
|
||||
@@ -1264,10 +1263,10 @@ export class BackupsService {
|
||||
|
||||
const duration = Date.now() - start;
|
||||
return {
|
||||
attachmentBackupJobs: recordStream.getAttachmentBackupJobs(),
|
||||
mediaNames: recordStream.getMediaNames(),
|
||||
attachmentBackupJobs: exportStream.getAttachmentBackupJobs(),
|
||||
mediaNames: exportStream.getMediaNames(),
|
||||
totalBytes,
|
||||
stats: recordStream.getStats(),
|
||||
stats: exportStream.getStats(),
|
||||
duration,
|
||||
};
|
||||
} finally {
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
// Copyright 2024 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import { type Readable, PassThrough } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import * as libsignal from '@signalapp/libsignal-client/dist/MessageBackup.js';
|
||||
import type { InputStream } from '@signalapp/libsignal-client/dist/io.js';
|
||||
|
||||
import { strictAssert } from '../../util/assert.std.ts';
|
||||
import { toAciObject } from '../../util/ServiceId.node.ts';
|
||||
import { missingCaseError } from '../../util/missingCaseError.std.ts';
|
||||
import { DelimitedStream } from '../../util/DelimitedStream.node.ts';
|
||||
import { itemStorage } from '../../textsecure/Storage.preload.ts';
|
||||
import { Backups } from '../../protobuf/index.std.ts';
|
||||
|
||||
export enum ValidationType {
|
||||
Export = 'Export',
|
||||
@@ -53,44 +51,31 @@ export async function validateBackup(
|
||||
}
|
||||
}
|
||||
|
||||
export async function validateBackupStream(
|
||||
readable: Readable
|
||||
export async function validateBackupIterator(
|
||||
info: Backups.BackupInfo.Params,
|
||||
iterable: AsyncIterable<NonNullable<Backups.Frame.Params['item']>>
|
||||
): Promise<number> {
|
||||
let validator: libsignal.OnlineBackupValidator | undefined;
|
||||
const infoBuf = Backups.BackupInfo.encode(info);
|
||||
let totalBytes = infoBuf.byteLength;
|
||||
|
||||
const validator = new libsignal.OnlineBackupValidator(
|
||||
infoBuf,
|
||||
libsignal.Purpose.RemoteBackup
|
||||
);
|
||||
|
||||
let totalBytes = 0;
|
||||
let frameCount = 0;
|
||||
const allErrorMessages: Array<string> = [];
|
||||
let frameCount = 0;
|
||||
for await (const item of iterable) {
|
||||
const frameBuf = Backups.Frame.encode({ item });
|
||||
|
||||
const countBytes = new PassThrough();
|
||||
countBytes.on('data', bytes => {
|
||||
totalBytes += bytes.byteLength;
|
||||
});
|
||||
|
||||
const delimited = new DelimitedStream();
|
||||
delimited.on('data', frame => {
|
||||
frameCount += 1;
|
||||
|
||||
// Info frame
|
||||
if (frameCount === 1) {
|
||||
validator = new libsignal.OnlineBackupValidator(
|
||||
frame,
|
||||
libsignal.Purpose.RemoteBackup
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
strictAssert(validator != null, 'validator must be already created');
|
||||
try {
|
||||
validator.addFrame(frame);
|
||||
totalBytes += frameBuf.byteLength;
|
||||
frameCount += 1;
|
||||
validator.addFrame(frameBuf);
|
||||
} catch (error) {
|
||||
allErrorMessages.push(error.message);
|
||||
}
|
||||
});
|
||||
|
||||
await pipeline(readable, countBytes, delimited);
|
||||
|
||||
strictAssert(validator != null, 'no frames');
|
||||
}
|
||||
|
||||
try {
|
||||
validator.finalize();
|
||||
|
||||
@@ -200,6 +200,23 @@ const DEFAULT_REMOTE_CONFIG = [
|
||||
'desktop.libsignalNet.grpc.BackupsAnonymousGetUploadForm',
|
||||
{ enabled: true },
|
||||
],
|
||||
// TODO: move to gRPC
|
||||
[
|
||||
'desktop.libsignalNet.grpc.AccountsAnonymousCheckAccountExistence.2',
|
||||
{ enabled: true, value: 'ws' },
|
||||
],
|
||||
[
|
||||
'desktop.libsignalNet.grpc.KeyTransparencyQueryServiceSearchV2',
|
||||
{ enabled: true, value: 'ws' },
|
||||
],
|
||||
[
|
||||
'desktop.libsignalNet.grpc.MessagesAnonymousSendSingleRecipientMessage',
|
||||
{ enabled: true, value: 'ws' },
|
||||
],
|
||||
[
|
||||
'desktop.libsignalNet.grpc.MessagesSendMessage',
|
||||
{ enabled: true, value: 'ws' },
|
||||
],
|
||||
] as const;
|
||||
|
||||
//
|
||||
|
||||
@@ -1248,10 +1248,10 @@ export default class AccountManager extends EventTarget {
|
||||
response.aci.getServiceIdString(),
|
||||
'#doCreateAccount'
|
||||
);
|
||||
ourPni = normalizePni(
|
||||
response.pni.getServiceIdString(),
|
||||
'#doCreateAccount'
|
||||
);
|
||||
ourPni =
|
||||
response.pni == null
|
||||
? undefined
|
||||
: normalizePni(response.pni.getServiceIdString(), '#doCreateAccount');
|
||||
deviceId = 1;
|
||||
|
||||
result = {
|
||||
|
||||
Reference in New Issue
Block a user