Signal/Desktop
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Desktop.git synced 2026-04-24 18:38:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

Normalize message attachments

Browse Source
This commit is contained in:
trevor-signal
2025-05-22 21:09:54 -04:00
committed by GitHub
parent 8d8e0329cf
commit d6e81eee11
39 changed files with 2540 additions and 807 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
ts/sql/Client.ts
View File

@@ -49,7 +49,6 @@ import type {
ItemType,
StoredItemType,
MessageType,
MessageTypeUnhydrated,
PreKeyIdType,
PreKeyType,
StoredPreKeyType,
@@ -62,7 +61,6 @@ import type {
ClientOnlyReadableInterface,
ClientOnlyWritableInterface,
} from './Interface';
import { hydrateMessage } from './hydration';
import type { MessageAttributesType } from '../model-types';
import type { AttachmentDownloadJobType } from '../types/AttachmentDownload';
@@ -546,7 +544,6 @@ function handleSearchMessageJSON(
messages: Array<ServerSearchResultMessageType>
): Array<ClientSearchResultMessageType> {
return messages.map<ClientSearchResultMessageType>(message => {
const parsedMessage = hydrateMessage(message);
assertDev(
message.ftsSnippet ?? typeof message.mentionStart === 'number',
'Neither ftsSnippet nor matching mention returned from message search'
@@ -554,7 +551,7 @@ function handleSearchMessageJSON(
const snippet =
message.ftsSnippet ??
generateSnippetAroundMention({
body: parsedMessage.body || '',
body: message.body || '',
mentionStart: message.mentionStart ?? 0,
mentionLength: message.mentionLength ?? 1,
});
@@ -562,7 +559,7 @@ function handleSearchMessageJSON(
return {
// Empty array is a default value. `message.json` has the real field
bodyRanges: [],
...parsedMessage,
...message,
snippet,
};
});
@@ -629,15 +626,17 @@ async function saveMessages(
forceSave,
ourAci,
postSaveUpdates,
_testOnlyAvoidNormalizingAttachments,
}: {
forceSave?: boolean;
ourAci: AciString;
postSaveUpdates: () => Promise<void>;
_testOnlyAvoidNormalizingAttachments?: boolean;
}
): Promise<Array<string>> {
const result = await writableChannel.saveMessages(
arrayOfMessages.map(message => _cleanMessageData(message)),
{ forceSave, ourAci }
{ forceSave, ourAci, _testOnlyAvoidNormalizingAttachments }
);
drop(postSaveUpdates?.());
@@ -730,19 +729,13 @@ async function removeMessages(
await writableChannel.removeMessages(messageIds);
}
function handleMessageJSON(
messages: Array<MessageTypeUnhydrated>
): Array<MessageType> {
return messages.map(message => hydrateMessage(message));
}
async function getNewerMessagesByConversation(
options: AdjacentMessagesByConversationOptionsType
): Promise<Array<MessageType>> {
const messages =
await readableChannel.getNewerMessagesByConversation(options);
return handleMessageJSON(messages);
return messages;
}
async function getRecentStoryReplies(
@@ -754,7 +747,7 @@ async function getRecentStoryReplies(
options
);
return handleMessageJSON(messages);
return messages;
}
async function getOlderMessagesByConversation(
@@ -763,7 +756,7 @@ async function getOlderMessagesByConversation(
const messages =
await readableChannel.getOlderMessagesByConversation(options);
return handleMessageJSON(messages);
return messages;
}
async function getConversationRangeCenteredOnMessage(
@@ -772,11 +765,7 @@ async function getConversationRangeCenteredOnMessage(
const result =
await readableChannel.getConversationRangeCenteredOnMessage(options);
return {
...result,
older: handleMessageJSON(result.older),
newer: handleMessageJSON(result.newer),
};
return result;
}
async function removeMessagesInConversation(
@@ -832,9 +821,13 @@ async function saveAttachmentDownloadJob(
// Other
async function cleanupOrphanedAttachments(): Promise<void> {
async function cleanupOrphanedAttachments({
_block = false,
}: {
_block?: boolean;
} = {}): Promise<void> {
try {
await invokeWithTimeout(CLEANUP_ORPHANED_ATTACHMENTS_KEY);
await invokeWithTimeout(CLEANUP_ORPHANED_ATTACHMENTS_KEY, { _block });
} catch (error) {
log.warn(
'sql/Client: cleanupOrphanedAttachments failure',
@@ -859,9 +852,12 @@ async function removeOtherData(): Promise<void> {
]);
}
async function invokeWithTimeout(name: string): Promise<void> {
async function invokeWithTimeout(
name: string,
...args: Array<unknown>
): Promise<void> {
return createTaskWithTimeout(
() => ipc.invoke(name),
() => ipc.invoke(name, ...args),
`callChannel call to ${name}`
)();
}

163
ts/sql/Interface.ts
View File

@@ -37,7 +37,10 @@ import type {
CallLinkType,
DefunctCallLinkType,
} from '../types/CallLink';
import type { AttachmentDownloadJobType } from '../types/AttachmentDownload';
import type {
AttachmentDownloadJobType,
AttachmentDownloadJobTypeType,
} from '../types/AttachmentDownload';
import type {
GroupSendEndorsementsData,
GroupSendMemberEndorsementRecord,
@@ -46,6 +49,7 @@ import type { SyncTaskType } from '../util/syncTasks';
import type { AttachmentBackupJobType } from '../types/AttachmentBackup';
import type { GifType } from '../components/fun/panels/FunPanelGifs';
import type { NotificationProfileType } from '../types/NotificationProfile';
import { strictAssert } from '../util/assert';
export type ReadableDB = Database & { __readable_db: never };
export type WritableDB = ReadableDB & { __writable_db: never };
@@ -217,7 +221,10 @@ export type StoredPreKeyType = PreKeyType & {
privateKey: string;
publicKey: string;
};
export type ServerSearchResultMessageType = MessageTypeUnhydrated & {
export type ServerSearchResultMessageType = MessageType &
ServerMessageSearchResultType;
export type ServerMessageSearchResultType = {
// If the FTS matches text in message.body, snippet will be populated
ftsSnippet: string | null;
@@ -537,6 +544,136 @@ export enum AttachmentDownloadSource {
BACKFILL = 'backfill',
}
export const MESSAGE_ATTACHMENT_COLUMNS = [
'messageId',
'conversationId',
'sentAt',
'attachmentType',
'orderInMessage',
'editHistoryIndex',
'clientUuid',
'size',
'contentType',
'path',
'localKey',
'plaintextHash',
'caption',
'fileName',
'blurHash',
'height',
'width',
'digest',
'key',
'iv',
'flags',
'downloadPath',
'transitCdnKey',
'transitCdnNumber',
'transitCdnUploadTimestamp',
'backupMediaName',
'backupCdnNumber',
'incrementalMac',
'incrementalMacChunkSize',
'isReencryptableToSameDigest',
'reencryptionIv',
'reencryptionKey',
'reencryptionDigest',
'thumbnailPath',
'thumbnailSize',
'thumbnailContentType',
'thumbnailLocalKey',
'thumbnailVersion',
'screenshotPath',
'screenshotSize',
'screenshotContentType',
'screenshotLocalKey',
'screenshotVersion',
'backupThumbnailPath',
'backupThumbnailSize',
'backupThumbnailContentType',
'backupThumbnailLocalKey',
'backupThumbnailVersion',
'storyTextAttachmentJson',
'localBackupPath',
'isCorrupted',
'backfillError',
'error',
'wasTooBig',
'copiedFromQuotedAttachment',
'version',
'pending',
] as const satisfies Array<keyof MessageAttachmentDBType>;
export type MessageAttachmentDBType = {
messageId: string;
attachmentType: AttachmentDownloadJobTypeType;
orderInMessage: number;
editHistoryIndex: number | null;
conversationId: string;
sentAt: number;
clientUuid: string | null;
size: number;
contentType: string;
path: string | null;
plaintextHash: string | null;
downloadPath: string | null;
caption: string | null;
blurHash: string | null;
width: number | null;
height: number | null;
flags: number | null;
key: string | null;
iv: string | null;
digest: string | null;
fileName: string | null;
incrementalMac: string | null;
incrementalMacChunkSize: number | null;
localKey: string | null;
version: 1 | 2 | null;
transitCdnKey: string | null;
transitCdnNumber: number | null;
transitCdnUploadTimestamp: number | null;
backupMediaName: string | null;
backupCdnNumber: number | null;
thumbnailPath: string | null;
thumbnailSize: number | null;
thumbnailContentType: string | null;
thumbnailLocalKey: string | null;
thumbnailVersion: 1 | 2 | null;
screenshotPath: string | null;
screenshotSize: number | null;
screenshotContentType: string | null;
screenshotLocalKey: string | null;
screenshotVersion: 1 | 2 | null;
backupThumbnailPath: string | null;
backupThumbnailSize: number | null;
backupThumbnailContentType: string | null;
backupThumbnailLocalKey: string | null;
backupThumbnailVersion: 1 | 2 | null;
reencryptionIv: string | null;
reencryptionKey: string | null;
reencryptionDigest: string | null;
storyTextAttachmentJson: string | null;
localBackupPath: string | null;
isCorrupted: 1 | 0 | null;
backfillError: 1 | 0 | null;
error: 1 | 0 | null;
wasTooBig: 1 | 0 | null;
pending: 1 | 0 | null;
isReencryptableToSameDigest: 1 | 0 | null;
copiedFromQuotedAttachment: 1 | 0 | null;
};
// Test to make sure that MESSAGE_ATTACHMENT_COLUMNS &
// MessageAttachmentDBReferenceType remain in sync!
const testDBRefTypeMatchesColumnNames = true as unknown as [
keyof MessageAttachmentDBType,
] satisfies [(typeof MESSAGE_ATTACHMENT_COLUMNS)[number]];
strictAssert(
testDBRefTypeMatchesColumnNames,
'attachment_columns must match DB fields type'
);
type ReadableInterface = {
close: () => void;
@@ -744,6 +881,9 @@ type ReadableInterface = {
getStatisticsForLogging(): Record<string, string>;
getSizeOfPendingBackupAttachmentDownloadJobs(): number;
getAttachmentReferencesForMessages: (
messageIds: Array<string>
) => Array<MessageAttachmentDBType>;
};
type WritableInterface = {
@@ -866,6 +1006,7 @@ type WritableInterface = {
) => void;
_removeAllReactions: () => void;
_removeAllMessages: () => void;
_removeMessage: (id: string) => void;
incrementMessagesMigrationAttempts: (
messageIds: ReadonlyArray<string>
) => void;
@@ -1072,16 +1213,16 @@ export type ServerReadableDirectInterface = ReadableInterface & {
getRecentStoryReplies(
storyId: string,
options?: GetRecentStoryRepliesOptionsType
): Array<MessageTypeUnhydrated>;
): Array<MessageType>;
getOlderMessagesByConversation: (
options: AdjacentMessagesByConversationOptionsType
) => Array<MessageTypeUnhydrated>;
) => Array<MessageType>;
getNewerMessagesByConversation: (
options: AdjacentMessagesByConversationOptionsType
) => Array<MessageTypeUnhydrated>;
) => Array<MessageType>;
getConversationRangeCenteredOnMessage: (
options: AdjacentMessagesByConversationOptionsType
) => GetConversationRangeCenteredOnMessageResultType<MessageTypeUnhydrated>;
) => GetConversationRangeCenteredOnMessageResultType<MessageType>;
getIdentityKeyById: (
id: IdentityKeyIdType
@@ -1141,7 +1282,11 @@ export type ServerWritableDirectInterface = WritableInterface & {
) => string;
saveMessages: (
arrayOfMessages: ReadonlyArray<ReadonlyDeep<MessageType>>,
options: { forceSave?: boolean; ourAci: AciString }
options: {
forceSave?: boolean;
ourAci: AciString;
_testOnlyAvoidNormalizingAttachments?: boolean;
}
) => Array<string>;
saveMessagesIndividually: (
arrayOfMessages: ReadonlyArray<ReadonlyDeep<MessageType>>,
@@ -1241,6 +1386,7 @@ export type ClientOnlyWritableInterface = ClientInterfaceWrap<{
forceSave?: boolean;
ourAci: AciString;
postSaveUpdates: () => Promise<void>;
_testOnlyAvoidNormalizingAttachments?: boolean;
}
) => string;
saveMessages: (
@@ -1249,6 +1395,7 @@ export type ClientOnlyWritableInterface = ClientInterfaceWrap<{
forceSave?: boolean;
ourAci: AciString;
postSaveUpdates: () => Promise<void>;
_testOnlyAvoidNormalizingAttachments?: boolean;
}
) => Array<string>;
saveMessagesIndividually: (
@@ -1311,7 +1458,7 @@ export type ClientOnlyWritableInterface = ClientInterfaceWrap<{
}
) => void;
removeOtherData: () => void;
cleanupOrphanedAttachments: () => void;
cleanupOrphanedAttachments: (options?: { _block: boolean }) => void;
ensureFilePermissions: () => void;
}>;

910
ts/sql/Server.ts
View File

File diff suppressed because it is too large Load Diff

343
ts/sql/hydration.ts
View File

@@ -1,10 +1,11 @@
// Copyright 2025 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { groupBy } from 'lodash';
import type { ReadStatus } from '../messages/MessageReadStatus';
import type { SeenStatus } from '../MessageSeenStatus';
import type { ServiceIdString } from '../types/ServiceId';
import { dropNull } from '../util/dropNull';
import { dropNull, shallowDropNull } from '../util/dropNull';
/* eslint-disable camelcase */
@@ -12,7 +13,23 @@ import type {
MessageTypeUnhydrated,
MessageType,
MESSAGE_COLUMNS,
ReadableDB,
MessageAttachmentDBType,
} from './Interface';
import {
batchMultiVarQuery,
convertOptionalIntegerToBoolean,
jsonToObject,
sql,
sqlJoin,
} from './util';
import type { AttachmentType } from '../types/Attachment';
import { IMAGE_JPEG, stringToMIMEType } from '../types/MIME';
import { strictAssert } from '../util/assert';
import { sqlLogger } from './sqlLogger';
import type { MessageAttributesType } from '../model-types';
export const ROOT_MESSAGE_ATTACHMENT_EDIT_HISTORY_INDEX = -1;
function toBoolean(value: number | null): boolean | undefined {
if (value == null) {
@@ -21,7 +38,27 @@ function toBoolean(value: number | null): boolean | undefined {
return value === 1;
}
export function hydrateMessage(row: MessageTypeUnhydrated): MessageType {
export function hydrateMessage(
db: ReadableDB,
row: MessageTypeUnhydrated
): MessageType {
return hydrateMessages(db, [row])[0];
}
export function hydrateMessages(
db: ReadableDB,
unhydratedMessages: Array<MessageTypeUnhydrated>
): Array<MessageType> {
const messagesWithColumnsHydrated = unhydratedMessages.map(
hydrateMessageTableColumns
);
return hydrateMessagesWithAttachments(db, messagesWithColumnsHydrated);
}
export function hydrateMessageTableColumns(
row: MessageTypeUnhydrated
): MessageType {
const {
json,
id,
@@ -29,9 +66,6 @@ export function hydrateMessage(row: MessageTypeUnhydrated): MessageType {
conversationId,
expirationStartTimestamp,
expireTimer,
hasAttachments,
hasFileAttachments,
hasVisualMediaAttachments,
isErased,
isViewOnce,
mentionsMe,
@@ -63,9 +97,6 @@ export function hydrateMessage(row: MessageTypeUnhydrated): MessageType {
conversationId: conversationId || '',
expirationStartTimestamp: dropNull(expirationStartTimestamp),
expireTimer: dropNull(expireTimer) as MessageType['expireTimer'],
hasAttachments: toBoolean(hasAttachments),
hasFileAttachments: toBoolean(hasFileAttachments),
hasVisualMediaAttachments: toBoolean(hasVisualMediaAttachments),
isErased: toBoolean(isErased),
isViewOnce: toBoolean(isViewOnce),
mentionsMe: toBoolean(mentionsMe),
@@ -86,3 +117,299 @@ export function hydrateMessage(row: MessageTypeUnhydrated): MessageType {
unidentifiedDeliveryReceived: toBoolean(unidentifiedDeliveryReceived),
};
}
export function getAttachmentReferencesForMessages(
db: ReadableDB,
messageIds: Array<string>
): Array<MessageAttachmentDBType> {
return batchMultiVarQuery(
db,
messageIds,
(
messageIdBatch: ReadonlyArray<string>,
persistent: boolean
): Array<MessageAttachmentDBType> => {
const [query, params] = sql`
SELECT * FROM message_attachments
WHERE messageId IN (${sqlJoin(messageIdBatch)});
`;
return db
.prepare(query, { persistent })
.all<MessageAttachmentDBType>(params);
}
);
}
function hydrateMessagesWithAttachments(
db: ReadableDB,
messagesWithoutAttachments: Array<MessageType>
): Array<MessageType> {
const attachmentReferencesForAllMessages = getAttachmentReferencesForMessages(
db,
messagesWithoutAttachments.map(msg => msg.id)
);
const attachmentReferencesByMessage = groupBy(
attachmentReferencesForAllMessages,
'messageId'
);
return messagesWithoutAttachments.map(msg => {
const attachmentReferences = attachmentReferencesByMessage[msg.id] ?? [];
if (!attachmentReferences.length) {
return msg;
}
const attachmentsByEditHistoryIndex = groupBy(
attachmentReferences,
'editHistoryIndex'
);
const message = hydrateMessageRootOrRevisionWithAttachments(
msg,
attachmentsByEditHistoryIndex[
ROOT_MESSAGE_ATTACHMENT_EDIT_HISTORY_INDEX
] ?? []
);
if (message.editHistory) {
message.editHistory = message.editHistory.map((editHistory, idx) => {
return hydrateMessageRootOrRevisionWithAttachments(
editHistory,
attachmentsByEditHistoryIndex[idx] ?? []
);
});
}
return message;
});
}
function hydrateMessageRootOrRevisionWithAttachments<
T extends Pick<
MessageAttributesType,
| 'attachments'
| 'bodyAttachment'
| 'contact'
| 'preview'
| 'quote'
| 'sticker'
>,
>(message: T, messageAttachments: Array<MessageAttachmentDBType>): T {
const attachmentsByType = groupBy(
messageAttachments,
'attachmentType'
) as Record<
MessageAttachmentDBType['attachmentType'],
Array<MessageAttachmentDBType>
>;
const standardAttachments = attachmentsByType.attachment ?? [];
const bodyAttachments = attachmentsByType['long-message'] ?? [];
const quoteAttachments = attachmentsByType.quote ?? [];
const previewAttachments = attachmentsByType.preview ?? [];
const contactAttachments = attachmentsByType.contact ?? [];
const stickerAttachment = (attachmentsByType.sticker ?? []).find(
sticker => sticker.orderInMessage === 0
);
const hydratedMessage = structuredClone(message);
if (standardAttachments.length) {
hydratedMessage.attachments = standardAttachments
.sort((a, b) => a.orderInMessage - b.orderInMessage)
.map(convertAttachmentDBFieldsToAttachmentType);
}
if (bodyAttachments[0]) {
hydratedMessage.bodyAttachment = convertAttachmentDBFieldsToAttachmentType(
bodyAttachments[0]
);
}
hydratedMessage.quote?.attachments.forEach((quoteAttachment, idx) => {
const quoteThumbnail = quoteAttachments.find(
attachment => attachment.orderInMessage === idx
);
if (quoteThumbnail) {
// eslint-disable-next-line no-param-reassign
quoteAttachment.thumbnail =
convertAttachmentDBFieldsToAttachmentType(quoteThumbnail);
}
});
hydratedMessage.preview?.forEach((preview, idx) => {
const previewAttachment = previewAttachments.find(
attachment => attachment.orderInMessage === idx
);
if (previewAttachment) {
// eslint-disable-next-line no-param-reassign
preview.image =
convertAttachmentDBFieldsToAttachmentType(previewAttachment);
}
});
hydratedMessage.contact?.forEach((contact, idx) => {
const contactAttachment = contactAttachments.find(
attachment => attachment.orderInMessage === idx
);
if (contactAttachment && contact.avatar) {
// eslint-disable-next-line no-param-reassign
contact.avatar.avatar =
convertAttachmentDBFieldsToAttachmentType(contactAttachment);
}
});
if (hydratedMessage.sticker && stickerAttachment) {
hydratedMessage.sticker.data =
convertAttachmentDBFieldsToAttachmentType(stickerAttachment);
}
return hydratedMessage;
}
function convertAttachmentDBFieldsToAttachmentType(
dbFields: MessageAttachmentDBType
): AttachmentType {
const messageAttachment = shallowDropNull(dbFields);
strictAssert(messageAttachment != null, 'must exist');
const {
clientUuid,
size,
contentType,
plaintextHash,
path,
localKey,
caption,
blurHash,
height,
width,
digest,
iv,
key,
downloadPath,
flags,
fileName,
version,
incrementalMac,
incrementalMacChunkSize: chunkSize,
transitCdnKey: cdnKey,
transitCdnNumber: cdnNumber,
transitCdnUploadTimestamp: uploadTimestamp,
error,
pending,
wasTooBig,
isCorrupted,
backfillError,
storyTextAttachmentJson,
copiedFromQuotedAttachment,
isReencryptableToSameDigest,
localBackupPath,
} = messageAttachment;
const result: AttachmentType = {
clientUuid,
size,
contentType: stringToMIMEType(contentType),
plaintextHash,
path,
localKey,
caption,
blurHash,
height,
width,
digest,
iv,
key,
downloadPath,
localBackupPath,
flags,
fileName,
version,
incrementalMac,
chunkSize,
cdnKey,
cdnNumber,
uploadTimestamp,
pending: convertOptionalIntegerToBoolean(pending),
error: convertOptionalIntegerToBoolean(error),
wasTooBig: convertOptionalIntegerToBoolean(wasTooBig),
copied: convertOptionalIntegerToBoolean(copiedFromQuotedAttachment),
isCorrupted: convertOptionalIntegerToBoolean(isCorrupted),
backfillError: convertOptionalIntegerToBoolean(backfillError),
isReencryptableToSameDigest: convertOptionalIntegerToBoolean(
isReencryptableToSameDigest
),
textAttachment: storyTextAttachmentJson
? jsonToObject(storyTextAttachmentJson)
: undefined,
...(messageAttachment.backupMediaName
? {
backupLocator: {
mediaName: messageAttachment.backupMediaName,
cdnNumber: messageAttachment.backupCdnNumber,
},
}
: {}),
...(messageAttachment.thumbnailPath
? {
thumbnail: {
path: messageAttachment.thumbnailPath,
size: messageAttachment.thumbnailSize ?? 0,
contentType: messageAttachment.thumbnailContentType
? stringToMIMEType(messageAttachment.thumbnailContentType)
: IMAGE_JPEG,
localKey: messageAttachment.thumbnailLocalKey,
version: messageAttachment.thumbnailVersion,
},
}
: {}),
...(messageAttachment.screenshotPath
? {
screenshot: {
path: messageAttachment.screenshotPath,
size: messageAttachment.screenshotSize ?? 0,
contentType: messageAttachment.screenshotContentType
? stringToMIMEType(messageAttachment.screenshotContentType)
: IMAGE_JPEG,
localKey: messageAttachment.screenshotLocalKey,
version: messageAttachment.screenshotVersion,
},
}
: {}),
...(messageAttachment.backupThumbnailPath
? {
thumbnailFromBackup: {
path: messageAttachment.backupThumbnailPath,
size: messageAttachment.backupThumbnailSize ?? 0,
contentType: messageAttachment.backupThumbnailContentType
? stringToMIMEType(messageAttachment.backupThumbnailContentType)
: IMAGE_JPEG,
localKey: messageAttachment.backupThumbnailLocalKey,
version: messageAttachment.backupThumbnailVersion,
},
}
: {}),
};
if (result.isReencryptableToSameDigest === false) {
if (
!messageAttachment.reencryptionIv ||
!messageAttachment.reencryptionKey ||
!messageAttachment.reencryptionDigest
) {
sqlLogger.warn(
'Attachment missing reencryption info despite not being reencryptable'
);
return result;
}
result.reencryptionInfo = {
iv: messageAttachment.reencryptionIv,
key: messageAttachment.reencryptionKey,
digest: messageAttachment.reencryptionDigest,
};
}
return result;
}

42
ts/sql/mainWorker.ts
View File

@@ -3,15 +3,11 @@
import { parentPort } from 'worker_threads';
import type { LoggerType } from '../types/Logging';
import type {
WrappedWorkerRequest,
WrappedWorkerResponse,
WrappedWorkerLogEntry,
} from './main';
import type { WrappedWorkerRequest, WrappedWorkerResponse } from './main';
import type { WritableDB } from './Interface';
import { initialize, DataReader, DataWriter, removeDB } from './Server';
import { SqliteErrorKind, parseSqliteError } from './errors';
import { sqlLogger as logger } from './sqlLogger';
if (!parentPort) {
throw new Error('Must run as a worker thread');
@@ -31,39 +27,6 @@ function respond(seq: number, response?: any) {
port.postMessage(wrappedResponse);
}
const log = (
level: WrappedWorkerLogEntry['level'],
args: Array<unknown>
): void => {
const wrappedResponse: WrappedWorkerResponse = {
type: 'log',
level,
args,
};
port.postMessage(wrappedResponse);
};
const logger: LoggerType = {
fatal(...args: Array<unknown>) {
log('fatal', args);
},
error(...args: Array<unknown>) {
log('error', args);
},
warn(...args: Array<unknown>) {
log('warn', args);
},
info(...args: Array<unknown>) {
log('info', args);
},
debug(...args: Array<unknown>) {
log('debug', args);
},
trace(...args: Array<unknown>) {
log('trace', args);
},
};
let db: WritableDB | undefined;
let isPrimary = false;
let isRemoved = false;
@@ -79,7 +42,6 @@ const onMessage = (
db = initialize({
...request.options,
isPrimary,
logger,
});
respond(seq, undefined);

111
ts/sql/migrations/1360-attachments.ts Normal file
View File

@@ -0,0 +1,111 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import type { LoggerType } from '../../types/Logging';
import type { WritableDB } from '../Interface';
export const version = 1360;
export function updateToSchemaVersion1360(
currentVersion: number,
db: WritableDB,
logger: LoggerType
): void {
if (currentVersion >= 1360) {
return;
}
db.transaction(() => {
db.exec(`
DROP TABLE IF EXISTS message_attachments;
`);
db.exec(`
CREATE TABLE message_attachments (
messageId TEXT NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
-- For editHistoryIndex to be part of the primary key, it cannot be NULL in strict tables.
-- For that reason, we use a value of -1 to indicate that it is the root message (not in editHistory)
editHistoryIndex INTEGER NOT NULL,
attachmentType TEXT NOT NULL, -- 'long-message' | 'quote' | 'attachment' | 'preview' | 'contact' | 'sticker'
orderInMessage INTEGER NOT NULL,
conversationId TEXT NOT NULL,
sentAt INTEGER NOT NULL,
clientUuid TEXT,
size INTEGER NOT NULL,
contentType TEXT NOT NULL,
path TEXT,
plaintextHash TEXT,
localKey TEXT,
caption TEXT,
fileName TEXT,
blurHash TEXT,
height INTEGER,
width INTEGER,
digest TEXT,
key TEXT,
iv TEXT,
downloadPath TEXT,
version INTEGER,
incrementalMac TEXT,
incrementalMacChunkSize INTEGER,
transitCdnKey TEXT,
transitCdnNumber INTEGER,
transitCdnUploadTimestamp INTEGER,
backupMediaName TEXT,
backupCdnNumber INTEGER,
isReencryptableToSameDigest INTEGER,
reencryptionIv TEXT,
reencryptionKey TEXT,
reencryptionDigest TEXT,
thumbnailPath TEXT,
thumbnailSize INTEGER,
thumbnailContentType TEXT,
thumbnailLocalKey TEXT,
thumbnailVersion INTEGER,
screenshotPath TEXT,
screenshotSize INTEGER,
screenshotContentType TEXT,
screenshotLocalKey TEXT,
screenshotVersion INTEGER,
backupThumbnailPath TEXT,
backupThumbnailSize INTEGER,
backupThumbnailContentType TEXT,
backupThumbnailLocalKey TEXT,
backupThumbnailVersion INTEGER,
storyTextAttachmentJson TEXT,
localBackupPath TEXT,
flags INTEGER,
error INTEGER,
wasTooBig INTEGER,
isCorrupted INTEGER,
copiedFromQuotedAttachment INTEGER,
pending INTEGER,
backfillError INTEGER,
PRIMARY KEY (messageId, editHistoryIndex, attachmentType, orderInMessage)
) STRICT;
`);
db.exec(
'CREATE INDEX message_attachments_messageId ON message_attachments (messageId);'
);
db.exec(
'CREATE INDEX message_attachments_plaintextHash ON message_attachments (plaintextHash);'
);
db.exec(
'CREATE INDEX message_attachments_path ON message_attachments (path);'
);
db.exec(
'CREATE INDEX message_attachments_all_thumbnailPath ON message_attachments (thumbnailPath);'
);
db.exec(
'CREATE INDEX message_attachments_all_screenshotPath ON message_attachments (screenshotPath);'
);
db.exec(
'CREATE INDEX message_attachments_all_backupThumbnailPath ON message_attachments (backupThumbnailPath);'
);
db.pragma('user_version = 1360');
})();
logger.info('updateToSchemaVersion1360: success!');
}

6
ts/sql/migrations/index.ts
View File

@@ -110,10 +110,11 @@ import { updateToSchemaVersion1310 } from './1310-muted-fixup';
import { updateToSchemaVersion1320 } from './1320-unprocessed-received-at-date';
import { updateToSchemaVersion1330 } from './1330-sync-tasks-type-index';
import { updateToSchemaVersion1340 } from './1340-recent-gifs';
import { updateToSchemaVersion1350 } from './1350-notification-profiles';
import {
updateToSchemaVersion1350,
updateToSchemaVersion1360,
version as MAX_VERSION,
} from './1350-notification-profiles';
} from './1360-attachments';
import { DataWriter } from '../Server';
@@ -2102,6 +2103,7 @@ export const SCHEMA_VERSIONS = [
updateToSchemaVersion1330,
updateToSchemaVersion1340,
updateToSchemaVersion1350,
updateToSchemaVersion1360,
];
export class DBVersionFromFutureError extends Error {

45
ts/sql/sqlLogger.ts Normal file
View File

@@ -0,0 +1,45 @@
// Copyright 2025 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { parentPort } from 'worker_threads';
import type { LoggerType } from '../types/Logging';
import type { WrappedWorkerLogEntry, WrappedWorkerResponse } from './main';
import { consoleLogger } from '../util/consoleLogger';
import { strictAssert } from '../util/assert';
const log = (
level: WrappedWorkerLogEntry['level'],
args: Array<unknown>
): void => {
if (parentPort) {
const wrappedResponse: WrappedWorkerResponse = {
type: 'log',
level,
args,
};
parentPort.postMessage(wrappedResponse);
} else {
strictAssert(process.env.NODE_ENV === 'test', 'must be test environment');
consoleLogger[level](...args);
}
};
export const sqlLogger: LoggerType = {
fatal(...args: Array<unknown>) {
log('fatal', args);
},
error(...args: Array<unknown>) {
log('error', args);
},
warn(...args: Array<unknown>) {
log('warn', args);
},
info(...args: Array<unknown>) {
log('info', args);
},
debug(...args: Array<unknown>) {
log('debug', args);
},
trace(...args: Array<unknown>) {
log('trace', args);
},
};

24
ts/sql/util.ts
View File

@@ -418,3 +418,27 @@ export class TableIterator<ObjectType extends { id: string }> {
}
}
}
export function convertOptionalIntegerToBoolean(
optionalInteger?: number
): boolean | undefined {
if (optionalInteger === 1) {
return true;
}
if (optionalInteger === 0) {
return false;
}
return undefined;
}
export function convertOptionalBooleanToNullableInteger(
optionalBoolean?: boolean
): 1 | 0 | null {
if (optionalBoolean === true) {
return 1;
}
if (optionalBoolean === false) {
return 0;
}
return null;
}