diff --git a/ts/jobs/AttachmentLocalBackupManager.preload.ts b/ts/jobs/AttachmentLocalBackupManager.preload.ts index 88fbd2f357..947d6cbaa4 100644 --- a/ts/jobs/AttachmentLocalBackupManager.preload.ts +++ b/ts/jobs/AttachmentLocalBackupManager.preload.ts @@ -9,8 +9,8 @@ import { mkdir, rename, } from 'node:fs/promises'; +import { exists } from 'fs-extra'; -import { createLogger } from '../logging/log.std.js'; import { getAbsoluteAttachmentPath as doGetAbsoluteAttachmentPath, getAbsoluteTempPath as doGetAbsoluteTempPath, @@ -25,8 +25,6 @@ import { redactGenericText } from '../util/privacy.node.js'; import type { CoreAttachmentLocalBackupJobType } from '../types/AttachmentBackup.std.js'; -const log = createLogger('AttachmentLocalBackupManager'); - export class AttachmentPermanentlyMissingError extends Error {} type RunAttachmentBackupJobDependenciesType = { @@ -50,11 +48,6 @@ export async function runAttachmentBackupJob( decryptAttachmentV2ToSink, } ): Promise { - const jobIdForLogging = getJobIdForLogging(job); - const logId = `AttachmentLocalBackupManager.runAttachmentBackupJob(${jobIdForLogging})`; - - log.info(`${logId}: starting...`); - const { isPlaintextExport, mediaName } = job; const { contentType, fileName, localKey, path, size } = job.data; @@ -97,6 +90,11 @@ export async function runAttachmentBackupJob( outFileStream ); } else { + if (await exists(destinationLocalBackupFilePath)) { + // File already backed up + return; + } + // File is already encrypted with localKey, so we just copy it to the backup dir const tempPath = dependencies.getAbsoluteTempPath(createName()); @@ -112,8 +110,6 @@ export async function runAttachmentBackupJob( ); await rename(tempPath, destinationLocalBackupFilePath); } - - log.info(`${logId}: complete!`); } function getExtension( diff --git a/ts/services/backups/index.preload.ts b/ts/services/backups/index.preload.ts index 62211080cb..4e2281cd01 100644 --- a/ts/services/backups/index.preload.ts +++ b/ts/services/backups/index.preload.ts @@ -6,7 +6,7 @@ import { PassThrough } from 'node:stream'; import type { Readable, Writable } from 'node:stream'; import { createReadStream, createWriteStream } from 'node:fs'; import { mkdir, rm, stat, unlink, writeFile } from 'node:fs/promises'; -import fsExtra from 'fs-extra'; +import fsExtra, { exists } from 'fs-extra'; import { join } from 'node:path'; import { createGzip, createGunzip } from 'node:zlib'; import { createCipheriv, createHmac, randomBytes } from 'node:crypto'; @@ -48,7 +48,7 @@ import { } from '../../types/backups.node.js'; import { HTTPError } from '../../types/HTTPError.std.js'; import { constantTimeEqual } from '../../Crypto.node.js'; -import { measureSize, safeUnlink } from '../../AttachmentCrypto.node.js'; +import { measureSize } from '../../AttachmentCrypto.node.js'; import { signalProtocolStore } from '../../SignalProtocolStore.preload.js'; import { isTestOrMockEnvironment } from '../../environment.std.js'; import { runStorageServiceSyncJob } from '../storage.preload.js'; @@ -90,9 +90,10 @@ import { verifyLocalBackupMetadata, writeLocalBackupFilesList, readLocalBackupFilesList, + pruneLocalBackups, validateLocalBackupStructure, - getAllPathsInLocalBackupFilesDirectory, getLocalBackupFilesDirectory, + getLocalBackupSnapshotDirectory, } from './util/localBackup.node.js'; import { AttachmentPermanentlyMissingError, @@ -130,6 +131,7 @@ const IV_LENGTH = 16; const BACKUP_REFRESH_INTERVAL = 24 * HOUR; const MIMINUM_DISK_SPACE_FOR_LOCAL_EXPORT = 200 * MEBIBYTE; +const LOCAL_BACKUP_SNAPSHOTS_TO_KEEP = 2; export type DownloadOptionsType = Readonly<{ onProgress?: ( @@ -371,16 +373,15 @@ export class BackupsService { await mkdir(filesDir, { recursive: true }); - const attachmentPathsWrittenBeforeThisBackup = - await getAllPathsInLocalBackupFilesDirectory({ - backupsBaseDir: options.backupsBaseDir, - }); - - const snapshotDir = join( + const snapshotDir = getLocalBackupSnapshotDirectory( options.backupsBaseDir, - `signal-backup-${getTimestampForFolder()}` + Date.now() ); + if (await exists(snapshotDir)) { + throw new Error('snapshotDir already exists'); + } + try { await mkdir(snapshotDir, { recursive: true }); @@ -389,7 +390,7 @@ export class BackupsService { abortSignal: options.abortSignal, }); - log.info('exportLocalBackup: writing local backup files list'); + fnLog.info('writing local backup files list'); const filesWritten = await writeLocalBackupFilesList({ snapshotDir, mediaNames: exportResult.mediaNames, @@ -400,7 +401,7 @@ export class BackupsService { 'exportBackup: Local backup files proto must match files written' ); - log.info('exportLocalBackup: writing metadata'); + fnLog.info('writing metadata'); const metadataArgs = { snapshotDir, backupId: getBackupId(), @@ -416,31 +417,41 @@ export class BackupsService { abortSignal: options.abortSignal, }); + try { + await pruneLocalBackups({ + backupsBaseDir: options.backupsBaseDir, + numSnapshotsToKeep: LOCAL_BACKUP_SNAPSHOTS_TO_KEEP, + }); + } catch (error) { + fnLog.warn( + 'failed to prune old local backups', + Errors.toLogFormat(error) + ); + } + return { ...exportResult, snapshotDir }; } catch (e) { if (options.abortSignal.aborted) { - log.warn('exportLocalBackup aborted', Errors.toLogFormat(e)); + fnLog.warn('aborted', Errors.toLogFormat(e)); } else { - log.error('exportLocalBackup encountered error', Errors.toLogFormat(e)); + fnLog.error('encountered error', Errors.toLogFormat(e)); } - log.info('Deleting snapshot directory'); + fnLog.info('Deleting just-created snapshot directory'); await rm(snapshotDir, { recursive: true, force: true }); - log.info('Deleted Snapshot directory'); + fnLog.info('Deleted just-created directory'); - const attachmentPathsAfterBackup = - await getAllPathsInLocalBackupFilesDirectory({ + // Prune to remove any files which may have been written before the error occurred + try { + await pruneLocalBackups({ backupsBaseDir: options.backupsBaseDir, + numSnapshotsToKeep: LOCAL_BACKUP_SNAPSHOTS_TO_KEEP, }); - - const pathsAdded = new Set(attachmentPathsAfterBackup).difference( - new Set(attachmentPathsWrittenBeforeThisBackup) - ); - - if (pathsAdded.size > 0) { - log.info(`Deleting ${pathsAdded.size} newly written files`); - await Promise.all([...pathsAdded].map(safeUnlink)); - log.info(`Deleted ${pathsAdded.size} files`); + } catch (error) { + fnLog.warn( + 'failed to prune local backups after export error', + Errors.toLogFormat(error) + ); } throw e; @@ -645,10 +656,10 @@ export class BackupsService { if (isOnline()) { await this.#waitForEmptyQueues('backups.exportPlaintext'); } else { - fnLog.info('exportPlaintext: Offline; skipping wait for empty queues'); + fnLog.info('offline; skipping wait for empty queues'); } - fnLog.info('exportPlaintext: starting...'); + fnLog.info('starting...'); await mkdir(exportDir, { recursive: true }); @@ -660,7 +671,7 @@ export class BackupsService { } ); - fnLog.info('exportPlaintext: writing metadata'); + fnLog.info('writing metadata'); const metadataPath = join(exportDir, 'metadata.json'); await writeFile( diff --git a/ts/services/backups/util/localBackup.node.ts b/ts/services/backups/util/localBackup.node.ts index 1cc1b90f35..6d4b43cb94 100644 --- a/ts/services/backups/util/localBackup.node.ts +++ b/ts/services/backups/util/localBackup.node.ts @@ -2,8 +2,8 @@ // SPDX-License-Identifier: AGPL-3.0-only import { randomBytes } from 'node:crypto'; -import { dirname, join } from 'node:path'; -import { readdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { basename, dirname, join } from 'node:path'; +import { readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { createReadStream, createWriteStream } from 'node:fs'; import { Readable, Writable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; @@ -19,9 +19,25 @@ import { LOCAL_BACKUP_VERSION, LOCAL_BACKUP_BACKUP_ID_IV_LENGTH, } from '../constants.std.js'; +import { getTimestampForFolder } from '../../../util/timestamp.std.js'; +import { isPathInside } from '../../../util/isPathInside.node.js'; const log = createLogger('localBackup'); +const LOCAL_BACKUP_SNAPSHOT_DIR_PREFIX = 'signal-backup-'; +const LOCAL_BACKUP_SNAPSHOT_DIR_PATTERN = + /^signal-backup-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}$/; + +export function getLocalBackupSnapshotDirectory( + backupsBaseDir: string, + timestamp: number +): string { + return join( + backupsBaseDir, + `${LOCAL_BACKUP_SNAPSHOT_DIR_PREFIX}${getTimestampForFolder(timestamp)}` + ); +} + export function getLocalBackupFilesDirectory({ backupsBaseDir, }: { @@ -45,6 +61,98 @@ export async function getAllPathsInLocalBackupFilesDirectory({ .map(entry => join(entry.parentPath, entry.name)); } +async function getSortedLocalBackupSnapshotDirs({ + backupsBaseDir, +}: { + backupsBaseDir: string; +}): Promise> { + const entries = await readdir(backupsBaseDir, { withFileTypes: true }); + const snapshotDirs = new Array(); + + for (const entry of entries) { + if (!entry.isDirectory()) { + continue; + } + if (!entry.name.startsWith(LOCAL_BACKUP_SNAPSHOT_DIR_PREFIX)) { + continue; + } + + if (LOCAL_BACKUP_SNAPSHOT_DIR_PATTERN.test(entry.name)) { + snapshotDirs.push(join(backupsBaseDir, entry.name)); + } + } + + return snapshotDirs.sort().reverse(); +} + +export async function pruneLocalBackups({ + backupsBaseDir, + numSnapshotsToKeep, +}: { + backupsBaseDir: string; + numSnapshotsToKeep: number; +}): Promise { + const fnLog = log.child('pruneLocalBackups'); + + const snapshotDirs = await getSortedLocalBackupSnapshotDirs({ + backupsBaseDir, + }); + + const snapshotDirsToKeep = snapshotDirs.slice(0, numSnapshotsToKeep); + const snapshotDirsToDelete = snapshotDirs.slice(numSnapshotsToKeep); + + if (snapshotDirsToDelete.length > 0) { + if (snapshotDirsToDelete.length === 1) { + fnLog.info('pruning one snapshot'); + } else { + fnLog.error(`pruning ${snapshotDirsToDelete.length} snapshots`); + } + + await Promise.all( + snapshotDirsToDelete.map(snapshotDir => { + strictAssert( + isPathInside(snapshotDir, backupsBaseDir), + 'ensure snapshot dir inside backups dir' + ); + return rm(snapshotDir, { recursive: true, force: true }); + }) + ); + } + + const referencedMediaNames = new Set(); + for (const snapshotDir of snapshotDirsToKeep) { + // eslint-disable-next-line no-await-in-loop + const mediaNames = await readLocalBackupFilesList(snapshotDir); + for (const mediaName of mediaNames) { + referencedMediaNames.add(mediaName); + } + } + + const allMediaPaths = await getAllPathsInLocalBackupFilesDirectory({ + backupsBaseDir, + }); + + const mediaPathsToDelete = allMediaPaths.filter( + mediaPath => !referencedMediaNames.has(basename(mediaPath)) + ); + + if (mediaPathsToDelete.length > 0) { + fnLog.info( + `Deleting ${mediaPathsToDelete.length} files no longer referenced` + ); + const filesDirectory = getLocalBackupFilesDirectory({ backupsBaseDir }); + await Promise.all( + mediaPathsToDelete.map(mediaPath => { + strictAssert( + isPathInside(mediaPath, filesDirectory), + 'ensure mediaPath inside backup files dir' + ); + return rm(mediaPath, { force: true }); + }) + ); + } +} + export function getLocalBackupDirectoryForMediaName({ backupsBaseDir, mediaName, diff --git a/ts/test-node/services/backups/localBackup_test.node.ts b/ts/test-node/services/backups/localBackup_test.node.ts new file mode 100644 index 0000000000..7d82ee7eaa --- /dev/null +++ b/ts/test-node/services/backups/localBackup_test.node.ts @@ -0,0 +1,116 @@ +// Copyright 2026 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { assert } from 'chai'; +import path from 'node:path'; +import { tmpdir } from 'node:os'; +import { mkdir, mkdtemp, rm, readdir, writeFile } from 'node:fs/promises'; +import { exists } from 'fs-extra'; + +import { + getLocalBackupFilesDirectory, + getLocalBackupPathForMediaName, + pruneLocalBackups, + writeLocalBackupFilesList, +} from '../../../services/backups/util/localBackup.node.js'; + +async function createSnapshot({ + backupsBaseDir, + snapshotName, + mediaNames, +}: { + backupsBaseDir: string; + snapshotName: string; + mediaNames: Array; +}): Promise { + const snapshotDir = path.join(backupsBaseDir, snapshotName); + await mkdir(snapshotDir, { recursive: true }); + await writeLocalBackupFilesList({ snapshotDir, mediaNames }); + + await Promise.all( + mediaNames.map(async mediaName => { + const mediaPath = getLocalBackupPathForMediaName({ + backupsBaseDir, + mediaName, + }); + await mkdir(path.dirname(mediaPath), { recursive: true }); + await writeFile(mediaPath, mediaName); + }) + ); + + return snapshotDir; +} + +describe('localBackup/pruneLocalBackups', () => { + let backupsBaseDir: string; + + beforeEach(async () => { + backupsBaseDir = await mkdtemp( + path.join(tmpdir(), 'signal-local-backup-prune-') + ); + }); + + afterEach(async () => { + await rm(backupsBaseDir, { recursive: true, force: true }); + }); + + async function listFiles() { + return ( + await readdir(getLocalBackupFilesDirectory({ backupsBaseDir }), { + withFileTypes: true, + recursive: true, + }) + ) + .filter(entry => entry.isFile()) + .map(entry => entry.name); + } + + it('keeps two newest snapshots and deletes unreferenced media files', async () => { + const snapshotOld = await createSnapshot({ + backupsBaseDir, + snapshotName: 'signal-backup-2026-03-08-12-00-01', + mediaNames: ['aa-old', 'dd-shared'], + }); + const snapshotMiddle = await createSnapshot({ + backupsBaseDir, + snapshotName: 'signal-backup-2026-03-08-12-00-02', + mediaNames: ['bb-middle', 'dd-shared'], + }); + const snapshotNewest = await createSnapshot({ + backupsBaseDir, + snapshotName: 'signal-backup-2026-03-08-12-00-03', + mediaNames: ['cc-newest', 'dd-shared'], + }); + + const orphanMediaPath = getLocalBackupPathForMediaName({ + backupsBaseDir, + mediaName: 'ee-orphan', + }); + + await mkdir(path.dirname(orphanMediaPath), { recursive: true }); + await writeFile(orphanMediaPath, 'orphan'); + + assert.sameDeepMembers(await listFiles(), [ + 'aa-old', + 'bb-middle', + 'cc-newest', + 'dd-shared', + 'ee-orphan', + ]); + + await pruneLocalBackups({ + backupsBaseDir, + numSnapshotsToKeep: 2, + }); + + assert.isTrue(await exists(snapshotNewest)); + assert.isTrue(await exists(snapshotMiddle)); + assert.isFalse(await exists(snapshotOld)); + + assert.sameDeepMembers(await listFiles(), [ + 'bb-middle', + 'cc-newest', + 'dd-shared', + ]); + }); +}); diff --git a/ts/util/timestamp.std.ts b/ts/util/timestamp.std.ts index 6730eac9f3..f624d73b1e 100644 --- a/ts/util/timestamp.std.ts +++ b/ts/util/timestamp.std.ts @@ -50,7 +50,7 @@ export function addLeadingZero(amount: number): string { return amount.toString(); } -export function getTimestampForFolder(): string { - const date = new Date(); - return `${date.getFullYear()}-${addLeadingZero(date.getMonth() + 1)}-${addLeadingZero(date.getDate())}-${addLeadingZero(date.getHours())}-${addLeadingZero(date.getMinutes())}-${addLeadingZero(date.getSeconds())}`; +export function getTimestampForFolder(timestamp?: number): string { + const date = timestamp != null ? new Date(timestamp) : new Date(); + return `${date.getUTCFullYear()}-${addLeadingZero(date.getUTCMonth() + 1)}-${addLeadingZero(date.getUTCDate())}-${addLeadingZero(date.getUTCHours())}-${addLeadingZero(date.getUTCMinutes())}-${addLeadingZero(date.getUTCSeconds())}`; }