From eb1cf8d62fb260baaf0c6a400f6afdd6400caa18 Mon Sep 17 00:00:00 2001 From: Michelle Tang Date: Fri, 4 Apr 2025 11:36:17 -0400 Subject: [PATCH] Add chat folder support to storage service. --- .../database/ChatFolderTablesTest.kt | 112 ++++- .../securesms/backup/v2/ArchiveErrorCases.kt | 4 + .../v2/processor/ChatFolderProcessor.kt | 20 +- .../app/chats/folders/ChatFolderId.kt | 31 ++ .../app/chats/folders/ChatFolderRecord.kt | 9 +- .../chats/folders/ChatFoldersRepository.kt | 14 +- .../chats/folders/CreateFoldersFragment.kt | 2 +- .../securesms/database/ChatFolderTables.kt | 396 +++++++++++++++--- .../helpers/SignalDatabaseMigrations.kt | 6 +- ...V270_FixChatFolderColumnsForStorageSync.kt | 38 ++ .../securesms/jobs/JobManagerFactories.java | 2 + .../securesms/jobs/StorageForcePushJob.kt | 26 ++ .../securesms/jobs/StorageSyncJob.kt | 25 +- .../migrations/ApplicationMigrations.java | 7 +- .../migrations/SyncChatFoldersMigrationJob.kt | 55 +++ .../storage/ChatFolderRecordProcessor.kt | 104 +++++ .../securesms/storage/StorageSyncModels.kt | 56 ++- .../storage/StorageSyncValidations.java | 8 + app/src/main/protowire/Backup.proto | 1 + .../securesms/StorageServicePlugin.kt | 3 + .../storage/ChatFolderRecordProcessorTest.kt | 216 ++++++++++ .../api/storage/SignalChatFolderRecord.kt | 27 ++ .../api/storage/SignalStorageRecord.kt | 2 +- .../signalservice/api/storage/StorageId.java | 4 + .../api/storage/StorageRecordConverters.kt | 9 + .../src/main/protowire/StorageService.proto | 38 +- 26 files changed, 1131 insertions(+), 84 deletions(-) create mode 100644 app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFolderId.kt create mode 100644 app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V270_FixChatFolderColumnsForStorageSync.kt create mode 100644 app/src/main/java/org/thoughtcrime/securesms/migrations/SyncChatFoldersMigrationJob.kt create mode 100644 app/src/main/java/org/thoughtcrime/securesms/storage/ChatFolderRecordProcessor.kt create mode 100644 app/src/test/java/org/thoughtcrime/securesms/storage/ChatFolderRecordProcessorTest.kt create mode 100644 libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalChatFolderRecord.kt diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/database/ChatFolderTablesTest.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/database/ChatFolderTablesTest.kt index 66aebb648c..993ce2fb25 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/database/ChatFolderTablesTest.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/database/ChatFolderTablesTest.kt @@ -6,16 +6,26 @@ package org.thoughtcrime.securesms.database import androidx.test.ext.junit.runners.AndroidJUnit4 +import okio.ByteString.Companion.toByteString import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotEquals import org.junit.Before import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith import org.signal.core.util.deleteAll +import org.thoughtcrime.securesms.components.settings.app.chats.folders.ChatFolderId import org.thoughtcrime.securesms.components.settings.app.chats.folders.ChatFolderRecord import org.thoughtcrime.securesms.recipients.Recipient import org.thoughtcrime.securesms.recipients.RecipientId +import org.thoughtcrime.securesms.storage.StorageSyncHelper import org.thoughtcrime.securesms.testing.SignalActivityRule +import org.whispersystems.signalservice.api.push.ServiceId.ACI +import org.whispersystems.signalservice.api.storage.SignalChatFolderRecord +import org.whispersystems.signalservice.api.storage.StorageId +import org.whispersystems.signalservice.api.util.UuidUtil +import java.util.UUID +import org.whispersystems.signalservice.internal.storage.protos.ChatFolderRecord as RemoteChatFolderRecord @RunWith(AndroidJUnit4::class) class ChatFolderTablesTest { @@ -31,15 +41,19 @@ class ChatFolderTablesTest { private lateinit var folder2: ChatFolderRecord private lateinit var folder3: ChatFolderRecord + private lateinit var recipientIds: List + private var aliceThread: Long = 0 private var bobThread: Long = 0 private var charlieThread: Long = 0 @Before fun setUp() { - alice = harness.others[1] - bob = harness.others[2] - charlie = harness.others[3] + recipientIds = createRecipients(5) + + alice = recipientIds[0] + bob = recipientIds[1] + charlie = recipientIds[2] aliceThread = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(alice)) bobThread = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(bob)) @@ -48,32 +62,40 @@ class ChatFolderTablesTest { folder1 = ChatFolderRecord( id = 2, name = "folder1", - position = 1, + position = 0, includedChats = listOf(aliceThread, bobThread), excludedChats = listOf(charlieThread), showUnread = true, showMutedChats = true, showIndividualChats = true, - folderType = ChatFolderRecord.FolderType.CUSTOM + folderType = ChatFolderRecord.FolderType.CUSTOM, + chatFolderId = ChatFolderId.generate(), + storageServiceId = StorageId.forChatFolder(byteArrayOf(1, 2, 3)) ) folder2 = ChatFolderRecord( name = "folder2", + position = 2, includedChats = listOf(bobThread), showUnread = true, showMutedChats = true, showIndividualChats = true, - folderType = ChatFolderRecord.FolderType.INDIVIDUAL + folderType = ChatFolderRecord.FolderType.INDIVIDUAL, + chatFolderId = ChatFolderId.generate(), + storageServiceId = StorageId.forChatFolder(byteArrayOf(2, 3, 4)) ) folder3 = ChatFolderRecord( name = "folder3", + position = 3, includedChats = listOf(bobThread), excludedChats = listOf(aliceThread, charlieThread), showUnread = true, showMutedChats = true, showGroupChats = true, - folderType = ChatFolderRecord.FolderType.GROUP + folderType = ChatFolderRecord.FolderType.GROUP, + chatFolderId = ChatFolderId.generate(), + storageServiceId = StorageId.forChatFolder(byteArrayOf(3, 4, 5)) ) SignalDatabase.chatFolders.writableDatabase.deleteAll(ChatFolderTables.ChatFolderTable.TABLE_NAME) @@ -83,7 +105,7 @@ class ChatFolderTablesTest { @Test fun givenChatFolder_whenIGetFolder_thenIExpectFolderWithChats() { SignalDatabase.chatFolders.createFolder(folder1) - val actualFolders = SignalDatabase.chatFolders.getChatFolders() + val actualFolders = SignalDatabase.chatFolders.getCurrentChatFolders() assertEquals(listOf(folder1), actualFolders) } @@ -91,7 +113,7 @@ class ChatFolderTablesTest { @Test fun givenChatFolder_whenIUpdateFolder_thenIExpectUpdatedFolderWithChats() { SignalDatabase.chatFolders.createFolder(folder2) - val folder = SignalDatabase.chatFolders.getChatFolders().first() + val folder = SignalDatabase.chatFolders.getCurrentChatFolders().first() val updatedFolder = folder.copy( name = "updatedFolder2", position = 1, @@ -100,7 +122,7 @@ class ChatFolderTablesTest { ) SignalDatabase.chatFolders.updateFolder(updatedFolder) - val actualFolder = SignalDatabase.chatFolders.getChatFolders().first() + val actualFolder = SignalDatabase.chatFolders.getCurrentChatFolders().first() assertEquals(updatedFolder, actualFolder) } @@ -109,11 +131,77 @@ class ChatFolderTablesTest { fun givenADeletedChatFolder_whenIGetFolders_thenIExpectAListWithoutThatFolder() { SignalDatabase.chatFolders.createFolder(folder1) SignalDatabase.chatFolders.createFolder(folder2) - val folders = SignalDatabase.chatFolders.getChatFolders() + val folders = SignalDatabase.chatFolders.getCurrentChatFolders() SignalDatabase.chatFolders.deleteChatFolder(folders.last()) - val actualFolders = SignalDatabase.chatFolders.getChatFolders() + val actualFolders = SignalDatabase.chatFolders.getCurrentChatFolders() assertEquals(listOf(folder1), actualFolders) } + + @Test + fun givenChatFolders_whenIUpdateTheirStorageSyncIds_thenIExpectAnUpdatedList() { + val existingMap = SignalDatabase.chatFolders.getStorageSyncIdsMap() + existingMap.forEach { (id, _) -> + SignalDatabase.chatFolders.applyStorageIdUpdate(id, StorageId.forChatFolder(StorageSyncHelper.generateKey())) + } + val updatedMap = SignalDatabase.chatFolders.getStorageSyncIdsMap() + + existingMap.forEach { (id, storageId) -> + assertNotEquals(storageId, updatedMap[id]) + } + } + + @Test + fun givenARemoteFolder_whenIInsertLocally_thenIExpectAListWithThatFolder() { + val remoteRecord = + SignalChatFolderRecord( + folder1.storageServiceId!!, + RemoteChatFolderRecord( + identifier = UuidUtil.toByteArray(folder1.chatFolderId.uuid).toByteString(), + name = folder1.name, + position = folder1.position, + showOnlyUnread = folder1.showUnread, + showMutedChats = folder1.showMutedChats, + includeAllIndividualChats = folder1.showIndividualChats, + includeAllGroupChats = folder1.showGroupChats, + folderType = RemoteChatFolderRecord.FolderType.CUSTOM, + deletedAtTimestampMs = folder1.deletedTimestampMs, + includedRecipients = listOf( + RemoteChatFolderRecord.Recipient(RemoteChatFolderRecord.Recipient.Contact(Recipient.resolved(alice).serviceId.get().toString())), + RemoteChatFolderRecord.Recipient(RemoteChatFolderRecord.Recipient.Contact(Recipient.resolved(bob).serviceId.get().toString())) + ), + excludedRecipients = listOf( + RemoteChatFolderRecord.Recipient(RemoteChatFolderRecord.Recipient.Contact(Recipient.resolved(charlie).serviceId.get().toString())) + ) + + ) + ) + + SignalDatabase.chatFolders.insertChatFolderFromStorageSync(remoteRecord) + val actualFolders = SignalDatabase.chatFolders.getCurrentChatFolders() + + assertEquals(listOf(folder1), actualFolders) + } + + @Test + fun givenADeletedChatFolder_whenIGetPositions_thenIExpectPositionsToStillBeConsecutive() { + SignalDatabase.chatFolders.createFolder(folder1) + SignalDatabase.chatFolders.createFolder(folder2) + SignalDatabase.chatFolders.createFolder(folder3) + + val folders = SignalDatabase.chatFolders.getCurrentChatFolders() + SignalDatabase.chatFolders.deleteChatFolder(folders[1]) + + val actualFolders = SignalDatabase.chatFolders.getCurrentChatFolders() + actualFolders.forEachIndexed { index, folder -> + assertEquals(folder.position, index) + } + } + + private fun createRecipients(count: Int): List { + return (1..count).map { + SignalDatabase.recipients.getOrInsertFromServiceId(ACI.from(UUID.randomUUID())) + } + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/ArchiveErrorCases.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/ArchiveErrorCases.kt index 5459887501..09ba782afc 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/ArchiveErrorCases.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/ArchiveErrorCases.kt @@ -183,6 +183,10 @@ object ImportSkips { return log(sentTimestamp, "Failed to find a threadId for the provided chatId. ChatId in backup: $chatId") } + fun chatFolderIdNotFound(): String { + return log(0, "Failed to parse chatFolderId for the provided chat folder.") + } + private fun log(sentTimestamp: Long, message: String): String { return "[SKIP][$sentTimestamp] $message" } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatFolderProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatFolderProcessor.kt index f228644737..a5fa9c6767 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatFolderProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatFolderProcessor.kt @@ -6,10 +6,13 @@ package org.thoughtcrime.securesms.backup.v2.processor import androidx.core.content.contentValuesOf +import okio.ByteString.Companion.toByteString +import org.signal.core.util.Base64 import org.signal.core.util.SqlUtil import org.signal.core.util.insertInto import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.backup.v2.ExportState +import org.thoughtcrime.securesms.backup.v2.ImportSkips import org.thoughtcrime.securesms.backup.v2.ImportState import org.thoughtcrime.securesms.backup.v2.proto.ChatFolder import org.thoughtcrime.securesms.backup.v2.proto.Frame @@ -19,6 +22,8 @@ import org.thoughtcrime.securesms.database.ChatFolderTables.ChatFolderMembership import org.thoughtcrime.securesms.database.ChatFolderTables.ChatFolderTable import org.thoughtcrime.securesms.database.ChatFolderTables.MembershipType import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.storage.StorageSyncHelper +import org.whispersystems.signalservice.api.util.UuidUtil import org.thoughtcrime.securesms.backup.v2.proto.ChatFolder as ChatFolderProto /** @@ -31,7 +36,7 @@ object ChatFolderProcessor { fun export(db: SignalDatabase, exportState: ExportState, emitter: BackupFrameEmitter) { val folders = db .chatFoldersTable - .getChatFolders() + .getCurrentChatFolders() .sortedBy { it.position } if (folders.isEmpty()) { @@ -66,6 +71,12 @@ object ChatFolderProcessor { } fun import(chatFolder: ChatFolderProto, importState: ImportState) { + val chatFolderUuid = UuidUtil.parseOrNull(chatFolder.id) + if (chatFolderUuid == null) { + ImportSkips.chatFolderIdNotFound() + return + } + val chatFolderId = SignalDatabase .writableDatabase .insertInto(ChatFolderTable.TABLE_NAME) @@ -76,7 +87,9 @@ object ChatFolderProcessor { ChatFolderTable.SHOW_MUTED to chatFolder.showMutedChats, ChatFolderTable.SHOW_INDIVIDUAL to chatFolder.includeAllIndividualChats, ChatFolderTable.SHOW_GROUPS to chatFolder.includeAllGroupChats, - ChatFolderTable.FOLDER_TYPE to chatFolder.folderType.toLocal().value + ChatFolderTable.FOLDER_TYPE to chatFolder.folderType.toLocal().value, + ChatFolderTable.CHAT_FOLDER_ID to chatFolderUuid.toString(), + ChatFolderTable.STORAGE_SERVICE_ID to Base64.encodeWithPadding(StorageSyncHelper.generateKey()) ) .run() @@ -110,7 +123,8 @@ private fun ChatFolderRecord.toBackupFrame(includedRecipientIds: List, exc else -> throw IllegalStateException("Only ALL or CUSTOM should be in the db") }, includedRecipientIds = includedRecipientIds, - excludedRecipientIds = excludedRecipientIds + excludedRecipientIds = excludedRecipientIds, + id = UuidUtil.toByteArray(this.chatFolderId.uuid).toByteString() ) return Frame(chatFolder = chatFolder) diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFolderId.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFolderId.kt new file mode 100644 index 0000000000..e78de89270 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFolderId.kt @@ -0,0 +1,31 @@ +package org.thoughtcrime.securesms.components.settings.app.chats.folders + +import android.os.Parcelable +import kotlinx.parcelize.Parcelize +import org.whispersystems.signalservice.api.util.UuidUtil +import java.util.UUID + +/** + * UUID wrapper class for chat folders, used in storage service + */ +@Parcelize +data class ChatFolderId(val uuid: UUID) : Parcelable { + + companion object { + fun from(id: String): ChatFolderId { + return ChatFolderId(UuidUtil.parseOrThrow(id)) + } + + fun from(uuid: UUID): ChatFolderId { + return ChatFolderId(uuid) + } + + fun generate(): ChatFolderId { + return ChatFolderId(UUID.randomUUID()) + } + } + + override fun toString(): String { + return uuid.toString() + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFolderRecord.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFolderRecord.kt index e5e004fcd5..1d911e7b20 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFolderRecord.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFolderRecord.kt @@ -1,7 +1,9 @@ package org.thoughtcrime.securesms.components.settings.app.chats.folders import android.os.Parcelable +import kotlinx.parcelize.IgnoredOnParcel import kotlinx.parcelize.Parcelize +import org.whispersystems.signalservice.api.storage.StorageId /** * Represents an entry in the [org.thoughtcrime.securesms.database.ChatFolderTables]. @@ -17,7 +19,12 @@ data class ChatFolderRecord( val showMutedChats: Boolean = true, val showIndividualChats: Boolean = false, val showGroupChats: Boolean = false, - val folderType: FolderType = FolderType.CUSTOM + val folderType: FolderType = FolderType.CUSTOM, + val chatFolderId: ChatFolderId = ChatFolderId.generate(), + @IgnoredOnParcel + val storageServiceId: StorageId? = null, + val storageServiceProto: ByteArray? = null, + val deletedTimestampMs: Long = 0 ) : Parcelable { enum class FolderType(val value: Int) { /** Folder containing all chats */ diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFoldersRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFoldersRepository.kt index 5ae05e112e..89baee08f3 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFoldersRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/ChatFoldersRepository.kt @@ -2,6 +2,7 @@ package org.thoughtcrime.securesms.components.settings.app.chats.folders import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.recipients.Recipient +import org.thoughtcrime.securesms.storage.StorageSyncHelper /** * Repository for chat folders that handles creation, deletion, listing, etc., @@ -9,7 +10,7 @@ import org.thoughtcrime.securesms.recipients.Recipient object ChatFoldersRepository { fun getCurrentFolders(): List { - return SignalDatabase.chatFolders.getChatFolders() + return SignalDatabase.chatFolders.getCurrentChatFolders() } fun getUnreadCountAndMutedStatusForFolders(folders: List): HashMap> { @@ -25,6 +26,7 @@ object ChatFoldersRepository { ) SignalDatabase.chatFolders.createFolder(updatedFolder) + StorageSyncHelper.scheduleSyncForDataChange() } fun updateFolder(folder: ChatFolderRecord, includedRecipients: Set, excludedRecipients: Set) { @@ -36,21 +38,29 @@ object ChatFoldersRepository { ) SignalDatabase.chatFolders.updateFolder(updatedFolder) + scheduleSync(updatedFolder.id) } fun deleteFolder(folder: ChatFolderRecord) { SignalDatabase.chatFolders.deleteChatFolder(folder) + scheduleSync(folder.id) } fun updatePositions(folders: List) { SignalDatabase.chatFolders.updatePositions(folders) + folders.forEach { scheduleSync(it.id) } } fun getFolder(id: Long): ChatFolderRecord { - return SignalDatabase.chatFolders.getChatFolder(id) + return SignalDatabase.chatFolders.getChatFolder(id)!! } fun getFolderCount(): Int { return SignalDatabase.chatFolders.getFolderCount() } + + private fun scheduleSync(id: Long) { + SignalDatabase.chatFolders.markNeedsSync(id) + StorageSyncHelper.scheduleSyncForDataChange() + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/CreateFoldersFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/CreateFoldersFragment.kt index ca6557de30..10a9dc6c59 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/CreateFoldersFragment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/chats/folders/CreateFoldersFragment.kt @@ -97,7 +97,7 @@ class CreateFoldersFragment : ComposeFragment() { val isNewFolder = state.originalFolder.folderRecord.id == -1L LaunchedEffect(Unit) { - if (state.originalFolder == state.currentFolder) { + if (state.originalFolder.folderRecord.id == state.currentFolder.folderRecord.id) { viewModel.setCurrentFolderId(arguments?.getLong(KEY_FOLDER_ID) ?: -1) viewModel.addThreadsToFolder(arguments?.getLongArray(KEY_THREAD_IDS)) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/ChatFolderTables.kt b/app/src/main/java/org/thoughtcrime/securesms/database/ChatFolderTables.kt index dccbcb7f93..e17f8c0596 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/ChatFolderTables.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/ChatFolderTables.kt @@ -3,23 +3,43 @@ package org.thoughtcrime.securesms.database import android.content.ContentValues import android.content.Context import androidx.core.content.contentValuesOf +import org.signal.core.util.Base64 import org.signal.core.util.SqlUtil import org.signal.core.util.count import org.signal.core.util.delete import org.signal.core.util.groupBy +import org.signal.core.util.hasUnknownFields import org.signal.core.util.insertInto +import org.signal.core.util.logging.Log import org.signal.core.util.readToList +import org.signal.core.util.readToMap import org.signal.core.util.readToSingleInt +import org.signal.core.util.readToSingleLongOrNull import org.signal.core.util.readToSingleObject import org.signal.core.util.requireBoolean import org.signal.core.util.requireInt import org.signal.core.util.requireLong +import org.signal.core.util.requireNonNullString import org.signal.core.util.requireString import org.signal.core.util.select import org.signal.core.util.update import org.signal.core.util.withinTransaction +import org.signal.libsignal.zkgroup.InvalidInputException +import org.signal.libsignal.zkgroup.groups.GroupMasterKey +import org.thoughtcrime.securesms.components.settings.app.chats.folders.ChatFolderId import org.thoughtcrime.securesms.components.settings.app.chats.folders.ChatFolderRecord +import org.thoughtcrime.securesms.database.ThreadTable.Companion.ID import org.thoughtcrime.securesms.dependencies.AppDependencies +import org.thoughtcrime.securesms.groups.BadGroupIdException +import org.thoughtcrime.securesms.groups.GroupId +import org.thoughtcrime.securesms.recipients.Recipient +import org.thoughtcrime.securesms.storage.StorageSyncHelper +import org.whispersystems.signalservice.api.push.ServiceId +import org.whispersystems.signalservice.api.push.SignalServiceAddress +import org.whispersystems.signalservice.api.storage.SignalChatFolderRecord +import org.whispersystems.signalservice.api.storage.StorageId +import org.whispersystems.signalservice.api.util.UuidUtil +import org.whispersystems.signalservice.internal.storage.protos.ChatFolderRecord as RemoteChatFolderRecord /** * Stores chat folders and the chats that belong in each chat folder @@ -27,6 +47,8 @@ import org.thoughtcrime.securesms.dependencies.AppDependencies class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : DatabaseTable(context, databaseHelper), ThreadIdDatabaseReference { companion object { + private val TAG = Log.tag(ChatFolderTable::class.java) + @JvmField val CREATE_TABLE: Array = arrayOf(ChatFolderTable.CREATE_TABLE, ChatFolderMembershipTable.CREATE_TABLE) @@ -43,7 +65,9 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat ChatFolderTable.FOLDER_TYPE to ChatFolderRecord.FolderType.ALL.value, ChatFolderTable.SHOW_INDIVIDUAL to 1, ChatFolderTable.SHOW_GROUPS to 1, - ChatFolderTable.SHOW_MUTED to 1 + ChatFolderTable.SHOW_MUTED to 1, + ChatFolderTable.CHAT_FOLDER_ID to ChatFolderId.generate().toString(), + ChatFolderTable.STORAGE_SERVICE_ID to Base64.encodeWithPadding(StorageSyncHelper.generateKey()) ) } } @@ -61,8 +85,11 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat const val SHOW_MUTED = "show_muted" const val SHOW_INDIVIDUAL = "show_individual" const val SHOW_GROUPS = "show_groups" - const val IS_MUTED = "is_muted" const val FOLDER_TYPE = "folder_type" + const val CHAT_FOLDER_ID = "chat_folder_id" + const val STORAGE_SERVICE_ID = "storage_service_id" + const val STORAGE_SERVICE_PROTO = "storage_service_proto" + const val DELETED_TIMESTAMP_MS = "deleted_timestamp_ms" val CREATE_TABLE = """ CREATE TABLE $TABLE_NAME ( @@ -73,8 +100,11 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat $SHOW_MUTED INTEGER DEFAULT 0, $SHOW_INDIVIDUAL INTEGER DEFAULT 0, $SHOW_GROUPS INTEGER DEFAULT 0, - $IS_MUTED INTEGER DEFAULT 0, - $FOLDER_TYPE INTEGER DEFAULT ${ChatFolderRecord.FolderType.CUSTOM.value} + $FOLDER_TYPE INTEGER DEFAULT ${ChatFolderRecord.FolderType.CUSTOM.value}, + $CHAT_FOLDER_ID TEXT DEFAULT NULL, + $STORAGE_SERVICE_ID TEXT DEFAULT NULL, + $STORAGE_SERVICE_PROTO TEXT DEFAULT NULL, + $DELETED_TIMESTAMP_MS INTEGER DEFAULT 0 ) """ @@ -119,10 +149,28 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat .run() } + /** + * Returns a single chat folder that corresponds to that query. + * Assumes query will only match to one chat folder. + */ + fun getChatFolder(query: SqlUtil.Query): ChatFolderRecord? { + val id = readableDatabase + .select(ChatFolderTable.ID) + .from(ChatFolderTable.TABLE_NAME) + .where(query.where, query.whereArgs) + .run() + .readToSingleLongOrNull() + + return getChatFolder(id) + } + /** * Returns a single chat folder that corresponds to that id */ - fun getChatFolder(id: Long): ChatFolderRecord { + fun getChatFolder(id: Long?): ChatFolderRecord? { + if (id == null) { + return null + } val includedChats: Map> = getIncludedChats(id) val excludedChats: Map> = getExcludedChats(id) @@ -142,7 +190,11 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat showGroupChats = cursor.requireBoolean(ChatFolderTable.SHOW_GROUPS), folderType = ChatFolderRecord.FolderType.deserialize(cursor.requireInt(ChatFolderTable.FOLDER_TYPE)), includedChats = includedChats[id] ?: emptyList(), - excludedChats = excludedChats[id] ?: emptyList() + excludedChats = excludedChats[id] ?: emptyList(), + chatFolderId = ChatFolderId.from(cursor.requireNonNullString(ChatFolderTable.CHAT_FOLDER_ID)), + storageServiceId = StorageId.forChatFolder(Base64.decodeNullableOrThrow(cursor.requireString(ChatFolderTable.STORAGE_SERVICE_ID))), + storageServiceProto = Base64.decodeOrNull(cursor.requireString(ChatFolderTable.STORAGE_SERVICE_PROTO)), + deletedTimestampMs = cursor.requireLong(ChatFolderTable.DELETED_TIMESTAMP_MS) ) } @@ -150,15 +202,16 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat } /** - * Maps the chat folder ids to its corresponding chat folder + * Returns all non-deleted chat folders */ - fun getChatFolders(): List { + fun getCurrentChatFolders(): List { val includedChats: Map> = getIncludedChats() val excludedChats: Map> = getExcludedChats() val folders = readableDatabase .select() .from(ChatFolderTable.TABLE_NAME) + .where("${ChatFolderTable.DELETED_TIMESTAMP_MS} = 0") .orderBy(ChatFolderTable.POSITION) .run() .readToList { cursor -> @@ -173,7 +226,10 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat showGroupChats = cursor.requireBoolean(ChatFolderTable.SHOW_GROUPS), folderType = ChatFolderRecord.FolderType.deserialize(cursor.requireInt(ChatFolderTable.FOLDER_TYPE)), includedChats = includedChats[id] ?: emptyList(), - excludedChats = excludedChats[id] ?: emptyList() + excludedChats = excludedChats[id] ?: emptyList(), + chatFolderId = ChatFolderId.from(cursor.requireNonNullString((ChatFolderTable.CHAT_FOLDER_ID))), + storageServiceId = StorageId.forChatFolder(Base64.decodeNullableOrThrow(cursor.requireString(RecipientTable.STORAGE_SERVICE_ID))), + storageServiceProto = Base64.decodeOrNull(cursor.requireString(ChatFolderTable.STORAGE_SERVICE_PROTO)) ) } @@ -194,54 +250,13 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat } /** - * Maps a chat folder id to all of its corresponding included chats. - * If an id is not specified, all chat folder ids will be mapped. - */ - private fun getIncludedChats(id: Long? = null): Map> { - val whereQuery = if (id != null) { - "${ChatFolderMembershipTable.MEMBERSHIP_TYPE} = ${MembershipType.INCLUDED.value} AND ${ChatFolderMembershipTable.CHAT_FOLDER_ID} = $id" - } else { - "${ChatFolderMembershipTable.MEMBERSHIP_TYPE} = ${MembershipType.INCLUDED.value}" - } - - return readableDatabase - .select() - .from(ChatFolderMembershipTable.TABLE_NAME) - .where(whereQuery) - .run() - .groupBy { cursor -> - cursor.requireLong(ChatFolderMembershipTable.CHAT_FOLDER_ID) to cursor.requireLong(ChatFolderMembershipTable.THREAD_ID) - } - } - - /** - * Maps a chat folder id to all of its corresponding excluded chats. - * If an id is not specified, all chat folder ids will be mapped. - */ - private fun getExcludedChats(id: Long? = null): Map> { - val whereQuery = if (id != null) { - "${ChatFolderMembershipTable.MEMBERSHIP_TYPE} = ${MembershipType.EXCLUDED.value} AND ${ChatFolderMembershipTable.CHAT_FOLDER_ID} = $id" - } else { - "${ChatFolderMembershipTable.MEMBERSHIP_TYPE} = ${MembershipType.EXCLUDED.value}" - } - - return readableDatabase - .select() - .from(ChatFolderMembershipTable.TABLE_NAME) - .where(whereQuery) - .run() - .groupBy { cursor -> - cursor.requireLong(ChatFolderMembershipTable.CHAT_FOLDER_ID) to cursor.requireLong(ChatFolderMembershipTable.THREAD_ID) - } - } - - /** - * Returns the number of folders a user has, including the default 'All Chats' + * Returns the number of non-deleted folders a user has, including the default 'All Chats' */ fun getFolderCount(): Int { return readableDatabase .count() .from(ChatFolderTable.TABLE_NAME) + .where("${ChatFolderTable.DELETED_TIMESTAMP_MS} = 0") .run() .readToSingleInt() } @@ -257,6 +272,8 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat .run() .readToSingleInt(0) + 1 + val storageId = chatFolder.storageServiceId?.raw ?: StorageSyncHelper.generateKey() + val storageServiceProto = if (chatFolder.storageServiceProto != null) Base64.encodeWithPadding(chatFolder.storageServiceProto) else null val id = db.insertInto(ChatFolderTable.TABLE_NAME) .values( contentValuesOf( @@ -265,7 +282,11 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat ChatFolderTable.SHOW_MUTED to chatFolder.showMutedChats, ChatFolderTable.SHOW_INDIVIDUAL to chatFolder.showIndividualChats, ChatFolderTable.SHOW_GROUPS to chatFolder.showGroupChats, - ChatFolderTable.POSITION to position + ChatFolderTable.POSITION to if (chatFolder.position == -1 && chatFolder.deletedTimestampMs == 0L) position else chatFolder.position, + ChatFolderTable.CHAT_FOLDER_ID to chatFolder.chatFolderId.toString(), + ChatFolderTable.STORAGE_SERVICE_ID to Base64.encodeWithPadding(storageId), + ChatFolderTable.STORAGE_SERVICE_PROTO to storageServiceProto, + ChatFolderTable.DELETED_TIMESTAMP_MS to chatFolder.deletedTimestampMs ) ) .run(SQLiteDatabase.CONFLICT_IGNORE) @@ -298,14 +319,18 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat * Updates the details for an existing folder like name, chat types, etc. */ fun updateFolder(chatFolder: ChatFolderRecord) { + val storageServiceProto = if (chatFolder.storageServiceProto != null) Base64.encodeWithPadding(chatFolder.storageServiceProto) else null writableDatabase.withinTransaction { db -> db.update(ChatFolderTable.TABLE_NAME) .values( ChatFolderTable.NAME to chatFolder.name, + ChatFolderTable.POSITION to chatFolder.position, ChatFolderTable.SHOW_UNREAD to chatFolder.showUnread, ChatFolderTable.SHOW_MUTED to chatFolder.showMutedChats, ChatFolderTable.SHOW_INDIVIDUAL to chatFolder.showIndividualChats, - ChatFolderTable.SHOW_GROUPS to chatFolder.showGroupChats + ChatFolderTable.SHOW_GROUPS to chatFolder.showGroupChats, + ChatFolderTable.STORAGE_SERVICE_PROTO to storageServiceProto, + ChatFolderTable.DELETED_TIMESTAMP_MS to chatFolder.deletedTimestampMs ) .where("${ChatFolderTable.ID} = ?", chatFolder.id) .run(SQLiteDatabase.CONFLICT_IGNORE) @@ -336,11 +361,23 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat } /** - * Deletes a chat folder + * Marks a chat folder as deleted and removes all associated chats */ fun deleteChatFolder(chatFolder: ChatFolderRecord) { writableDatabase.withinTransaction { db -> - db.delete(ChatFolderTable.TABLE_NAME, "${ChatFolderTable.ID} = ?", SqlUtil.buildArgs(chatFolder.id)) + db.update(ChatFolderTable.TABLE_NAME) + .values( + ChatFolderTable.DELETED_TIMESTAMP_MS to System.currentTimeMillis(), + ChatFolderTable.POSITION to -1 + ) + .where("$ID = ?", chatFolder.id) + .run() + + db.delete(ChatFolderMembershipTable.TABLE_NAME) + .where("${ChatFolderMembershipTable.CHAT_FOLDER_ID} = ?", chatFolder.id) + .run() + + resetPositions() AppDependencies.databaseObserver.notifyChatFolderObservers() } } @@ -405,6 +442,253 @@ class ChatFolderTables(context: Context?, databaseHelper: SignalDatabase?) : Dat } } + /** + * Saves the new storage id for a chat folder + */ + fun applyStorageIdUpdate(id: ChatFolderId, storageId: StorageId) { + applyStorageIdUpdates(hashMapOf(id to storageId)) + } + + /** + * Saves the new storage ids for all the chat folders in the map + */ + fun applyStorageIdUpdates(storageIds: Map) { + writableDatabase.withinTransaction { db -> + storageIds.forEach { (chatFolderId, storageId) -> + db.update(ChatFolderTable.TABLE_NAME) + .values(ChatFolderTable.STORAGE_SERVICE_ID to Base64.encodeWithPadding(storageId.raw)) + .where("${ChatFolderTable.CHAT_FOLDER_ID} = ?", chatFolderId.toString()) + .run() + } + } + } + + /** + * Maps all chat folder ids to storage ids + */ + fun getStorageSyncIdsMap(): Map { + return readableDatabase + .select(ChatFolderTable.CHAT_FOLDER_ID, ChatFolderTable.STORAGE_SERVICE_ID) + .from(ChatFolderTable.TABLE_NAME) + .run() + .readToMap { cursor -> + val id = ChatFolderId.from(cursor.requireNonNullString(ChatFolderTable.CHAT_FOLDER_ID)) + val encodedKey = cursor.requireNonNullString(ChatFolderTable.STORAGE_SERVICE_ID) + val key = Base64.decodeOrThrow(encodedKey) + id to StorageId.forChatFolder(key) + } + } + + /** + * Returns a list of all the storage ids used in chat folders + */ + fun getStorageSyncIds(): List { + return readableDatabase + .select(ChatFolderTable.STORAGE_SERVICE_ID) + .from(ChatFolderTable.TABLE_NAME) + .run() + .readToList { cursor -> + val encodedKey = cursor.requireNonNullString(ChatFolderTable.STORAGE_SERVICE_ID) + val key = Base64.decodeOrThrow(encodedKey) + StorageId.forChatFolder(key) + } + } + + /** + * Creates a new storage id for a folder. Assumption is that StorageSyncHelper.scheduleSyncForDataChange() will be called after. + */ + fun markNeedsSync(folderId: Long) { + markNeedsSync(listOf(folderId)) + } + + /** + * Creates new storage ids for multiple folders. Assumption is that StorageSyncHelper.scheduleSyncForDataChange() will be called after. + */ + fun markNeedsSync(folderIds: List) { + writableDatabase.withinTransaction { + for (id in folderIds) { + rotateStorageId(id) + } + } + } + + /** + * Inserts a remote chat folder into the database + */ + fun insertChatFolderFromStorageSync(record: SignalChatFolderRecord) { + if (record.proto.folderType == RemoteChatFolderRecord.FolderType.ALL) { + Log.i(TAG, "All chats should already exists. Avoiding inserting another and only updating relevant fields") + val storageServiceProto = if (record.proto.hasUnknownFields()) Base64.encodeWithPadding(record.serializedUnknowns!!) else null + writableDatabase.withinTransaction { db -> + db.update(ChatFolderTable.TABLE_NAME) + .values( + ChatFolderTable.POSITION to record.proto.position, + ChatFolderTable.CHAT_FOLDER_ID to ChatFolderId.from(UuidUtil.parseOrThrow(record.proto.identifier)).toString(), + ChatFolderTable.STORAGE_SERVICE_ID to Base64.encodeWithPadding(record.id.raw), + ChatFolderTable.STORAGE_SERVICE_PROTO to storageServiceProto + ) + .where("${ChatFolderTable.FOLDER_TYPE} = ?", ChatFolderRecord.FolderType.ALL.value) + .run(SQLiteDatabase.CONFLICT_IGNORE) + } + } else { + createFolder(remoteChatFolderRecordToLocal(record)) + } + } + + /** + * Updates an existing local chat folder with the details of the remote chat folder + */ + fun updateChatFolderFromStorageSync(record: SignalChatFolderRecord) { + updateFolder(remoteChatFolderRecordToLocal(record)) + } + + /** + * Maps a chat folder id to all of its corresponding included chats. + * If an id is not specified, all chat folder ids will be mapped. + */ + private fun getIncludedChats(id: Long? = null): Map> { + val whereQuery = if (id != null) { + "${ChatFolderMembershipTable.MEMBERSHIP_TYPE} = ${MembershipType.INCLUDED.value} AND ${ChatFolderMembershipTable.CHAT_FOLDER_ID} = $id" + } else { + "${ChatFolderMembershipTable.MEMBERSHIP_TYPE} = ${MembershipType.INCLUDED.value}" + } + + return readableDatabase + .select() + .from(ChatFolderMembershipTable.TABLE_NAME) + .where(whereQuery) + .run() + .groupBy { cursor -> + cursor.requireLong(ChatFolderMembershipTable.CHAT_FOLDER_ID) to cursor.requireLong(ChatFolderMembershipTable.THREAD_ID) + } + } + + /** + * Maps a chat folder id to all of its corresponding excluded chats. + * If an id is not specified, all chat folder ids will be mapped. + */ + private fun getExcludedChats(id: Long? = null): Map> { + val whereQuery = if (id != null) { + "${ChatFolderMembershipTable.MEMBERSHIP_TYPE} = ${MembershipType.EXCLUDED.value} AND ${ChatFolderMembershipTable.CHAT_FOLDER_ID} = $id" + } else { + "${ChatFolderMembershipTable.MEMBERSHIP_TYPE} = ${MembershipType.EXCLUDED.value}" + } + + return readableDatabase + .select() + .from(ChatFolderMembershipTable.TABLE_NAME) + .where(whereQuery) + .run() + .groupBy { cursor -> + cursor.requireLong(ChatFolderMembershipTable.CHAT_FOLDER_ID) to cursor.requireLong(ChatFolderMembershipTable.THREAD_ID) + } + } + + /** + * Ensures that chat folders positions are 0-indexed and consecutive + */ + private fun resetPositions() { + val folders = readableDatabase + .select(ChatFolderTable.ID) + .from(ChatFolderTable.TABLE_NAME) + .where("${ChatFolderTable.DELETED_TIMESTAMP_MS} = 0") + .orderBy("${ChatFolderTable.POSITION} ASC") + .run() + .readToList { cursor -> cursor.requireLong(ChatFolderTable.ID) } + + writableDatabase.withinTransaction { db -> + folders.forEachIndexed { index, id -> + db.update(ChatFolderTable.TABLE_NAME) + .values(ChatFolderTable.POSITION to index) + .where("${ChatFolderTable.ID} = ?", id) + .run(SQLiteDatabase.CONFLICT_IGNORE) + } + } + } + + /** + * Rotates the storage id for a chat folder + */ + private fun rotateStorageId(id: Long) { + writableDatabase + .update(ChatFolderTable.TABLE_NAME) + .values(ChatFolderTable.STORAGE_SERVICE_ID to Base64.encodeWithPadding(StorageSyncHelper.generateKey())) + .where("${ChatFolderTable.ID} = ?", id) + .run() + } + + /** + * Given a [ChatFolderId], return the id for that folder + */ + private fun getIdByRemoteChatFolderId(chatFolderId: ChatFolderId): Long? { + return readableDatabase + .select(ChatFolderTable.ID) + .from(ChatFolderTable.TABLE_NAME) + .where("${ChatFolderTable.CHAT_FOLDER_ID} = ?", chatFolderId.toString()) + .run() + .readToSingleLongOrNull() + } + + /** + * Parses a remote chat folder [SignalChatFolderRecord] into a local folder [ChatFolderRecord] + */ + private fun remoteChatFolderRecordToLocal(record: SignalChatFolderRecord): ChatFolderRecord { + val chatFolderId = ChatFolderId.from(UuidUtil.parseOrThrow(record.proto.identifier)) + val id = getIdByRemoteChatFolderId(chatFolderId) + return ChatFolderRecord( + id = id ?: -1, + name = record.proto.name, + position = record.proto.position, + showUnread = record.proto.showOnlyUnread, + showMutedChats = record.proto.showMutedChats, + showIndividualChats = record.proto.includeAllIndividualChats, + showGroupChats = record.proto.includeAllGroupChats, + folderType = when (record.proto.folderType) { + RemoteChatFolderRecord.FolderType.ALL -> ChatFolderRecord.FolderType.ALL + RemoteChatFolderRecord.FolderType.CUSTOM -> ChatFolderRecord.FolderType.CUSTOM + RemoteChatFolderRecord.FolderType.UNKNOWN -> throw AssertionError("Folder type cannot be unknown") + }, + includedChats = record.proto.includedRecipients + .mapNotNull { remoteRecipient -> getRecipientIdFromRemoteRecipient(remoteRecipient) } + .map { recipient -> SignalDatabase.threads.getOrCreateThreadIdFor(recipient) }, + excludedChats = record.proto.excludedRecipients + .mapNotNull { remoteRecipient -> getRecipientIdFromRemoteRecipient(remoteRecipient) } + .map { recipient -> SignalDatabase.threads.getOrCreateThreadIdFor(recipient) }, + chatFolderId = chatFolderId, + storageServiceId = StorageId.forChatFolder(record.id.raw), + storageServiceProto = record.serializedUnknowns, + deletedTimestampMs = record.proto.deletedAtTimestampMs + ) + } + + /** + * Parses a remote recipient into a local one. Used when configuring the chats of a remote chat folder into a local one. + */ + private fun getRecipientIdFromRemoteRecipient(remoteRecipient: RemoteChatFolderRecord.Recipient): Recipient? { + return if (remoteRecipient.contact != null) { + val serviceId = ServiceId.parseOrNull(remoteRecipient.contact!!.serviceId) + val e164 = remoteRecipient.contact!!.e164 + Recipient.externalPush(SignalServiceAddress(serviceId, e164)) + } else if (remoteRecipient.legacyGroupId != null) { + try { + Recipient.externalGroupExact(GroupId.v1(remoteRecipient.legacyGroupId!!.toByteArray())) + } catch (e: BadGroupIdException) { + Log.w(TAG, "Failed to parse groupV1 ID!", e) + null + } + } else if (remoteRecipient.groupMasterKey != null) { + try { + Recipient.externalGroupExact(GroupId.v2(GroupMasterKey(remoteRecipient.groupMasterKey!!.toByteArray()))) + } catch (e: InvalidInputException) { + Log.w(TAG, "Failed to parse groupV2 master key!", e) + null + } + } else { + Log.w(TAG, "Could not find recipient") + null + } + } + private fun Collection.toContentValues(chatFolderId: Long, membershipType: MembershipType): List { return map { contentValuesOf( diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt index 071abf0cec..019a2479a4 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt @@ -124,6 +124,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V266_UniqueThreadPi import org.thoughtcrime.securesms.database.helpers.migration.V267_FixGroupInvitationDeclinedUpdate import org.thoughtcrime.securesms.database.helpers.migration.V268_FixInAppPaymentsErrorStateConsistency import org.thoughtcrime.securesms.database.helpers.migration.V269_BackupMediaSnapshotChanges +import org.thoughtcrime.securesms.database.helpers.migration.V270_FixChatFolderColumnsForStorageSync import org.thoughtcrime.securesms.database.SQLiteDatabase as SignalSqliteDatabase /** @@ -253,10 +254,11 @@ object SignalDatabaseMigrations { 266 to V266_UniqueThreadPinOrder, 267 to V267_FixGroupInvitationDeclinedUpdate, 268 to V268_FixInAppPaymentsErrorStateConsistency, - 269 to V269_BackupMediaSnapshotChanges + 269 to V269_BackupMediaSnapshotChanges, + 270 to V270_FixChatFolderColumnsForStorageSync ) - const val DATABASE_VERSION = 269 + const val DATABASE_VERSION = 270 @JvmStatic fun migrate(context: Application, db: SignalSqliteDatabase, oldVersion: Int, newVersion: Int) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V270_FixChatFolderColumnsForStorageSync.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V270_FixChatFolderColumnsForStorageSync.kt new file mode 100644 index 0000000000..af01ffe644 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V270_FixChatFolderColumnsForStorageSync.kt @@ -0,0 +1,38 @@ +package org.thoughtcrime.securesms.database.helpers.migration + +import android.app.Application +import org.signal.core.util.readToList +import org.signal.core.util.requireLong +import org.thoughtcrime.securesms.database.SQLiteDatabase +import java.util.UUID + +/** + * Adds four columns chat_folder_id, storage_service_id, storage_service_proto and deleted_timestamp_ms to support chat folders in storage sync. + * Removes unnecessary is_muted column in chat folders table. + */ +@Suppress("ClassName") +object V270_FixChatFolderColumnsForStorageSync : SignalDatabaseMigration { + + override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + db.execSQL("ALTER TABLE chat_folder DROP COLUMN is_muted") + db.execSQL("ALTER TABLE chat_folder ADD COLUMN chat_folder_id TEXT DEFAULT NULL") + db.execSQL("ALTER TABLE chat_folder ADD COLUMN storage_service_id TEXT DEFAULT NULL") + db.execSQL("ALTER TABLE chat_folder ADD COLUMN storage_service_proto TEXT DEFAULT NULL") + db.execSQL("ALTER TABLE chat_folder ADD COLUMN deleted_timestamp_ms INTEGER DEFAULT 0") + + // Assign all of the folders with a [ChatFolderId] and reset position + db.rawQuery("SELECT _id FROM chat_folder ORDER BY position ASC") + .readToList { it.requireLong("_id") } + .forEachIndexed { index, id -> resetPositionAndSetChatFolderId(db, id, index) } + } + + private fun resetPositionAndSetChatFolderId(db: SQLiteDatabase, id: Long, newPosition: Int) { + db.rawQuery( + """ + UPDATE chat_folder + SET chat_folder_id = '${UUID.randomUUID()}', position = $newPosition + WHERE _id = $id + """ + ) + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index 85e6d85671..77782eba20 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -95,6 +95,7 @@ import org.thoughtcrime.securesms.migrations.StoryViewedReceiptsStateMigrationJo import org.thoughtcrime.securesms.migrations.SubscriberIdMigrationJob; import org.thoughtcrime.securesms.migrations.Svr2MirrorMigrationJob; import org.thoughtcrime.securesms.migrations.SyncCallLinksMigrationJob; +import org.thoughtcrime.securesms.migrations.SyncChatFoldersMigrationJob; import org.thoughtcrime.securesms.migrations.SyncDistributionListsMigrationJob; import org.thoughtcrime.securesms.migrations.SyncKeysMigrationJob; import org.thoughtcrime.securesms.migrations.TrimByLengthSettingsMigrationJob; @@ -327,6 +328,7 @@ public final class JobManagerFactories { put(SubscriberIdMigrationJob.KEY, new SubscriberIdMigrationJob.Factory()); put(Svr2MirrorMigrationJob.KEY, new Svr2MirrorMigrationJob.Factory()); put(SyncCallLinksMigrationJob.KEY, new SyncCallLinksMigrationJob.Factory()); + put(SyncChatFoldersMigrationJob.KEY, new SyncChatFoldersMigrationJob.Factory()); put(SyncDistributionListsMigrationJob.KEY, new SyncDistributionListsMigrationJob.Factory()); put(SyncKeysMigrationJob.KEY, new SyncKeysMigrationJob.Factory()); put(TrimByLengthSettingsMigrationJob.KEY, new TrimByLengthSettingsMigrationJob.Factory()); diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageForcePushJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageForcePushJob.kt index 72ba26842c..b60715901b 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageForcePushJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageForcePushJob.kt @@ -1,7 +1,10 @@ package org.thoughtcrime.securesms.jobs +import org.signal.core.util.SqlUtil import org.signal.core.util.logging.Log import org.signal.core.util.logging.logI +import org.thoughtcrime.securesms.components.settings.app.chats.folders.ChatFolderId +import org.thoughtcrime.securesms.database.ChatFolderTables.ChatFolderTable import org.thoughtcrime.securesms.database.RecipientTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -95,6 +98,18 @@ class StorageForcePushJob private constructor(parameters: Parameters) : BaseJob( inserts.add(accountRecord) allNewStorageIds.add(accountRecord.id) + val oldChatFolderStorageIds = SignalDatabase.chatFolders.getStorageSyncIdsMap() + val newChatFolderStorageIds = generateChatFolderStorageIds(oldChatFolderStorageIds) + val newChatFolderInserts: List = oldChatFolderStorageIds.keys + .mapNotNull { + val query = SqlUtil.buildQuery("${ChatFolderTable.CHAT_FOLDER_ID} = ?", it) + SignalDatabase.chatFolders.getChatFolder(query) + } + .map { record -> StorageSyncModels.localToRemoteRecord(record, newChatFolderStorageIds[record.chatFolderId]!!.raw) } + + inserts.addAll(newChatFolderInserts) + allNewStorageIds.addAll(newChatFolderStorageIds.values) + val recordIkm: RecordIkm? = if (Recipient.self().storageServiceEncryptionV2Capability.isSupported) { Log.i(TAG, "Generating and including a new recordIkm.") RecordIkm.generate() @@ -135,6 +150,7 @@ class StorageForcePushJob private constructor(parameters: Parameters) : BaseJob( SignalStore.svr.masterKeyForInitialDataRestore = null SignalDatabase.recipients.applyStorageIdUpdates(newContactStorageIds) SignalDatabase.recipients.applyStorageIdUpdates(Collections.singletonMap(Recipient.self().id, accountRecord.id)) + SignalDatabase.chatFolders.applyStorageIdUpdates(newChatFolderStorageIds) SignalDatabase.unknownStorageIds.deleteAll() } @@ -154,6 +170,16 @@ class StorageForcePushJob private constructor(parameters: Parameters) : BaseJob( return out } + private fun generateChatFolderStorageIds(oldKeys: Map): Map { + val out: MutableMap = mutableMapOf() + + for ((key, value) in oldKeys) { + out[key] = value.withNewBytes(StorageSyncHelper.generateKey()) + } + + return out + } + class Factory : Job.Factory { override fun create(parameters: Parameters, serializedData: ByteArray?): StorageForcePushJob { return StorageForcePushJob(parameters) diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageSyncJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageSyncJob.kt index 509d8d2dad..541ec67a26 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageSyncJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageSyncJob.kt @@ -2,10 +2,13 @@ package org.thoughtcrime.securesms.jobs import android.content.Context import com.annimon.stream.Stream +import org.signal.core.util.Base64 +import org.signal.core.util.SqlUtil import org.signal.core.util.Stopwatch import org.signal.core.util.logging.Log import org.signal.core.util.withinTransaction import org.signal.libsignal.protocol.InvalidKeyException +import org.thoughtcrime.securesms.database.ChatFolderTables.ChatFolderTable import org.thoughtcrime.securesms.database.RecipientTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -16,6 +19,7 @@ import org.thoughtcrime.securesms.net.SignalNetwork import org.thoughtcrime.securesms.recipients.Recipient import org.thoughtcrime.securesms.storage.AccountRecordProcessor import org.thoughtcrime.securesms.storage.CallLinkRecordProcessor +import org.thoughtcrime.securesms.storage.ChatFolderRecordProcessor import org.thoughtcrime.securesms.storage.ContactRecordProcessor import org.thoughtcrime.securesms.storage.GroupV1RecordProcessor import org.thoughtcrime.securesms.storage.GroupV2RecordProcessor @@ -31,6 +35,7 @@ import org.whispersystems.signalservice.api.messages.multidevice.SignalServiceSy import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException import org.whispersystems.signalservice.api.storage.SignalAccountRecord import org.whispersystems.signalservice.api.storage.SignalCallLinkRecord +import org.whispersystems.signalservice.api.storage.SignalChatFolderRecord import org.whispersystems.signalservice.api.storage.SignalContactRecord import org.whispersystems.signalservice.api.storage.SignalGroupV1Record import org.whispersystems.signalservice.api.storage.SignalGroupV2Record @@ -43,6 +48,7 @@ import org.whispersystems.signalservice.api.storage.StorageServiceRepository import org.whispersystems.signalservice.api.storage.StorageServiceRepository.ManifestIfDifferentVersionResult import org.whispersystems.signalservice.api.storage.toSignalAccountRecord import org.whispersystems.signalservice.api.storage.toSignalCallLinkRecord +import org.whispersystems.signalservice.api.storage.toSignalChatFolderRecord import org.whispersystems.signalservice.api.storage.toSignalContactRecord import org.whispersystems.signalservice.api.storage.toSignalGroupV1Record import org.whispersystems.signalservice.api.storage.toSignalGroupV2Record @@ -283,7 +289,7 @@ class StorageSyncJob private constructor(parameters: Parameters) : BaseJob(param db.beginTransaction() try { - Log.i(TAG, "[Remote Sync] Remote-Only :: Contacts: ${remoteOnly.contacts.size}, GV1: ${remoteOnly.gv1.size}, GV2: ${remoteOnly.gv2.size}, Account: ${remoteOnly.account.size}, DLists: ${remoteOnly.storyDistributionLists.size}, call links: ${remoteOnly.callLinkRecords.size}") + Log.i(TAG, "[Remote Sync] Remote-Only :: Contacts: ${remoteOnly.contacts.size}, GV1: ${remoteOnly.gv1.size}, GV2: ${remoteOnly.gv2.size}, Account: ${remoteOnly.account.size}, DLists: ${remoteOnly.storyDistributionLists.size}, call links: ${remoteOnly.callLinkRecords.size}, chat folders: ${remoteOnly.chatFolderRecords.size}") processKnownRecords(context, remoteOnly) @@ -420,11 +426,13 @@ class StorageSyncJob private constructor(parameters: Parameters) : BaseJob(param AccountRecordProcessor(context, freshSelf()).process(records.account, StorageSyncHelper.KEY_GENERATOR) StoryDistributionListRecordProcessor().process(records.storyDistributionLists, StorageSyncHelper.KEY_GENERATOR) CallLinkRecordProcessor().process(records.callLinkRecords, StorageSyncHelper.KEY_GENERATOR) + ChatFolderRecordProcessor().process(records.chatFolderRecords, StorageSyncHelper.KEY_GENERATOR) } private fun getAllLocalStorageIds(self: Recipient): List { return SignalDatabase.recipients.getContactStorageSyncIds() + listOf(StorageId.forAccount(self.storageId)) + + SignalDatabase.chatFolders.getStorageSyncIds() + SignalDatabase.unknownStorageIds.allUnknownIds } @@ -488,6 +496,16 @@ class StorageSyncJob private constructor(parameters: Parameters) : BaseJob(param } } + ManifestRecord.Identifier.Type.CHAT_FOLDER -> { + val query = SqlUtil.buildQuery("${ChatFolderTable.STORAGE_SERVICE_ID} = ?", Base64.encodeWithPadding(id.raw)) + val chatFolderRecord = SignalDatabase.chatFolders.getChatFolder(query) + if (chatFolderRecord?.chatFolderId != null) { + records.add(StorageSyncModels.localToRemoteRecord(chatFolderRecord, id.raw)) + } else { + throw MissingChatFolderModelError("Missing local chat folder model!") + } + } + else -> { val unknown = SignalDatabase.unknownStorageIds.getById(id.raw) if (unknown != null) { @@ -521,6 +539,7 @@ class StorageSyncJob private constructor(parameters: Parameters) : BaseJob(param val unknown: MutableList = mutableListOf() val storyDistributionLists: MutableList = mutableListOf() val callLinkRecords: MutableList = mutableListOf() + val chatFolderRecords: MutableList = mutableListOf() init { for (record in records) { @@ -536,6 +555,8 @@ class StorageSyncJob private constructor(parameters: Parameters) : BaseJob(param storyDistributionLists += record.proto.storyDistributionList!!.toSignalStoryDistributionListRecord(record.id) } else if (record.proto.callLink != null) { callLinkRecords += record.proto.callLink!!.toSignalCallLinkRecord(record.id) + } else if (record.proto.chatFolder != null) { + chatFolderRecords += record.proto.chatFolder!!.toSignalChatFolderRecord(record.id) } else if (record.id.isUnknown) { unknown += record } else { @@ -549,6 +570,8 @@ class StorageSyncJob private constructor(parameters: Parameters) : BaseJob(param private class MissingRecipientModelError(message: String?) : Error(message) + private class MissingChatFolderModelError(message: String?) : Error(message) + private class MissingUnknownModelError(message: String?) : Error(message) class Factory : Job.Factory { diff --git a/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java b/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java index b42a570250..a199ff25d5 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java +++ b/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java @@ -176,9 +176,10 @@ public class ApplicationMigrations { static final int AVATAR_COLOR_MIGRATION_JOB = 132; static final int DUPLICATE_E164_FIX_2 = 133; static final int E164_FORMATTING = 134; + static final int CHAT_FOLDER_STORAGE_SYNC = 135; } - public static final int CURRENT_VERSION = 134; + public static final int CURRENT_VERSION = 135; /** * This *must* be called after the {@link JobManager} has been instantiated, but *before* the call @@ -813,6 +814,10 @@ public class ApplicationMigrations { jobs.put(Version.E164_FORMATTING, new E164FormattingMigrationJob()); } + if (lastSeenVersion < Version.CHAT_FOLDER_STORAGE_SYNC) { + jobs.put(Version.CHAT_FOLDER_STORAGE_SYNC, new SyncChatFoldersMigrationJob()); + } + return jobs; } diff --git a/app/src/main/java/org/thoughtcrime/securesms/migrations/SyncChatFoldersMigrationJob.kt b/app/src/main/java/org/thoughtcrime/securesms/migrations/SyncChatFoldersMigrationJob.kt new file mode 100644 index 0000000000..40865181df --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/migrations/SyncChatFoldersMigrationJob.kt @@ -0,0 +1,55 @@ +package org.thoughtcrime.securesms.migrations + +import org.signal.core.util.logging.Log +import org.signal.core.util.readToList +import org.signal.core.util.requireLong +import org.signal.core.util.select +import org.thoughtcrime.securesms.database.ChatFolderTables +import org.thoughtcrime.securesms.database.ChatFolderTables.ChatFolderTable +import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.jobmanager.Job +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.storage.StorageSyncHelper + +/** + * Marks all chat folders as needing to be synced for storage service. + */ +internal class SyncChatFoldersMigrationJob(parameters: Parameters = Parameters.Builder().build()) : MigrationJob(parameters) { + companion object { + const val KEY = "SyncChatFoldersMigrationJob" + + private val TAG = Log.tag(SyncChatFoldersMigrationJob::class) + } + + override fun getFactoryKey(): String = KEY + + override fun isUiBlocking(): Boolean = false + + override fun performMigration() { + if (SignalStore.account.aci == null) { + Log.w(TAG, "Self not available yet.") + return + } + + val folderIds = SignalDatabase.chatFolders.getAllFoldersForSync() + + SignalDatabase.chatFolders.markNeedsSync(folderIds) + StorageSyncHelper.scheduleSyncForDataChange() + } + + override fun shouldRetry(e: Exception): Boolean = false + + private fun ChatFolderTables.getAllFoldersForSync(): List { + return readableDatabase + .select(ChatFolderTable.ID) + .from(ChatFolderTable.TABLE_NAME) + .run() + .readToList { cursor -> cursor.requireLong(ChatFolderTable.ID) } + } + + class Factory : Job.Factory { + override fun create(parameters: Parameters, serializedData: ByteArray?): SyncChatFoldersMigrationJob { + return SyncChatFoldersMigrationJob(parameters) + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/storage/ChatFolderRecordProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/storage/ChatFolderRecordProcessor.kt new file mode 100644 index 0000000000..99f25d4a5b --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/storage/ChatFolderRecordProcessor.kt @@ -0,0 +1,104 @@ +package org.thoughtcrime.securesms.storage + +import org.signal.core.util.SqlUtil +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.components.settings.app.chats.folders.ChatFolderId +import org.thoughtcrime.securesms.database.ChatFolderTables +import org.thoughtcrime.securesms.database.SignalDatabase +import org.whispersystems.signalservice.api.push.ServiceId +import org.whispersystems.signalservice.api.storage.SignalChatFolderRecord +import org.whispersystems.signalservice.api.storage.StorageId +import org.whispersystems.signalservice.api.util.OptionalUtil.asOptional +import org.whispersystems.signalservice.api.util.UuidUtil +import org.whispersystems.signalservice.internal.storage.protos.ChatFolderRecord +import java.util.Optional +import java.util.UUID + +/** + * Record processor for [SignalChatFolderRecord]. + * Handles merging and updating our local store when processing remote chat folder storage records. + */ +class ChatFolderRecordProcessor : DefaultStorageRecordProcessor() { + + companion object { + private val TAG = Log.tag(ChatFolderRecordProcessor::class) + } + + override fun compare(o1: SignalChatFolderRecord, o2: SignalChatFolderRecord): Int { + return if (o1.proto.identifier == o2.proto.identifier) { + 0 + } else { + 1 + } + } + + /** + * Folders must have a valid identifier and known folder type + * Custom chat folders must have a name. + * If a folder is deleted, it must have a -1 position + * If a folder is not deleted, it must have a non-negative position + * All recipients must have a valid serviceId + */ + override fun isInvalid(remote: SignalChatFolderRecord): Boolean { + return UuidUtil.parseOrNull(remote.proto.identifier) == null || + remote.proto.folderType == ChatFolderRecord.FolderType.UNKNOWN || + (remote.proto.folderType == ChatFolderRecord.FolderType.CUSTOM && remote.proto.name.isEmpty()) || + (remote.proto.deletedAtTimestampMs > 0 && remote.proto.position != -1) || + (remote.proto.deletedAtTimestampMs == 0L && remote.proto.position < 0) || + containsInvalidServiceId(remote.proto.includedRecipients) || + containsInvalidServiceId(remote.proto.excludedRecipients) + } + + override fun getMatching(remote: SignalChatFolderRecord, keyGenerator: StorageKeyGenerator): Optional { + Log.d(TAG, "Attempting to get matching record...") + val uuid: UUID = UuidUtil.parseOrThrow(remote.proto.identifier) + val query = SqlUtil.buildQuery("${ChatFolderTables.ChatFolderTable.CHAT_FOLDER_ID} = ?", ChatFolderId.from(uuid)) + val folder = SignalDatabase.chatFolders.getChatFolder(query) + + return if (folder?.storageServiceId != null) { + StorageSyncModels.localToRemoteChatFolder(folder, folder.storageServiceId.raw).asOptional() + } else if (folder != null) { + Log.d(TAG, "Folder was missing a storage service id, generating one") + val storageId = StorageId.forChatFolder(keyGenerator.generate()) + SignalDatabase.chatFolders.applyStorageIdUpdate(folder.chatFolderId, storageId) + StorageSyncModels.localToRemoteChatFolder(folder, storageId.raw).asOptional() + } else { + Log.d(TAG, "Could not find a matching record. Returning an empty.") + Optional.empty() + } + } + + /** + * A deleted record takes precedence over a non-deleted record + * while an earlier deletion takes precedence over a later deletion + */ + override fun merge(remote: SignalChatFolderRecord, local: SignalChatFolderRecord, keyGenerator: StorageKeyGenerator): SignalChatFolderRecord { + return if (remote.proto.deletedAtTimestampMs > 0 && local.proto.deletedAtTimestampMs > 0) { + if (remote.proto.deletedAtTimestampMs < local.proto.deletedAtTimestampMs) { + remote + } else { + local + } + } else if (remote.proto.deletedAtTimestampMs > 0) { + remote + } else if (local.proto.deletedAtTimestampMs > 0) { + local + } else { + remote + } + } + + override fun insertLocal(record: SignalChatFolderRecord) { + SignalDatabase.chatFolders.insertChatFolderFromStorageSync(record) + } + + override fun updateLocal(update: StorageRecordUpdate) { + SignalDatabase.chatFolders.updateChatFolderFromStorageSync(update.new) + } + + private fun containsInvalidServiceId(recipients: List): Boolean { + return recipients.any { recipient -> + recipient.contact != null && ServiceId.parseOrNull(recipient.contact!!.serviceId) == null + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncModels.kt b/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncModels.kt index 568c44aea2..fd3e999e39 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncModels.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncModels.kt @@ -5,12 +5,14 @@ import okio.ByteString.Companion.toByteString import org.signal.core.util.isNotEmpty import org.signal.core.util.isNullOrEmpty import org.signal.libsignal.zkgroup.groups.GroupMasterKey +import org.thoughtcrime.securesms.components.settings.app.chats.folders.ChatFolderRecord import org.thoughtcrime.securesms.components.settings.app.usernamelinks.UsernameQrCodeColorScheme import org.thoughtcrime.securesms.conversation.colors.AvatarColor import org.thoughtcrime.securesms.database.GroupTable.ShowAsStoryState import org.thoughtcrime.securesms.database.IdentityTable.VerifiedStatus import org.thoughtcrime.securesms.database.RecipientTable import org.thoughtcrime.securesms.database.RecipientTable.RecipientType +import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.SignalDatabase.Companion.callLinks import org.thoughtcrime.securesms.database.SignalDatabase.Companion.distributionLists import org.thoughtcrime.securesms.database.SignalDatabase.Companion.groups @@ -22,6 +24,7 @@ import org.thoughtcrime.securesms.keyvalue.PhoneNumberPrivacyValues import org.thoughtcrime.securesms.recipients.Recipient import org.whispersystems.signalservice.api.storage.IAPSubscriptionId import org.whispersystems.signalservice.api.storage.SignalCallLinkRecord +import org.whispersystems.signalservice.api.storage.SignalChatFolderRecord import org.whispersystems.signalservice.api.storage.SignalContactRecord import org.whispersystems.signalservice.api.storage.SignalGroupV1Record import org.whispersystems.signalservice.api.storage.SignalGroupV2Record @@ -29,6 +32,7 @@ import org.whispersystems.signalservice.api.storage.SignalStorageRecord import org.whispersystems.signalservice.api.storage.SignalStoryDistributionListRecord import org.whispersystems.signalservice.api.storage.StorageId import org.whispersystems.signalservice.api.storage.toSignalCallLinkRecord +import org.whispersystems.signalservice.api.storage.toSignalChatFolderRecord import org.whispersystems.signalservice.api.storage.toSignalContactRecord import org.whispersystems.signalservice.api.storage.toSignalGroupV1Record import org.whispersystems.signalservice.api.storage.toSignalGroupV2Record @@ -43,10 +47,10 @@ import org.whispersystems.signalservice.internal.storage.protos.GroupV2Record import java.util.Currency import kotlin.math.max import org.whispersystems.signalservice.internal.storage.protos.AvatarColor as RemoteAvatarColor +import org.whispersystems.signalservice.internal.storage.protos.ChatFolderRecord as RemoteChatFolder object StorageSyncModels { - @JvmStatic fun localToRemoteRecord(settings: RecipientRecord): SignalStorageRecord { if (settings.storageId == null) { throw AssertionError("Must have a storage key!") @@ -55,7 +59,6 @@ object StorageSyncModels { return localToRemoteRecord(settings, settings.storageId) } - @JvmStatic fun localToRemoteRecord(settings: RecipientRecord, groupMasterKey: GroupMasterKey): SignalStorageRecord { if (settings.storageId == null) { throw AssertionError("Must have a storage key!") @@ -64,7 +67,6 @@ object StorageSyncModels { return localToRemoteGroupV2(settings, settings.storageId, groupMasterKey).toSignalStorageRecord() } - @JvmStatic fun localToRemoteRecord(settings: RecipientRecord, rawStorageId: ByteArray): SignalStorageRecord { return when (settings.recipientType) { RecipientType.INDIVIDUAL -> localToRemoteContact(settings, rawStorageId).toSignalStorageRecord() @@ -76,6 +78,10 @@ object StorageSyncModels { } } + fun localToRemoteRecord(folder: ChatFolderRecord, rawStorageId: ByteArray): SignalStorageRecord { + return localToRemoteChatFolder(folder, rawStorageId).toSignalStorageRecord() + } + @JvmStatic fun localToRemotePhoneNumberSharingMode(phoneNumberPhoneNumberSharingMode: PhoneNumberPrivacyValues.PhoneNumberSharingMode): AccountRecord.PhoneNumberSharingMode { return when (phoneNumberPhoneNumberSharingMode) { @@ -364,4 +370,48 @@ object StorageSyncModels { AvatarColor.ON_SURFACE_VARIANT -> RemoteAvatarColor.A100 } } + + fun localToRemoteChatFolder(folder: ChatFolderRecord, rawStorageId: ByteArray?): SignalChatFolderRecord { + if (folder.chatFolderId == null) { + throw AssertionError("Chat folder must have a chat folder id.") + } + return SignalChatFolderRecord.newBuilder(folder.storageServiceProto).apply { + identifier = UuidUtil.toByteArray(folder.chatFolderId.uuid).toByteString() + name = folder.name + position = folder.position + showOnlyUnread = folder.showUnread + showMutedChats = folder.showMutedChats + includeAllIndividualChats = folder.showIndividualChats + includeAllGroupChats = folder.showGroupChats + folderType = when (folder.folderType) { + ChatFolderRecord.FolderType.ALL -> RemoteChatFolder.FolderType.ALL + ChatFolderRecord.FolderType.INDIVIDUAL, + ChatFolderRecord.FolderType.GROUP, + ChatFolderRecord.FolderType.UNREAD, + ChatFolderRecord.FolderType.CUSTOM -> RemoteChatFolder.FolderType.CUSTOM + } + includedRecipients = localToRemoteChatFolderRecipients(folder.includedChats) + excludedRecipients = localToRemoteChatFolderRecipients(folder.excludedChats) + deletedAtTimestampMs = folder.deletedTimestampMs + }.build().toSignalChatFolderRecord(StorageId.forChatFolder(rawStorageId)) + } + + private fun localToRemoteChatFolderRecipients(threadIds: List): List { + val recipientIds = SignalDatabase.threads.getRecipientIdsForThreadIds(threadIds) + return recipientIds.mapNotNull { id -> + val recipient = SignalDatabase.recipients.getRecordForSync(id) ?: throw AssertionError("Missing recipient for id") + when (recipient.recipientType) { + RecipientType.INDIVIDUAL -> { + RemoteChatFolder.Recipient(contact = RemoteChatFolder.Recipient.Contact(serviceId = recipient.serviceId?.toString() ?: "", e164 = recipient.e164 ?: "")) + } + RecipientType.GV1 -> { + RemoteChatFolder.Recipient(legacyGroupId = recipient.groupId!!.requireV1().decodedId.toByteString()) + } + RecipientType.GV2 -> { + RemoteChatFolder.Recipient(groupMasterKey = recipient.syncExtras.groupMasterKey!!.serialize().toByteString()) + } + else -> null + } + } + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncValidations.java b/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncValidations.java index 65fcd668fe..104d0aa31a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncValidations.java +++ b/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncValidations.java @@ -151,6 +151,11 @@ public final class StorageSyncValidations { throw new DuplicateCallLinkError(); } + ids = manifest.getStorageIdsByType().get(ManifestRecord.Identifier.Type.CHAT_FOLDER.getValue()); + if (ids.size() != new HashSet<>(ids).size()) { + throw new DuplicateChatFolderError(); + } + throw new DuplicateRawIdAcrossTypesError(); } @@ -206,6 +211,9 @@ public final class StorageSyncValidations { private static final class DuplicateCallLinkError extends Error { } + private static final class DuplicateChatFolderError extends Error { + } + private static final class DuplicateInsertInWriteError extends Error { } diff --git a/app/src/main/protowire/Backup.proto b/app/src/main/protowire/Backup.proto index e40652f96f..be94e934ea 100644 --- a/app/src/main/protowire/Backup.proto +++ b/app/src/main/protowire/Backup.proto @@ -1311,4 +1311,5 @@ message ChatFolder { FolderType folderType = 6; repeated uint64 includedRecipientIds = 7; // generated recipient id of groups, contacts, and/or note to self repeated uint64 excludedRecipientIds = 8; // generated recipient id of groups, contacts, and/or note to self + bytes id = 9; // should be 16 bytes } \ No newline at end of file diff --git a/app/src/spinner/java/org/thoughtcrime/securesms/StorageServicePlugin.kt b/app/src/spinner/java/org/thoughtcrime/securesms/StorageServicePlugin.kt index 5d1f3bb0de..b7f272151a 100644 --- a/app/src/spinner/java/org/thoughtcrime/securesms/StorageServicePlugin.kt +++ b/app/src/spinner/java/org/thoughtcrime/securesms/StorageServicePlugin.kt @@ -49,6 +49,9 @@ class StorageServicePlugin : Plugin { } else if (record.proto.callLink != null) { row += "Call Link" row += record.proto.callLink.toString() + } else if (record.proto.chatFolder != null) { + row += "Chat Folder" + row += record.proto.chatFolder.toString() } else { row += "Unknown" row += "" diff --git a/app/src/test/java/org/thoughtcrime/securesms/storage/ChatFolderRecordProcessorTest.kt b/app/src/test/java/org/thoughtcrime/securesms/storage/ChatFolderRecordProcessorTest.kt new file mode 100644 index 0000000000..9aa6100d24 --- /dev/null +++ b/app/src/test/java/org/thoughtcrime/securesms/storage/ChatFolderRecordProcessorTest.kt @@ -0,0 +1,216 @@ +package org.thoughtcrime.securesms.storage + +import okio.ByteString.Companion.toByteString +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.BeforeClass +import org.junit.Test +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.testutil.EmptyLogger +import org.whispersystems.signalservice.api.storage.SignalChatFolderRecord +import org.whispersystems.signalservice.api.storage.StorageId +import org.whispersystems.signalservice.api.util.UuidUtil +import org.whispersystems.signalservice.internal.storage.protos.ChatFolderRecord +import java.util.UUID + +/** + * Tests for [ChatFolderRecordProcessor] + */ +class ChatFolderRecordProcessorTest { + companion object { + val STORAGE_ID: StorageId = StorageId.forChatFolder(byteArrayOf(1, 2, 3, 4)) + + @JvmStatic + @BeforeClass + fun setUpClass() { + Log.initialize(EmptyLogger()) + } + } + + private val testSubject = ChatFolderRecordProcessor() + + @Test + fun `Given a valid proto with a known name and folder type, assert valid`() { + // GIVEN + val proto = ChatFolderRecord.Builder().apply { + identifier = UuidUtil.toByteArray(UUID.randomUUID()).toByteString() + name = "Folder1" + position = 1 + showOnlyUnread = false + showMutedChats = false + includeAllIndividualChats = false + includeAllGroupChats = false + folderType = ChatFolderRecord.FolderType.CUSTOM + deletedAtTimestampMs = 0L + }.build() + val record = SignalChatFolderRecord(STORAGE_ID, proto) + + // WHEN + val result = testSubject.isInvalid(record) + + // THEN + assertFalse(result) + } + + @Test + fun `Given an invalid proto with no name, assert invalid`() { + // GIVEN + val proto = ChatFolderRecord.Builder().apply { + identifier = UuidUtil.toByteArray(UUID.randomUUID()).toByteString() + name = "" + position = 1 + showOnlyUnread = false + showMutedChats = false + includeAllIndividualChats = false + includeAllGroupChats = false + folderType = ChatFolderRecord.FolderType.CUSTOM + deletedAtTimestampMs = 0L + }.build() + val record = SignalChatFolderRecord(STORAGE_ID, proto) + + // WHEN + val result = testSubject.isInvalid(record) + + // THEN + assertTrue(result) + } + + @Test + fun `Given a valid proto with no folder type, assert invalid`() { + // GIVEN + val proto = ChatFolderRecord.Builder().apply { + identifier = UuidUtil.toByteArray(UUID.randomUUID()).toByteString() + name = "Folder1" + position = 1 + showOnlyUnread = false + showMutedChats = false + includeAllIndividualChats = false + includeAllGroupChats = false + folderType = ChatFolderRecord.FolderType.UNKNOWN + deletedAtTimestampMs = 0L + }.build() + val record = SignalChatFolderRecord(STORAGE_ID, proto) + + // WHEN + val result = testSubject.isInvalid(record) + + // THEN + assertTrue(result) + } + + @Test + fun `Given a valid proto with a deleted timestamp and negative position, assert valid`() { + // GIVEN + val proto = ChatFolderRecord.Builder().apply { + identifier = UuidUtil.toByteArray(UUID.randomUUID()).toByteString() + name = "Folder1" + position = -1 + showOnlyUnread = false + showMutedChats = false + includeAllIndividualChats = false + includeAllGroupChats = false + folderType = ChatFolderRecord.FolderType.CUSTOM + deletedAtTimestampMs = 100L + }.build() + val record = SignalChatFolderRecord(STORAGE_ID, proto) + + // WHEN + val result = testSubject.isInvalid(record) + + // THEN + assertFalse(result) + } + + @Test + fun `Given an invalid proto with a deleted timestamp and positive position, assert invalid`() { + // GIVEN + val proto = ChatFolderRecord.Builder().apply { + identifier = UuidUtil.toByteArray(UUID.randomUUID()).toByteString() + name = "Folder1" + position = 1 + showOnlyUnread = false + showMutedChats = false + includeAllIndividualChats = false + includeAllGroupChats = false + folderType = ChatFolderRecord.FolderType.CUSTOM + deletedAtTimestampMs = 100L + }.build() + val record = SignalChatFolderRecord(STORAGE_ID, proto) + + // WHEN + val result = testSubject.isInvalid(record) + + // THEN + assertTrue(result) + } + + @Test + fun `Given an invalid proto with a negative position, assert invalid`() { + // GIVEN + val proto = ChatFolderRecord.Builder().apply { + identifier = UuidUtil.toByteArray(UUID.randomUUID()).toByteString() + name = "Folder1" + position = -1 + showOnlyUnread = false + showMutedChats = false + includeAllIndividualChats = false + includeAllGroupChats = false + folderType = ChatFolderRecord.FolderType.CUSTOM + deletedAtTimestampMs = 0L + }.build() + val record = SignalChatFolderRecord(STORAGE_ID, proto) + + // WHEN + val result = testSubject.isInvalid(record) + + // THEN + assertTrue(result) + } + + @Test + fun `Given an invalid proto with a bad id, assert invalid`() { + // GIVEN + val proto = ChatFolderRecord.Builder().apply { + identifier = "bad".toByteArray().toByteString() + name = "Folder1" + position = -1 + showOnlyUnread = false + showMutedChats = false + includeAllIndividualChats = false + includeAllGroupChats = false + folderType = ChatFolderRecord.FolderType.CUSTOM + deletedAtTimestampMs = 0L + }.build() + val record = SignalChatFolderRecord(STORAGE_ID, proto) + + // WHEN + val result = testSubject.isInvalid(record) + + // THEN + assertTrue(result) + } + + @Test + fun `Given an invalid proto with a bad recipient, assert invalid`() { + // GIVEN + val proto = ChatFolderRecord.Builder().apply { + identifier = UuidUtil.toByteArray(UUID.randomUUID()).toByteString() + name = "Folder1" + position = -1 + showOnlyUnread = false + showMutedChats = false + includeAllIndividualChats = false + includeAllGroupChats = false + folderType = ChatFolderRecord.FolderType.CUSTOM + deletedAtTimestampMs = 0L + includedRecipients = listOf(ChatFolderRecord.Recipient(contact = ChatFolderRecord.Recipient.Contact(serviceId = "bad"))) + }.build() + val record = SignalChatFolderRecord(STORAGE_ID, proto) + + // WHEN + val result = testSubject.isInvalid(record) + + // THEN + assertTrue(result) + } +} diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalChatFolderRecord.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalChatFolderRecord.kt new file mode 100644 index 0000000000..e006952a10 --- /dev/null +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalChatFolderRecord.kt @@ -0,0 +1,27 @@ +package org.whispersystems.signalservice.api.storage + +import org.whispersystems.signalservice.internal.storage.protos.ChatFolderRecord +import java.io.IOException + +/** + * Wrapper around a [ChatFolderRecord] to pair it with a [StorageId]. + */ +data class SignalChatFolderRecord( + override val id: StorageId, + override val proto: ChatFolderRecord +) : SignalRecord { + + companion object { + fun newBuilder(serializedUnknowns: ByteArray?): ChatFolderRecord.Builder { + return serializedUnknowns?.let { builderFromUnknowns(it) } ?: ChatFolderRecord.Builder() + } + + private fun builderFromUnknowns(serializedUnknowns: ByteArray): ChatFolderRecord.Builder { + return try { + ChatFolderRecord.ADAPTER.decode(serializedUnknowns).newBuilder() + } catch (e: IOException) { + ChatFolderRecord.Builder() + } + } + } +} diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageRecord.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageRecord.kt index 48093a3a00..644c6d65a9 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageRecord.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageRecord.kt @@ -10,7 +10,7 @@ data class SignalStorageRecord( val proto: StorageRecord ) { val isUnknown: Boolean - get() = proto.contact == null && proto.groupV1 == null && proto.groupV2 == null && proto.account == null && proto.storyDistributionList == null && proto.callLink == null + get() = proto.contact == null && proto.groupV1 == null && proto.groupV2 == null && proto.account == null && proto.storyDistributionList == null && proto.callLink == null && proto.chatFolder == null companion object { @JvmStatic diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageId.java b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageId.java index 702533dedf..79417b3669 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageId.java +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageId.java @@ -38,6 +38,10 @@ public class StorageId { return new StorageId(ManifestRecord.Identifier.Type.CALL_LINK.getValue(), Preconditions.checkNotNull(raw)); } + public static StorageId forChatFolder(byte[] raw) { + return new StorageId(ManifestRecord.Identifier.Type.CHAT_FOLDER.getValue(), Preconditions.checkNotNull(raw)); + } + public static StorageId forType(byte[] raw, int type) { return new StorageId(type, raw); } diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageRecordConverters.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageRecordConverters.kt index d427d0e2e4..b58acd56e8 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageRecordConverters.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageRecordConverters.kt @@ -7,6 +7,7 @@ package org.whispersystems.signalservice.api.storage import org.whispersystems.signalservice.internal.storage.protos.AccountRecord import org.whispersystems.signalservice.internal.storage.protos.CallLinkRecord +import org.whispersystems.signalservice.internal.storage.protos.ChatFolderRecord import org.whispersystems.signalservice.internal.storage.protos.ContactRecord import org.whispersystems.signalservice.internal.storage.protos.GroupV1Record import org.whispersystems.signalservice.internal.storage.protos.GroupV2Record @@ -41,6 +42,10 @@ fun CallLinkRecord.toSignalCallLinkRecord(storageId: StorageId): SignalCallLinkR return SignalCallLinkRecord(storageId, this) } +fun ChatFolderRecord.toSignalChatFolderRecord(storageId: StorageId): SignalChatFolderRecord { + return SignalChatFolderRecord(storageId, this) +} + fun SignalContactRecord.toSignalStorageRecord(): SignalStorageRecord { return SignalStorageRecord(id, StorageRecord(contact = this.proto)) } @@ -64,3 +69,7 @@ fun SignalStoryDistributionListRecord.toSignalStorageRecord(): SignalStorageReco fun SignalCallLinkRecord.toSignalStorageRecord(): SignalStorageRecord { return SignalStorageRecord(id, StorageRecord(callLink = this.proto)) } + +fun SignalChatFolderRecord.toSignalStorageRecord(): SignalStorageRecord { + return SignalStorageRecord(id, StorageRecord(chatFolder = this.proto)) +} diff --git a/libsignal-service/src/main/protowire/StorageService.proto b/libsignal-service/src/main/protowire/StorageService.proto index 04046e825b..de8517f675 100644 --- a/libsignal-service/src/main/protowire/StorageService.proto +++ b/libsignal-service/src/main/protowire/StorageService.proto @@ -51,6 +51,7 @@ message ManifestRecord { ACCOUNT = 4; STORY_DISTRIBUTION_LIST = 5; CALL_LINK = 7; + CHAT_FOLDER = 8; } bytes raw = 1; @@ -72,6 +73,7 @@ message StorageRecord { AccountRecord account = 4; StoryDistributionListRecord storyDistributionList = 5; CallLinkRecord callLink = 7; + ChatFolderRecord chatFolder = 8; } } @@ -280,4 +282,38 @@ message CallLinkRecord { bytes rootKey = 1; bytes adminPasskey = 2; uint64 deletedAtTimestampMs = 3; -} \ No newline at end of file +} + +message ChatFolderRecord { + message Recipient { + message Contact { + string serviceId = 1; + string e164 = 2; + } + + oneof identifier { + Contact contact = 1; + bytes legacyGroupId = 2; + bytes groupMasterKey = 3; + } + } + + // Represents the default "All chats" folder record vs all other custom folders + enum FolderType { + UNKNOWN = 0; + ALL = 1; + CUSTOM = 2; + } + + bytes identifier = 1; + string name = 2; + uint32 position = 3; + bool showOnlyUnread = 4; + bool showMutedChats = 5; + bool includeAllIndividualChats = 6; // Folder includes all 1:1 chats, unless excluded + bool includeAllGroupChats = 7; // Folder includes all group chats, unless excluded + FolderType folderType = 8; + repeated Recipient includedRecipients = 9; + repeated Recipient excludedRecipients = 10; + uint64 deletedAtTimestampMs = 11; // When non-zero, `position` should be set to -1 and includedRecipients should be empty +}