diff --git a/app/src/main/java/org/thoughtcrime/securesms/ApplicationContext.java b/app/src/main/java/org/thoughtcrime/securesms/ApplicationContext.java index 9efd93de10..1ed56b89d3 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/ApplicationContext.java +++ b/app/src/main/java/org/thoughtcrime/securesms/ApplicationContext.java @@ -34,6 +34,7 @@ import org.signal.core.util.concurrent.AnrDetector; import org.signal.core.util.concurrent.SignalExecutors; import org.signal.core.util.logging.AndroidLogger; import org.signal.core.util.logging.Log; +import org.signal.core.util.logging.Scrubber; import org.signal.core.util.tracing.Tracer; import org.signal.glide.SignalGlideCodecs; import org.signal.libsignal.protocol.logging.SignalProtocolLoggerProvider; @@ -159,6 +160,7 @@ public class ApplicationContext extends MultiDexApplication implements AppForegr .addBlocking("rx-init", this::initializeRx) .addBlocking("event-bus", () -> EventBus.builder().logNoSubscriberMessages(false).installDefaultEventBus()) .addBlocking("app-dependencies", this::initializeAppDependencies) + .addBlocking("scrubber", () -> Scrubber.setIdentifierHmacKeyProvider(() -> SignalStore.svr().getOrCreateMasterKey().deriveLoggingKey())) .addBlocking("first-launch", this::initializeFirstEverAppLaunch) .addBlocking("app-migrations", this::initializeApplicationMigrations) .addBlocking("mark-registration", () -> RegistrationUtil.maybeMarkRegistrationComplete()) diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/RecipientTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/RecipientTable.kt index 4e696e4cf7..f3c7b8af5f 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/RecipientTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/RecipientTable.kt @@ -473,9 +473,7 @@ open class RecipientTable(context: Context, databaseHelper: SignalDatabase) : Da result = processPnpTuple(e164 = e164, pni = pni, aci = aci, pniVerified = pniVerified, changeSelf = changeSelf) if (result.operations.isNotEmpty() || result.requiredInsert) { - val pniString = if (pni == null) "null" else if (aci == null && e164 == null) pni.toString() else "" - val e164String = if (e164 == null) "null" else if (aci == null) e164 else "" - Log.i(TAG, "[getAndPossiblyMerge] ($aci, $pniString, $e164String) BreadCrumbs: ${result.breadCrumbs}, Operations: ${result.operations}, RequiredInsert: ${result.requiredInsert}, FinalId: ${result.finalId}") + Log.i(TAG, "[getAndPossiblyMerge] ($aci, $pni, $e164) BreadCrumbs: ${result.breadCrumbs}, Operations: ${result.operations}, RequiredInsert: ${result.requiredInsert}, FinalId: ${result.finalId}") } db.runPostSuccessfulTransaction { diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/crypto/CryptoUtil.java b/core-util-jvm/src/main/java/org/signal/core/util/CryptoUtil.java similarity index 87% rename from libsignal-service/src/main/java/org/whispersystems/signalservice/api/crypto/CryptoUtil.java rename to core-util-jvm/src/main/java/org/signal/core/util/CryptoUtil.java index b0f023cfe5..acad95f3b6 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/crypto/CryptoUtil.java +++ b/core-util-jvm/src/main/java/org/signal/core/util/CryptoUtil.java @@ -1,4 +1,9 @@ -package org.whispersystems.signalservice.api.crypto; +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util; import java.security.InvalidKeyException; import java.security.MessageDigest; diff --git a/core-util-jvm/src/main/java/org/signal/core/util/logging/Scrubber.kt b/core-util-jvm/src/main/java/org/signal/core/util/logging/Scrubber.kt index aab3c0ae76..d7c784f17f 100644 --- a/core-util-jvm/src/main/java/org/signal/core/util/logging/Scrubber.kt +++ b/core-util-jvm/src/main/java/org/signal/core/util/logging/Scrubber.kt @@ -5,6 +5,8 @@ package org.signal.core.util.logging +import org.signal.core.util.CryptoUtil +import org.signal.core.util.Hex import java.util.regex.Matcher import java.util.regex.Pattern @@ -20,10 +22,8 @@ object Scrubber { * Supposedly, the shortest international phone numbers in use contain seven digits. * Handles URL encoded +, %2B */ - private val E164_PATTERN = Pattern.compile("(\\+|%2B)(\\d{5,13})(\\d{2})") - private const val E164_CENSOR = "*************" - - private val E164_ZERO_PATTERN = Pattern.compile("\\b(0)(\\d{8})(\\d{2})\\b") + private val E164_PATTERN = Pattern.compile("(\\+|%2B)(\\d{7,15})") + private val E164_ZERO_PATTERN = Pattern.compile("\\b0(\\d{10})\\b") /** The second group will be censored.*/ private val CRUDE_EMAIL_PATTERN = Pattern.compile("\\b([^\\s/])([^\\s/]*@[^\\s]+)") @@ -41,6 +41,8 @@ object Scrubber { private val UUID_PATTERN = Pattern.compile("(JOB::)?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{9})([0-9a-f]{3})", Pattern.CASE_INSENSITIVE) private const val UUID_CENSOR = "********-****-****-****-*********" + private val PNI_PATTERN = Pattern.compile("PNI:([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{9}[0-9a-f]{3})", Pattern.CASE_INSENSITIVE) + /** * The entire string is censored. Note: left as concatenated strings because kotlin string literals leave trailing newlines, and removing them breaks * syntax highlighting. @@ -75,6 +77,14 @@ object Scrubber { private val CALL_LINK_PATTERN = Pattern.compile("([bBcCdDfFgGhHkKmMnNpPqQrRsStTxXzZ]{4})(-[bBcCdDfFgGhHkKmMnNpPqQrRsStTxXzZ]{4}){7}") private const val CALL_LINK_CENSOR_SUFFIX = "-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX" + @JvmStatic + @Volatile + var identifierHmacKeyProvider: () -> ByteArray? = { null } + + @JvmStatic + @Volatile + private var identifierHmacKey: ByteArray? = null + @JvmStatic fun scrub(input: CharSequence): CharSequence { return input @@ -83,6 +93,7 @@ object Scrubber { .scrubEmail() .scrubGroupsV1() .scrubGroupsV2() + .scrubPnis() .scrubUuids() .scrubDomains() .scrubIpv4() @@ -93,18 +104,16 @@ object Scrubber { private fun CharSequence.scrubE164(): CharSequence { return scrub(this, E164_PATTERN) { matcher, output -> output - .append(matcher.group(1)) - .append(E164_CENSOR, 0, matcher.group(2)!!.length) - .append(matcher.group(3)) + .append("E164:") + .append(hash(matcher.group(2))) } } private fun CharSequence.scrubE164Zero(): CharSequence { return scrub(this, E164_ZERO_PATTERN) { matcher, output -> output - .append(matcher.group(1)) - .append(E164_CENSOR, 0, matcher.group(2)!!.length) - .append(matcher.group(3)) + .append("E164:") + .append(hash(matcher.group(1))) } } @@ -134,6 +143,14 @@ object Scrubber { } } + private fun CharSequence.scrubPnis(): CharSequence { + return scrub(this, PNI_PATTERN) { matcher, output -> + output + .append("PNI:") + .append(hash(matcher.group(1))) + } + } + private fun CharSequence.scrubUuids(): CharSequence { return scrub(this, UUID_PATTERN) { matcher, output -> if (matcher.group(1) != null && matcher.group(1)!!.isNotEmpty()) { @@ -198,4 +215,14 @@ object Scrubber { output } } + + private fun hash(value: String): String { + if (identifierHmacKey == null) { + identifierHmacKey = identifierHmacKeyProvider() + } + + val key: ByteArray = identifierHmacKey ?: return "" + val hash = CryptoUtil.hmacSha256(key, value.toByteArray()) + return "<${Hex.toStringCondensed(hash).take(5)}>" + } } diff --git a/core-util-jvm/src/test/java/org/signal/core/util/logging/ScrubberTest.kt b/core-util-jvm/src/test/java/org/signal/core/util/logging/ScrubberTest.kt index 1203c513cc..f16323fbec 100644 --- a/core-util-jvm/src/test/java/org/signal/core/util/logging/ScrubberTest.kt +++ b/core-util-jvm/src/test/java/org/signal/core/util/logging/ScrubberTest.kt @@ -6,37 +6,45 @@ package org.signal.core.util.logging import org.junit.Assert +import org.junit.BeforeClass import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.Parameterized @RunWith(Parameterized::class) class ScrubberTest(private val input: String, private val expected: String) { + @Test fun scrub() { Assert.assertEquals(expected, Scrubber.scrub(input).toString()) } companion object { + @JvmStatic + @BeforeClass + fun setup() { + Scrubber.identifierHmacKeyProvider = { ByteArray(32) } + } + @JvmStatic @Parameterized.Parameters fun data(): Iterable> { return listOf( arrayOf( "An E164 number +15551234567", - "An E164 number +*********67" + "An E164 number E164:<9f683>" ), arrayOf( "A UK number +447700900000", - "A UK number +**********00" + "A UK number E164:" ), arrayOf( "A Japanese number 08011112222", - "A Japanese number 0********22" + "A Japanese number E164:" ), arrayOf( "A Japanese number (08011112222)", - "A Japanese number (0********22)" + "A Japanese number (E164:)" ), arrayOf( "Not a Japanese number 08011112222333344445555", @@ -48,11 +56,11 @@ class ScrubberTest(private val input: String, private val expected: String) { ), arrayOf( "An avatar filename: file:///data/user/0/org.thoughtcrime.securesms/files/avatars/%2B447700900099", - "An avatar filename: file:///data/user/0/org.thoughtcrime.securesms/files/avatars/%2B**********99" + "An avatar filename: file:///data/user/0/org.thoughtcrime.securesms/files/avatars/E164:<3106a>" ), arrayOf( "Multiple numbers +447700900001 +447700900002", - "Multiple numbers +**********01 +**********02" + "Multiple numbers E164:<87035> E164:<1e488>" ), arrayOf( "One less than shortest number +155556", @@ -60,15 +68,15 @@ class ScrubberTest(private val input: String, private val expected: String) { ), arrayOf( "Shortest number +1555567", - "Shortest number +*****67" + "Shortest number E164:<8edd2>" ), arrayOf( "Longest number +155556789012345", - "Longest number +*************45" + "Longest number E164:<90596>" ), arrayOf( "One more than longest number +1234567890123456", - "One more than longest number +*************456" + "One more than longest number E164:<78d5b>6" ), arrayOf( "abc@def.com", @@ -92,7 +100,7 @@ class ScrubberTest(private val input: String, private val expected: String) { ), arrayOf( "An email and a number abc@def.com +155556789012345", - "An email and a number a...@... +*************45" + "An email and a number a...@... E164:<90596>" ), arrayOf( "__textsecure_group__!000102030405060708090a0b0c0d0e0f", @@ -118,13 +126,21 @@ class ScrubberTest(private val input: String, private val expected: String) { "A UUID a37cb654-c9e0-4c1e-93df-3d11ca3c97f4 surrounded with text", "A UUID ********-****-****-****-*********7f4 surrounded with text" ), + arrayOf( + "An ACI:a37cb654-c9e0-4c1e-93df-3d11ca3c97f4 surrounded with text", + "An ACI:********-****-****-****-*********7f4 surrounded with text" + ), + arrayOf( + "A PNI:a37cb654-c9e0-4c1e-93df-3d11ca3c97f4 surrounded with text", + "A PNI: surrounded with text" + ), arrayOf( "JOB::a37cb654-c9e0-4c1e-93df-3d11ca3c97f4", "JOB::a37cb654-c9e0-4c1e-93df-3d11ca3c97f4" ), arrayOf( - "All patterns in a row __textsecure_group__!abcdefg1234567890 +1234567890123456 abc@def.com a37cb654-c9e0-4c1e-93df-3d11ca3c97f4 nl.motorsport.com 192.168.1.1 with text after", - "All patterns in a row __...group...90 +*************456 a...@... ********-****-****-****-*********7f4 ***.com ...ipv4... with text after" + "All patterns in a row __textsecure_group__!abcdefg1234567890 +123456789012345 abc@def.com a37cb654-c9e0-4c1e-93df-3d11ca3c97f4 nl.motorsport.com 192.168.1.1 with text after", + "All patterns in a row __...group...90 E164:<78d5b> a...@... ********-****-****-****-*********7f4 ***.com ...ipv4... with text after" ), arrayOf( "java.net.UnknownServiceException: CLEARTEXT communication to nl.motorsport.com not permitted by network security policy", diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/crypto/HmacSIV.java b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/crypto/HmacSIV.java index 603466bf14..11596a0b28 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/crypto/HmacSIV.java +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/crypto/HmacSIV.java @@ -4,7 +4,7 @@ import org.whispersystems.util.StringUtil; import java.util.Arrays; -import static org.whispersystems.signalservice.api.crypto.CryptoUtil.hmacSha256; +import static org.signal.core.util.CryptoUtil.hmacSha256; import static org.whispersystems.util.ByteArrayUtil.concat; import static org.whispersystems.util.ByteArrayUtil.xor; import static java.util.Arrays.copyOfRange; diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/kbs/MasterKey.java b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/kbs/MasterKey.java index c4708c7b23..ca60a0f5c7 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/kbs/MasterKey.java +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/kbs/MasterKey.java @@ -8,7 +8,7 @@ import org.whispersystems.util.StringUtil; import java.security.SecureRandom; import java.util.Arrays; -import static org.whispersystems.signalservice.api.crypto.CryptoUtil.hmacSha256; +import static org.signal.core.util.CryptoUtil.hmacSha256; public final class MasterKey { @@ -40,6 +40,10 @@ public final class MasterKey { return new StorageKey(derive("Storage Service Encryption")); } + public byte[] deriveLoggingKey() { + return derive("Logging Key"); + } + private byte[] derive(String keyName) { return hmacSha256(masterKey, StringUtil.utf8(keyName)); } diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageKey.java b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageKey.java index 9e3ea5df32..12bca1393b 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageKey.java +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageKey.java @@ -6,7 +6,7 @@ import org.whispersystems.util.StringUtil; import java.util.Arrays; -import static org.whispersystems.signalservice.api.crypto.CryptoUtil.hmacSha256; +import static org.signal.core.util.CryptoUtil.hmacSha256; /** * Key used to encrypt data on the storage service. Not used directly -- instead we used keys that diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/contacts/crypto/ContactDiscoveryCipher.java b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/contacts/crypto/ContactDiscoveryCipher.java index 6e53a63747..46d42502c8 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/contacts/crypto/ContactDiscoveryCipher.java +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/contacts/crypto/ContactDiscoveryCipher.java @@ -1,7 +1,7 @@ package org.whispersystems.signalservice.internal.contacts.crypto; import org.signal.libsignal.protocol.util.ByteUtil; -import org.whispersystems.signalservice.api.crypto.CryptoUtil; +import org.signal.core.util.CryptoUtil; import org.whispersystems.signalservice.api.crypto.InvalidCiphertextException; import org.whispersystems.signalservice.internal.contacts.crypto.AESCipher.AESEncryptedResult; import org.whispersystems.signalservice.internal.contacts.entities.DiscoveryRequest;