From 88d2d4d9c7837b577dee5a7069fb9f72727ead44 Mon Sep 17 00:00:00 2001 From: gram-signal <84339875+gram-signal@users.noreply.github.com> Date: Tue, 22 Feb 2022 15:30:42 -0700 Subject: [PATCH] Switch from binary to streaming protos when using CDSHv1. Co-authored-by: Greyson Parrelli --- app/build.gradle | 2 +- .../api/services/CdshService.java | 111 ++++++++++++------ libsignal/service/src/main/proto/CDSH.proto | 55 +++++++++ 3 files changed, 132 insertions(+), 36 deletions(-) create mode 100644 libsignal/service/src/main/proto/CDSH.proto diff --git a/app/build.gradle b/app/build.gradle index 8b6eddbfbe..69182dc6ca 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -188,7 +188,7 @@ android { buildConfigField "int", "CONTENT_PROXY_PORT", "443" buildConfigField "String", "SIGNAL_AGENT", "\"OWA\"" buildConfigField "String", "CDSH_PUBLIC_KEY", "\"2fe57da347cd62431528daac5fbb290730fff684afc4cfc2ed90995f58cb3b74\"" - buildConfigField "String", "CDSH_CODE_HASH", "\"ec58c0d7561de8d5657f3a4b22a635eaa305204e9359dcc80a99dfd0c5f1cbf2\"" + buildConfigField "String", "CDSH_CODE_HASH", "\"2f79dc6c1599b71c70fc2d14f3ea2e3bc65134436eb87011c88845b137af673a\"" buildConfigField "String", "CDS_MRENCLAVE", "\"c98e00a4e3ff977a56afefe7362a27e4961e4f19e211febfbb19b897e6b80b15\"" buildConfigField "org.thoughtcrime.securesms.KbsEnclave", "KBS_ENCLAVE", "new org.thoughtcrime.securesms.KbsEnclave(\"0cedba03535b41b67729ce9924185f831d7767928a1d1689acb689bc079c375f\", " + "\"187d2739d22be65e74b65f0055e74d31310e4267e5fac2b1246cc8beba81af39\", " + diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/services/CdshService.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/services/CdshService.java index cbe26e217e..e8a6c6f7a2 100644 --- a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/services/CdshService.java +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/services/CdshService.java @@ -1,5 +1,9 @@ package org.whispersystems.signalservice.api.services; +import com.google.protobuf.ByteString; + +import org.signal.cds.ClientRequest; +import org.signal.cds.ClientResponse; import org.signal.libsignal.hsmenclave.HsmEnclaveClient; import org.whispersystems.libsignal.logging.Log; import org.whispersystems.libsignal.util.ByteUtil; @@ -13,14 +17,17 @@ import org.whispersystems.signalservice.internal.configuration.SignalServiceConf import org.whispersystems.signalservice.internal.util.BlacklistingTrustManager; import org.whispersystems.signalservice.internal.util.Hex; import org.whispersystems.signalservice.internal.util.Util; +import org.whispersystems.util.Base64; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -43,7 +50,6 @@ import okhttp3.Request; import okhttp3.Response; import okhttp3.WebSocket; import okhttp3.WebSocketListener; -import okio.ByteString; /** * Handles network interactions with CDSH, the HSM-backed CDS service. @@ -52,7 +58,10 @@ public final class CdshService { private static final String TAG = CdshService.class.getSimpleName(); - private static final int VERSION = 1; + private static final int VERSION = 1; + private static final int MAX_E164S_PER_REQUEST = 5000; + private static final UUID EMPTY_ACI = new UUID(0, 0); + private static final int RESPONSE_ITEM_SIZE = 8 + 16 + 16; // 1 uint64 + 2 UUIDs private final OkHttpClient client; private final HsmEnclaveClient enclave; @@ -87,32 +96,35 @@ public final class CdshService { return Single.create(emitter -> { AtomicReference stage = new AtomicReference<>(Stage.WAITING_TO_INITIALIZE); List addressBook = e164Numbers.stream().map(e -> e.substring(1)).collect(Collectors.toList()); + final Map out = new HashMap<>(); String url = String.format("%s/discovery/%s/%s", baseUrl, hexPublicKey, hexCodeHash); - Request request = new Request.Builder().url(url).build(); + Request request = new Request.Builder() + .url(url) + .addHeader("Authorization", basicAuth(username, password)) + .build(); + WebSocket webSocket = client.newWebSocket(request, new WebSocketListener() { @Override - public void onMessage(WebSocket webSocket, ByteString bytes) { + public void onMessage(WebSocket webSocket, okio.ByteString bytes) { switch (stage.get()) { case WAITING_TO_INITIALIZE: enclave.completeHandshake(bytes.toByteArray()); - byte[] request = enclave.establishedSend(buildPlaintextRequest(username, password, addressBook)); - stage.set(Stage.WAITING_FOR_RESPONSE); - webSocket.send(ByteString.of(request)); + for (byte[] request : buildPlaintextRequests(addressBook)) { + webSocket.send(okio.ByteString.of(enclave.establishedSend(request))); + } break; case WAITING_FOR_RESPONSE: - byte[] response = enclave.establishedRecv(bytes.toByteArray()); + byte[] rawResponse = enclave.establishedRecv(bytes.toByteArray()); try { - Map out = parseResponse(addressBook, response); - emitter.onSuccess(ServiceResponse.forResult(out, 200, null)); + ClientResponse clientResponse = ClientResponse.parseFrom(rawResponse); + addClientResponseToOutput(clientResponse, out); } catch (IOException e) { emitter.onSuccess(ServiceResponse.forUnknownError(e)); - } finally { - webSocket.close(1000, "OK"); } break; @@ -125,7 +137,9 @@ public final class CdshService { @Override public void onClosing(WebSocket webSocket, int code, String reason) { - if (code != 1000) { + if (code == 1000) { + emitter.onSuccess(ServiceResponse.forResult(out, 200, null)); + } else { Log.w(TAG, "Remote side is closing with non-normal code " + code); webSocket.close(1000, "Remote closed with code " + code); stage.set(Stage.FAILURE); @@ -141,41 +155,68 @@ public final class CdshService { } }); - webSocket.send(ByteString.of(enclave.initialRequest())); + webSocket.send(okio.ByteString.of(enclave.initialRequest())); emitter.setCancellable(() -> webSocket.close(1000, "OK")); }); } - private static byte[] buildPlaintextRequest(String username, String password, List addressBook) { - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { - outputStream.write(VERSION); - outputStream.write(username.getBytes(StandardCharsets.UTF_8)); - outputStream.write(password.getBytes(StandardCharsets.UTF_8)); + private static void addClientResponseToOutput(ClientResponse responsePB, Map out) { + ByteBuffer parser = responsePB.getE164PniAciTriples().asReadOnlyByteBuffer(); + while (parser.remaining() >= RESPONSE_ITEM_SIZE) { + String e164 = "+" + parser.getLong(); + UUID unusedPni = new UUID(parser.getLong(), parser.getLong()); + UUID aci = new UUID(parser.getLong(), parser.getLong()); - for (String e164 : addressBook) { - outputStream.write(ByteUtil.longToByteArray(Long.parseLong(e164))); + if (!aci.equals(EMPTY_ACI)) { + out.put(e164, ACI.from(aci)); } - - return outputStream.toByteArray(); - } catch (IOException e) { - throw new AssertionError("Failed to write bytes to the output stream?"); } } - private static Map parseResponse(List addressBook, byte[] plaintextResponse) throws IOException { - Map results = new HashMap<>(); + private String basicAuth(String username, String password) { + return "Basic " + Base64.encodeBytes((username + ":" + password).getBytes(StandardCharsets.UTF_8)); + } - try (DataInputStream uuidInputStream = new DataInputStream(new ByteArrayInputStream(plaintextResponse))) { - for (String candidate : addressBook) { - long candidateUuidHigh = uuidInputStream.readLong(); - long candidateUuidLow = uuidInputStream.readLong(); - if (candidateUuidHigh != 0 || candidateUuidLow != 0) { - results.put('+' + candidate, ACI.from(new UUID(candidateUuidHigh, candidateUuidLow))); - } + private static byte[] e164sToRequest(ByteString e164s, boolean more) { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + outputStream.write(VERSION); + ClientRequest.newBuilder() + .setNewE164S(e164s) + .setHasMore(more) + .build() + .writeTo(outputStream); + return outputStream.toByteArray(); + } catch (IOException e) { + throw new AssertionError("Failed to write protobuf to the output stream?"); + } + } + + private static List buildPlaintextRequests(List addressBook) { + List out = new ArrayList<>((addressBook.size() / MAX_E164S_PER_REQUEST) + 1); + ByteString.Output e164Page = ByteString.newOutput(); + int pageSize = 0; + + for (String address : addressBook) { + if (pageSize >= MAX_E164S_PER_REQUEST) { + pageSize = 0; + out.add(e164sToRequest(e164Page.toByteString(), true)); + e164Page = ByteString.newOutput(); } + + try { + e164Page.write(ByteUtil.longToByteArray(Long.parseLong(address))); + } catch (IOException e) { + throw new AssertionError("Failed to write long to ByteString", e); + } + + pageSize++; } - return results; + if (pageSize > 0) { + out.add(e164sToRequest(e164Page.toByteString(), false)); + } + + return out; } private static Pair createTlsSocketFactory(TrustStore trustStore) { diff --git a/libsignal/service/src/main/proto/CDSH.proto b/libsignal/service/src/main/proto/CDSH.proto new file mode 100644 index 0000000000..1107018e07 --- /dev/null +++ b/libsignal/service/src/main/proto/CDSH.proto @@ -0,0 +1,55 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "org.signal.cds"; +option java_outer_classname = "Cds"; + +package org.signal.cds; + +message ClientRequest { + // Each ACI/UAK pair is a 32-byte buffer, containing the 16-byte ACI followed + // by its 16-byte UAK. + bytes aci_uak_pairs = 1; + + // Each E164 is an 8-byte big-endian number, as 8 bytes. + bytes prev_e164s = 2; + bytes new_e164s = 3; + bytes discard_e164s = 4; + + // If true, the client has more pairs or e164s to send. If false or unset, + // this is the client's last request, and processing should commence. + bool has_more = 5; + + // If set, a token which allows rate limiting to discount the e164s in + // the request's prev_e164s, only counting new_e164s. If not set, then + // rate limiting considers both prev_e164s' and new_e164s' size. + bytes token = 6; +} + +message ClientResponse { + // Each triple is an 8-byte e164, a 16-byte PNI, and a 16-byte ACI. + // If the e164 was not found, PNI and ACI are all zeros. If the PNI + // was found but the ACI was not, the PNI will be non-zero and the ACI + // will be all zeros. ACI will be returned if one of the returned + // PNIs has an ACI/UAK pair that matches. + // + // Should the request be successful (IE: a successful status returned), + // |e164_pni_aci_triple| will always equal |e164| of the request, + // so the entire marshalled size of the response will be (2+32)*|e164|, + // where the additional 2 bytes are the id/type/length additions of the + // protobuf marshaling added to each byte array. This avoids any data + // leakage based on the size of the encrypted output. + bytes e164_pni_aci_triples = 1; + + // If the user has run out of quota for lookups, they will receive + // a response with just the following field set, followed by a websocket + // closure of type 4008 (RESOURCE_EXHAUSTED). Should they retry exactly + // the same request after the provided number of seconds has passed, + // we expect it should work. + int32 retry_after_secs = 2; + + // A token which allows subsequent calls' rate limiting to discount the + // e164s sent up in this request, only counting those in the next + // request's new_e164s. + bytes token = 3; +}