Add a binary format for incoming messages

The existing, general incoming message endpoint accepts messages as JSON strings containing base64 data, along with all the metadata as other JSON keys. That's not very efficient, and we don't make use of that full generality anyway. This commit introduces a new binary format that supports everything we're using from the old format (with the help of some query parameters like multi-recipient messages).
2026-04-20 08:58:13 +01:00 · 2022-02-07 16:05:03 -08:00
parent 51bac394ec
commit 41bf2b2c42
9 changed files with 790 additions and 188 deletions
--- a/service/src/main/java/org/whispersystems/textsecuregcm/providers/BinaryProviderBase.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/providers/BinaryProviderBase.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2021 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.providers;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.UUID;
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.WebApplicationException;
+
+public abstract class BinaryProviderBase {
+
+  /**
+   * Reads a UUID in network byte order and converts to a UUID object.
+   */
+  UUID readUuid(InputStream stream) throws IOException {
+    byte[] buffer = new byte[8];
+
+    int read = stream.readNBytes(buffer, 0, 8);
+    if (read != 8) {
+      throw new IOException("Insufficient bytes for UUID");
+    }
+    long msb = convertNetworkByteOrderToLong(buffer);
+
+    read = stream.readNBytes(buffer, 0, 8);
+    if (read != 8) {
+      throw new IOException("Insufficient bytes for UUID");
+    }
+    long lsb = convertNetworkByteOrderToLong(buffer);
+
+    return new UUID(msb, lsb);
+  }
+
+  private long convertNetworkByteOrderToLong(byte[] buffer) {
+    long result = 0;
+    for (int i = 0; i < 8; i++) {
+      result = (result << 8) | (buffer[i] & 0xFFL);
+    }
+    return result;
+  }
+
+  /**
+   * Reads a varint. A varint larger than 64 bits is rejected with a {@code WebApplicationException}. An
+   * {@code IOException} is thrown if the stream ends before we finish reading the varint.
+   *
+   * @return the varint value
+   */
+  static long readVarint(InputStream stream) throws IOException, WebApplicationException {
+    boolean hasMore = true;
+    int currentOffset = 0;
+    long result = 0;
+    while (hasMore) {
+      if (currentOffset >= 64) {
+        throw new BadRequestException("varint is too large");
+      }
+      int b = stream.read();
+      if (b == -1) {
+        throw new IOException("Missing byte " + (currentOffset / 7) + " of varint");
+      }
+      if (currentOffset == 63 && (b & 0xFE) != 0) {
+        throw new BadRequestException("varint is too large");
+      }
+      hasMore = (b & 0x80) != 0;
+      result |= ((long)(b & 0x7F)) << currentOffset;
+      currentOffset += 7;
+    }
+    return result;
+  }
+
+  /**
+   * Reads two bytes with most significant byte first. Treats the value as unsigned so the range returned is
+   * {@code [0, 65535]}.
+   */
+  @VisibleForTesting
+  static int readU16(InputStream stream) throws IOException {
+    int b1 = stream.read();
+    if (b1 == -1) {
+      throw new IOException("Missing byte 1 of U16");
+    }
+    int b2 = stream.read();
+    if (b2 == -1) {
+      throw new IOException("Missing byte 2 of U16");
+    }
+    return (b1 << 8) | b2;
+  }
+}
--- a/service/src/main/java/org/whispersystems/textsecuregcm/providers/MultiDeviceMessageListProvider.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/providers/MultiDeviceMessageListProvider.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2021 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.providers;
+
+import io.dropwizard.util.DataSizeUnit;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Type;
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.NoContentException;
+import javax.ws.rs.ext.MessageBodyReader;
+import javax.ws.rs.ext.Provider;
+import org.whispersystems.textsecuregcm.entities.IncomingDeviceMessage;
+
+@Provider
+@Consumes(MultiDeviceMessageListProvider.MEDIA_TYPE)
+public class MultiDeviceMessageListProvider extends BinaryProviderBase implements MessageBodyReader<IncomingDeviceMessage[]> {
+
+  public static final String MEDIA_TYPE = "application/vnd.signal-messenger.mdml";
+  public static final int MAX_MESSAGE_COUNT = 50;
+  public static final int MAX_MESSAGE_SIZE = Math.toIntExact(DataSizeUnit.KIBIBYTES.toBytes(256));
+  public static final byte VERSION = 0x01;
+
+  @Override
+  public boolean isReadable(Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
+    return MEDIA_TYPE.equals(mediaType.toString()) && IncomingDeviceMessage[].class.isAssignableFrom(type);
+  }
+
+  @Override
+  public IncomingDeviceMessage[]
+  readFrom(Class<IncomingDeviceMessage[]> resultType, Type genericType,
+      Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, String> httpHeaders,
+      InputStream entityStream)
+      throws IOException, WebApplicationException {
+    int versionByte = entityStream.read();
+    if (versionByte == -1) {
+      throw new NoContentException("Empty body not allowed");
+    }
+    if (versionByte != VERSION) {
+      throw new BadRequestException("Unsupported version");
+    }
+    int count = entityStream.read();
+    if (count == -1) {
+      throw new IOException("Missing count");
+    }
+    if (count > MAX_MESSAGE_COUNT) {
+      throw new BadRequestException("Maximum recipient count exceeded");
+    }
+    IncomingDeviceMessage[] messages = new IncomingDeviceMessage[count];
+    for (int i = 0; i < count; i++) {
+      long deviceId = readVarint(entityStream);
+      int registrationId = readU16(entityStream);
+
+      int type = entityStream.read();
+      if (type == -1) {
+        throw new IOException("Unexpected end of stream reading message type");
+      }
+
+      long messageLength = readVarint(entityStream);
+      if (messageLength > MAX_MESSAGE_SIZE) {
+        throw new BadRequestException("Message body too large");
+      }
+      byte[] contents = entityStream.readNBytes(Math.toIntExact(messageLength));
+      if (contents.length != messageLength) {
+        throw new IOException("Unexpected end of stream in the middle of message contents");
+      }
+
+      messages[i] = new IncomingDeviceMessage(type, deviceId, registrationId, contents);
+    }
+    return messages;
+  }
+}
--- a/service/src/main/java/org/whispersystems/textsecuregcm/providers/MultiRecipientMessageProvider.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/providers/MultiRecipientMessageProvider.java
@@ -5,7 +5,6 @@

 package org.whispersystems.textsecuregcm.providers;

-import com.google.common.annotations.VisibleForTesting;
 import io.dropwizard.util.DataSizeUnit;
 import java.io.IOException;
 import java.io.InputStream;
@@ -24,7 +23,7 @@ import org.whispersystems.textsecuregcm.entities.MultiRecipientMessage;

@Provider
@Consumes(MultiRecipientMessageProvider.MEDIA_TYPE)
-public class MultiRecipientMessageProvider implements MessageBodyReader<MultiRecipientMessage> {
+public class MultiRecipientMessageProvider extends BinaryProviderBase implements MessageBodyReader<MultiRecipientMessage> {

  public static final String MEDIA_TYPE = "application/vnd.signal-messenger.mrm";
  public static final int MAX_RECIPIENT_COUNT = 5000;
@@ -71,78 +70,4 @@ public class MultiRecipientMessageProvider implements MessageBodyReader<MultiRec
    }
    return new MultiRecipientMessage(recipients, commonPayload);
  }
-
-  /**
-   * Reads a UUID in network byte order and converts to a UUID object.
-   */
-  private UUID readUuid(InputStream stream) throws IOException {
-    byte[] buffer = new byte[8];
-
-    int read = stream.readNBytes(buffer, 0, 8);
-    if (read != 8) {
-      throw new IOException("Insufficient bytes for UUID");
-    }
-    long msb = convertNetworkByteOrderToLong(buffer);
-
-    read = stream.readNBytes(buffer, 0, 8);
-    if (read != 8) {
-      throw new IOException("Insufficient bytes for UUID");
-    }
-    long lsb = convertNetworkByteOrderToLong(buffer);
-
-    return new UUID(msb, lsb);
-  }
-
-  private long convertNetworkByteOrderToLong(byte[] buffer) {
-    long result = 0;
-    for (int i = 0; i < 8; i++) {
-      result = (result << 8) | (buffer[i] & 0xFFL);
-    }
-    return result;
-  }
-
-  /**
-   * Reads a varint. A varint larger than 64 bits is rejected with a {@code WebApplicationException}. An
-   * {@code IOException} is thrown if the stream ends before we finish reading the varint.
-   *
-   * @return the varint value
-   */
-  private long readVarint(InputStream stream) throws IOException, WebApplicationException {
-    boolean hasMore = true;
-    int currentOffset = 0;
-    int result = 0;
-    while (hasMore) {
-      if (currentOffset >= 64) {
-        throw new BadRequestException("varint is too large");
-      }
-      int b = stream.read();
-      if (b == -1) {
-        throw new IOException("Missing byte " + (currentOffset / 7) + " of varint");
-      }
-      if (currentOffset == 63 && (b & 0xFE) != 0) {
-        throw new BadRequestException("varint is too large");
-      }
-      hasMore = (b & 0x80) != 0;
-      result |= (b & 0x7F) << currentOffset;
-      currentOffset += 7;
-    }
-    return result;
-  }
-
-  /**
-   * Reads two bytes with most significant byte first. Treats the value as unsigned so the range returned is
-   * {@code [0, 65535]}.
-   */
-  @VisibleForTesting
-  static int readU16(InputStream stream) throws IOException {
-    int b1 = stream.read();
-    if (b1 == -1) {
-      throw new IOException("Missing byte 1 of U16");
-    }
-    int b2 = stream.read();
-    if (b2 == -1) {
-      throw new IOException("Missing byte 2 of U16");
-    }
-    return (b1 << 8) | b2;
-  }
 }