Minor improvement to Android Backup file format.

app/src/main/java/org/thoughtcrime/securesms/backup/BackupFrameOutputStream.java

@@ -0,0 +1,245 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup;
+
+import androidx.annotation.NonNull;
+
+import org.signal.core.util.Conversions;
+import org.signal.core.util.logging.Log;
+import org.signal.libsignal.protocol.kdf.HKDF;
+import org.signal.libsignal.protocol.util.ByteUtil;
+import org.thoughtcrime.securesms.attachments.AttachmentId;
+import org.thoughtcrime.securesms.backup.proto.Attachment;
+import org.thoughtcrime.securesms.backup.proto.Avatar;
+import org.thoughtcrime.securesms.backup.proto.BackupFrame;
+import org.thoughtcrime.securesms.backup.proto.DatabaseVersion;
+import org.thoughtcrime.securesms.backup.proto.Header;
+import org.thoughtcrime.securesms.backup.proto.KeyValue;
+import org.thoughtcrime.securesms.backup.proto.SharedPreference;
+import org.thoughtcrime.securesms.backup.proto.SqlStatement;
+import org.thoughtcrime.securesms.backup.proto.Sticker;
+import org.thoughtcrime.securesms.util.Util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.Mac;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+
+class BackupFrameOutputStream extends FullBackupBase.BackupStream {
+
+  private static final String TAG = Log.tag(BackupFrameOutputStream.class);
+
+  private final OutputStream outputStream;
+  private final Cipher       cipher;
+  private final Mac          mac;
+
+  private final byte[] cipherKey;
+  private final byte[] iv;
+  private       int    counter;
+
+  private int frames;
+
+  BackupFrameOutputStream(@NonNull OutputStream output, @NonNull String passphrase) throws IOException {
+    try {
+      byte[]   salt    = Util.getSecretBytes(32);
+      byte[]   key     = getBackupKey(passphrase, salt);
+      byte[]   derived = HKDF.deriveSecrets(key, "Backup Export".getBytes(), 64);
+      byte[][] split   = ByteUtil.split(derived, 32, 32);
+
+      this.cipherKey = split[0];
+      byte[] macKey = split[1];
+
+      this.cipher       = Cipher.getInstance("AES/CTR/NoPadding");
+      this.mac          = Mac.getInstance("HmacSHA256");
+      this.outputStream = output;
+      this.iv           = Util.getSecretBytes(16);
+      this.counter      = Conversions.byteArrayToInt(iv);
+
+      mac.init(new SecretKeySpec(macKey, "HmacSHA256"));
+
+      byte[] header = new BackupFrame.Builder().header_(new Header.Builder()
+                                                            .iv(new okio.ByteString(iv))
+                                                            .salt(new okio.ByteString(salt))
+                                                            .version(BackupVersions.CURRENT_VERSION)
+                                                            .build())
+                                               .build()
+                                               .encode();
+
+      outputStream.write(Conversions.intToByteArray(header.length));
+      outputStream.write(header);
+    } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException e) {
+      throw new AssertionError(e);
+    }
+  }
+
+  public void write(SharedPreference preference) throws IOException {
+    write(outputStream, new BackupFrame.Builder().preference(preference).build());
+  }
+
+  public void write(KeyValue keyValue) throws IOException {
+    write(outputStream, new BackupFrame.Builder().keyValue(keyValue).build());
+  }
+
+  public void write(SqlStatement statement) throws IOException {
+    write(outputStream, new BackupFrame.Builder().statement(statement).build());
+  }
+
+  public void write(@NonNull String avatarName, @NonNull InputStream in, long size) throws IOException {
+    try {
+      write(outputStream, new BackupFrame.Builder()
+          .avatar(new Avatar.Builder()
+                      .recipientId(avatarName)
+                      .length(Util.toIntExact(size))
+                      .build())
+          .build());
+    } catch (ArithmeticException e) {
+      Log.w(TAG, "Unable to write avatar to backup", e);
+      throw new FullBackupExporter.InvalidBackupStreamException();
+    }
+
+    if (writeStream(in) != size) {
+      throw new IOException("Size mismatch!");
+    }
+  }
+
+  public void write(@NonNull AttachmentId attachmentId, @NonNull InputStream in, long size) throws IOException {
+    try {
+      write(outputStream, new BackupFrame.Builder()
+          .attachment(new Attachment.Builder()
+                          .rowId(attachmentId.getRowId())
+                          .attachmentId(attachmentId.getUniqueId())
+                          .length(Util.toIntExact(size))
+                          .build())
+          .build());
+    } catch (ArithmeticException e) {
+      Log.w(TAG, "Unable to write " + attachmentId + " to backup", e);
+      throw new FullBackupExporter.InvalidBackupStreamException();
+    }
+
+    if (writeStream(in) != size) {
+      throw new IOException("Size mismatch!");
+    }
+  }
+
+  public void writeSticker(long rowId, @NonNull InputStream in, long size) throws IOException {
+    try {
+      write(outputStream, new BackupFrame.Builder()
+          .sticker(new Sticker.Builder()
+                       .rowId(rowId)
+                       .length(Util.toIntExact(size))
+                       .build())
+          .build());
+    } catch (ArithmeticException e) {
+      Log.w(TAG, "Unable to write sticker to backup", e);
+      throw new FullBackupExporter.InvalidBackupStreamException();
+    }
+
+    if (writeStream(in) != size) {
+      throw new IOException("Size mismatch!");
+    }
+  }
+
+  void writeDatabaseVersion(int version) throws IOException {
+    write(outputStream, new BackupFrame.Builder()
+        .version(new DatabaseVersion.Builder().version(version).build())
+        .build());
+  }
+
+  void writeEnd() throws IOException {
+    write(outputStream, new BackupFrame.Builder().end(true).build());
+  }
+
+  /**
+   * @return The amount of data written from the provided InputStream.
+   */
+  private long writeStream(@NonNull InputStream inputStream) throws IOException {
+    try {
+      Conversions.intToByteArray(iv, 0, counter++);
+      cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv));
+      mac.update(iv);
+
+      byte[] buffer = new byte[8192];
+      long   total  = 0;
+
+      int read;
+
+      while ((read = inputStream.read(buffer)) != -1) {
+        byte[] ciphertext = cipher.update(buffer, 0, read);
+
+        if (ciphertext != null) {
+          outputStream.write(ciphertext);
+          mac.update(ciphertext);
+        }
+
+        total += read;
+      }
+
+      byte[] remainder = cipher.doFinal();
+      outputStream.write(remainder);
+      mac.update(remainder);
+
+      byte[] attachmentDigest = mac.doFinal();
+      outputStream.write(attachmentDigest, 0, 10);
+
+      return total;
+    } catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) {
+      throw new AssertionError(e);
+    }
+  }
+
+  private void write(@NonNull OutputStream out, @NonNull BackupFrame frame) throws IOException {
+    try {
+      Conversions.intToByteArray(iv, 0, counter++);
+      cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv));
+
+      byte[] encodedFrame = frame.encode();
+
+      // this assumes a stream cipher
+      byte[] length = Conversions.intToByteArray(encodedFrame.length + 10);
+      if (BackupVersions.isFrameLengthEncrypted(BackupVersions.CURRENT_VERSION)) {
+        byte[] encryptedLength = cipher.update(length);
+        if (encryptedLength.length != length.length) {
+          throw new IOException("Stream cipher assumption has been violated!");
+        }
+        mac.update(encryptedLength);
+        length = encryptedLength;
+      }
+
+      byte[] frameCiphertext = cipher.doFinal(frame.encode());
+      if (frameCiphertext.length != encodedFrame.length) {
+        throw new IOException("Stream cipher assumption has been violated!");
+      }
+
+      byte[] frameMac = mac.doFinal(frameCiphertext);
+
+      out.write(length);
+      out.write(frameCiphertext);
+      out.write(frameMac, 0, 10);
+      frames++;
+    } catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) {
+      throw new AssertionError(e);
+    }
+  }
+
+  public void close() throws IOException {
+    outputStream.flush();
+    outputStream.close();
+  }
+
+  public int getFrames() {
+    return frames;
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/backup/BackupRecordInputStream.java

@@ -27,6 +27,7 @@ import javax.crypto.spec.SecretKeySpec;
 
 class BackupRecordInputStream extends FullBackupBase.BackupStream {
 
+  private final int         version;
   private final InputStream in;
   private final Cipher      cipher;
   private final Mac         mac;
@@ -55,12 +56,21 @@ class BackupRecordInputStream extends FullBackupBase.BackupStream {
 
       Header header = frame.header_;
 
+      if (header.iv == null) {
+        throw new IOException("Missing IV!");
+      }
+
       this.iv = header.iv.toByteArray();
 
       if (iv.length != 16) {
         throw new IOException("Invalid IV length!");
       }
 
+      this.version = header.version != null ? header.version : 0;
+      if (!BackupVersions.isCompatible(version)) {
+        throw new IOException("Invalid backup version: " + version);
+      }
+
       byte[]   key     = getBackupKey(passphrase, header.salt != null ? header.salt.toByteArray() : null);
       byte[]   derived = HKDF.deriveSecrets(key, "Backup Export".getBytes(), 64);
       byte[][] split   = ByteUtil.split(derived, 32, 32);
@@ -135,7 +145,23 @@ class BackupRecordInputStream extends FullBackupBase.BackupStream {
       byte[] length = new byte[4];
       StreamUtil.readFully(in, length);
 
-      byte[] frame = new byte[Conversions.byteArrayToInt(length)];
+      Conversions.intToByteArray(iv, 0, counter++);
+      cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv));
+
+      int frameLength;
+      if (BackupVersions.isFrameLengthEncrypted(version)) {
+        mac.update(length);
+        // this depends upon cipher being a stream cipher mode in order to get back the length without needing a full AES block-size input
+        byte[] decryptedLength = cipher.update(length);
+        if (decryptedLength.length != length.length) {
+          throw new IOException("Cipher was not a stream cipher!");
+        }
+        frameLength = Conversions.byteArrayToInt(decryptedLength);
+      } else {
+        frameLength = Conversions.byteArrayToInt(length);
+      }
+
+      byte[] frame = new byte[frameLength];
       StreamUtil.readFully(in, frame);
 
       byte[] theirMac = new byte[10];
@@ -148,9 +174,6 @@ class BackupRecordInputStream extends FullBackupBase.BackupStream {
         throw new IOException("Bad MAC");
       }
 
-      Conversions.intToByteArray(iv, 0, counter++);
-      cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv));
-
       byte[] plaintext = cipher.doFinal(frame, 0, frame.length - 10);
 
       return BackupFrame.ADAPTER.decode(plaintext);

app/src/main/java/org/thoughtcrime/securesms/backup/BackupVersions.kt

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup
+
+object BackupVersions {
+  const val CURRENT_VERSION = 1
+  const val MINIMUM_VERSION = 0
+
+  @JvmStatic
+  fun isCompatible(version: Int): Boolean {
+    return version in MINIMUM_VERSION..CURRENT_VERSION
+  }
+
+  @JvmStatic
+  fun isFrameLengthEncrypted(version: Int): Boolean {
+    return version >= 1
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupExporter.java

@@ -16,24 +16,15 @@ import com.annimon.stream.function.Predicate;
 import net.zetetic.database.sqlcipher.SQLiteDatabase;
 
 import org.greenrobot.eventbus.EventBus;
-import org.signal.core.util.Conversions;
 import org.signal.core.util.CursorUtil;
 import org.signal.core.util.SetUtil;
 import org.signal.core.util.SqlUtil;
 import org.signal.core.util.Stopwatch;
 import org.signal.core.util.logging.Log;
-import org.signal.libsignal.protocol.kdf.HKDF;
-import org.signal.libsignal.protocol.util.ByteUtil;
 import org.thoughtcrime.securesms.attachments.AttachmentId;
-import org.thoughtcrime.securesms.backup.proto.Attachment;
-import org.thoughtcrime.securesms.backup.proto.Avatar;
-import org.thoughtcrime.securesms.backup.proto.BackupFrame;
-import org.thoughtcrime.securesms.backup.proto.DatabaseVersion;
-import org.thoughtcrime.securesms.backup.proto.Header;
 import org.thoughtcrime.securesms.backup.proto.KeyValue;
 import org.thoughtcrime.securesms.backup.proto.SharedPreference;
 import org.thoughtcrime.securesms.backup.proto.SqlStatement;
-import org.thoughtcrime.securesms.backup.proto.Sticker;
 import org.thoughtcrime.securesms.crypto.AttachmentSecret;
 import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream;
 import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream;
@@ -59,7 +50,6 @@ import org.thoughtcrime.securesms.keyvalue.KeyValueDataSet;
 import org.thoughtcrime.securesms.keyvalue.SignalStore;
 import org.thoughtcrime.securesms.profiles.AvatarHelper;
 import org.thoughtcrime.securesms.util.TextSecurePreferences;
-import org.thoughtcrime.securesms.util.Util;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -67,9 +57,6 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.security.InvalidAlgorithmParameterException;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -80,14 +67,6 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.Cipher;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.Mac;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.spec.IvParameterSpec;
-import javax.crypto.spec.SecretKeySpec;
-
 import okio.ByteString;
 
 public class FullBackupExporter extends FullBackupBase {
@@ -228,7 +207,7 @@ public class FullBackupExporter extends FullBackupBase {
         outputStream.close();
       }
     }
-    return new BackupEvent(BackupEvent.Type.FINISHED, outputStream.frames, estimatedCountOutside);
+    return new BackupEvent(BackupEvent.Type.FINISHED, outputStream.getFrames(), estimatedCountOutside);
   }
 
   private static long calculateCount(@NonNull Context context, @NonNull SQLiteDatabase input, List<String> tables) {
@@ -627,190 +606,6 @@ public class FullBackupExporter extends FullBackupBase {
     return false;
   }
 
-  private static class BackupFrameOutputStream extends BackupStream {
-
-    private final OutputStream outputStream;
-    private final Cipher       cipher;
-    private final Mac          mac;
-
-    private final byte[] cipherKey;
-    private final byte[] iv;
-    private       int    counter;
-
-    private       int    frames;
-
-    private BackupFrameOutputStream(@NonNull OutputStream output, @NonNull String passphrase) throws IOException {
-      try {
-        byte[]   salt    = Util.getSecretBytes(32);
-        byte[]   key     = getBackupKey(passphrase, salt);
-        byte[]   derived = HKDF.deriveSecrets(key, "Backup Export".getBytes(), 64);
-        byte[][] split   = ByteUtil.split(derived, 32, 32);
-
-        this.cipherKey = split[0];
-        byte[] macKey  = split[1];
-
-        this.cipher       = Cipher.getInstance("AES/CTR/NoPadding");
-        this.mac          = Mac.getInstance("HmacSHA256");
-        this.outputStream = output;
-        this.iv           = Util.getSecretBytes(16);
-        this.counter      = Conversions.byteArrayToInt(iv);
-
-        mac.init(new SecretKeySpec(macKey, "HmacSHA256"));
-
-        byte[] header = new BackupFrame.Builder().header_(new Header.Builder()
-                                                                    .iv(new okio.ByteString(iv))
-                                                                    .salt(new okio.ByteString(salt))
-                                                                    .build())
-                                                 .build()
-                                                 .encode();
-
-        outputStream.write(Conversions.intToByteArray(header.length));
-        outputStream.write(header);
-      } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException e) {
-        throw new AssertionError(e);
-      }
-    }
-
-    public void write(SharedPreference preference) throws IOException {
-      write(outputStream, new BackupFrame.Builder().preference(preference).build());
-    }
-
-    public void write(KeyValue keyValue) throws IOException {
-      write(outputStream, new BackupFrame.Builder().keyValue(keyValue).build());
-    }
-
-    public void write(SqlStatement statement) throws IOException {
-      write(outputStream, new BackupFrame.Builder().statement(statement).build());
-    }
-
-    public void write(@NonNull String avatarName, @NonNull InputStream in, long size) throws IOException {
-      try {
-        write(outputStream, new BackupFrame.Builder()
-                                           .avatar(new Avatar.Builder()
-                                                             .recipientId(avatarName)
-                                                             .length(Util.toIntExact(size))
-                                                             .build())
-                                           .build());
-      } catch (ArithmeticException e) {
-        Log.w(TAG, "Unable to write avatar to backup", e);
-        throw new InvalidBackupStreamException();
-      }
-
-      if (writeStream(in) != size) {
-        throw new IOException("Size mismatch!");
-      }
-    }
-
-    public void write(@NonNull AttachmentId attachmentId, @NonNull InputStream in, long size) throws IOException {
-      try {
-        write(outputStream, new BackupFrame.Builder()
-                                           .attachment(new Attachment.Builder()
-                                                                     .rowId(attachmentId.getRowId())
-                                                                     .attachmentId(attachmentId.getUniqueId())
-                                                                     .length(Util.toIntExact(size))
-                                                                     .build())
-                                           .build());
-      } catch (ArithmeticException e) {
-        Log.w(TAG, "Unable to write " + attachmentId + " to backup", e);
-        throw new InvalidBackupStreamException();
-      }
-
-      if (writeStream(in) != size) {
-        throw new IOException("Size mismatch!");
-      }
-    }
-
-    public void writeSticker(long rowId, @NonNull InputStream in, long size) throws IOException {
-      try {
-        write(outputStream, new BackupFrame.Builder()
-                                           .sticker(new Sticker.Builder()
-                                                               .rowId(rowId)
-                                                               .length(Util.toIntExact(size))
-                                                               .build())
-                                           .build());
-      } catch (ArithmeticException e) {
-        Log.w(TAG, "Unable to write sticker to backup", e);
-        throw new InvalidBackupStreamException();
-      }
-
-      if (writeStream(in) != size) {
-        throw new IOException("Size mismatch!");
-      }
-    }
-
-    void writeDatabaseVersion(int version) throws IOException {
-      write(outputStream, new BackupFrame.Builder()
-                                         .version(new DatabaseVersion.Builder().version(version).build())
-                                         .build());
-    }
-
-    void writeEnd() throws IOException {
-      write(outputStream, new BackupFrame.Builder().end(true).build());
-    }
-
-    /**
-     * @return The amount of data written from the provided InputStream.
-     */
-    private long writeStream(@NonNull InputStream inputStream) throws IOException {
-      try {
-        Conversions.intToByteArray(iv, 0, counter++);
-        cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv));
-        mac.update(iv);
-
-        byte[] buffer = new byte[8192];
-        long   total  = 0;
-
-        int read;
-
-        while ((read = inputStream.read(buffer)) != -1) {
-          byte[] ciphertext = cipher.update(buffer, 0, read);
-
-          if (ciphertext != null) {
-            outputStream.write(ciphertext);
-            mac.update(ciphertext);
-          }
-
-          total += read;
-        }
-
-        byte[] remainder = cipher.doFinal();
-        outputStream.write(remainder);
-        mac.update(remainder);
-
-        byte[] attachmentDigest = mac.doFinal();
-        outputStream.write(attachmentDigest, 0, 10);
-
-        return total;
-      } catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) {
-        throw new AssertionError(e);
-      }
-    }
-
-    private void write(@NonNull OutputStream out, @NonNull BackupFrame frame) throws IOException {
-      try {
-        Conversions.intToByteArray(iv, 0, counter++);
-        cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv));
-
-        byte[] frameCiphertext = cipher.doFinal(frame.encode());
-        byte[] frameMac        = mac.doFinal(frameCiphertext);
-        byte[] length          = Conversions.intToByteArray(frameCiphertext.length + 10);
-
-        out.write(length);
-        out.write(frameCiphertext);
-        out.write(frameMac, 0, 10);
-        frames++;
-      } catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) {
-        throw new AssertionError(e);
-      }
-    }
-
-
-    public void close() throws IOException {
-      outputStream.flush();
-      outputStream.close();
-    }
-  }
-
   public interface PostProcessor {
     int postProcess(@NonNull Cursor cursor, int count) throws IOException;
   }

app/src/main/protowire/Backups.proto

@@ -54,8 +54,9 @@ message DatabaseVersion {
 }
 
 message Header {
-    optional bytes iv   = 1;
+    optional bytes  iv      = 1;
-    optional bytes salt = 2;
+    optional bytes  salt    = 2;
+    optional uint32 version = 3;
 }
 
 message KeyValue {