From c6473ca9e63236af3eae9959a50cfa643d53272e Mon Sep 17 00:00:00 2001 From: Ehren Kret Date: Thu, 6 Jul 2023 12:33:48 -0500 Subject: [PATCH] Minor improvement to Android Backup file format. --- .../backup/BackupFrameOutputStream.java | 245 ++++++++++++++++++ .../backup/BackupRecordInputStream.java | 31 ++- .../securesms/backup/BackupVersions.kt | 21 ++ .../securesms/backup/FullBackupExporter.java | 207 +-------------- app/src/main/protowire/Backups.proto | 5 +- 5 files changed, 297 insertions(+), 212 deletions(-) create mode 100644 app/src/main/java/org/thoughtcrime/securesms/backup/BackupFrameOutputStream.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/backup/BackupVersions.kt diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/BackupFrameOutputStream.java b/app/src/main/java/org/thoughtcrime/securesms/backup/BackupFrameOutputStream.java new file mode 100644 index 0000000000..0da664e6e7 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/BackupFrameOutputStream.java @@ -0,0 +1,245 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup; + +import androidx.annotation.NonNull; + +import org.signal.core.util.Conversions; +import org.signal.core.util.logging.Log; +import org.signal.libsignal.protocol.kdf.HKDF; +import org.signal.libsignal.protocol.util.ByteUtil; +import org.thoughtcrime.securesms.attachments.AttachmentId; +import org.thoughtcrime.securesms.backup.proto.Attachment; +import org.thoughtcrime.securesms.backup.proto.Avatar; +import org.thoughtcrime.securesms.backup.proto.BackupFrame; +import org.thoughtcrime.securesms.backup.proto.DatabaseVersion; +import org.thoughtcrime.securesms.backup.proto.Header; +import org.thoughtcrime.securesms.backup.proto.KeyValue; +import org.thoughtcrime.securesms.backup.proto.SharedPreference; +import org.thoughtcrime.securesms.backup.proto.SqlStatement; +import org.thoughtcrime.securesms.backup.proto.Sticker; +import org.thoughtcrime.securesms.util.Util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.Mac; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +class BackupFrameOutputStream extends FullBackupBase.BackupStream { + + private static final String TAG = Log.tag(BackupFrameOutputStream.class); + + private final OutputStream outputStream; + private final Cipher cipher; + private final Mac mac; + + private final byte[] cipherKey; + private final byte[] iv; + private int counter; + + private int frames; + + BackupFrameOutputStream(@NonNull OutputStream output, @NonNull String passphrase) throws IOException { + try { + byte[] salt = Util.getSecretBytes(32); + byte[] key = getBackupKey(passphrase, salt); + byte[] derived = HKDF.deriveSecrets(key, "Backup Export".getBytes(), 64); + byte[][] split = ByteUtil.split(derived, 32, 32); + + this.cipherKey = split[0]; + byte[] macKey = split[1]; + + this.cipher = Cipher.getInstance("AES/CTR/NoPadding"); + this.mac = Mac.getInstance("HmacSHA256"); + this.outputStream = output; + this.iv = Util.getSecretBytes(16); + this.counter = Conversions.byteArrayToInt(iv); + + mac.init(new SecretKeySpec(macKey, "HmacSHA256")); + + byte[] header = new BackupFrame.Builder().header_(new Header.Builder() + .iv(new okio.ByteString(iv)) + .salt(new okio.ByteString(salt)) + .version(BackupVersions.CURRENT_VERSION) + .build()) + .build() + .encode(); + + outputStream.write(Conversions.intToByteArray(header.length)); + outputStream.write(header); + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException e) { + throw new AssertionError(e); + } + } + + public void write(SharedPreference preference) throws IOException { + write(outputStream, new BackupFrame.Builder().preference(preference).build()); + } + + public void write(KeyValue keyValue) throws IOException { + write(outputStream, new BackupFrame.Builder().keyValue(keyValue).build()); + } + + public void write(SqlStatement statement) throws IOException { + write(outputStream, new BackupFrame.Builder().statement(statement).build()); + } + + public void write(@NonNull String avatarName, @NonNull InputStream in, long size) throws IOException { + try { + write(outputStream, new BackupFrame.Builder() + .avatar(new Avatar.Builder() + .recipientId(avatarName) + .length(Util.toIntExact(size)) + .build()) + .build()); + } catch (ArithmeticException e) { + Log.w(TAG, "Unable to write avatar to backup", e); + throw new FullBackupExporter.InvalidBackupStreamException(); + } + + if (writeStream(in) != size) { + throw new IOException("Size mismatch!"); + } + } + + public void write(@NonNull AttachmentId attachmentId, @NonNull InputStream in, long size) throws IOException { + try { + write(outputStream, new BackupFrame.Builder() + .attachment(new Attachment.Builder() + .rowId(attachmentId.getRowId()) + .attachmentId(attachmentId.getUniqueId()) + .length(Util.toIntExact(size)) + .build()) + .build()); + } catch (ArithmeticException e) { + Log.w(TAG, "Unable to write " + attachmentId + " to backup", e); + throw new FullBackupExporter.InvalidBackupStreamException(); + } + + if (writeStream(in) != size) { + throw new IOException("Size mismatch!"); + } + } + + public void writeSticker(long rowId, @NonNull InputStream in, long size) throws IOException { + try { + write(outputStream, new BackupFrame.Builder() + .sticker(new Sticker.Builder() + .rowId(rowId) + .length(Util.toIntExact(size)) + .build()) + .build()); + } catch (ArithmeticException e) { + Log.w(TAG, "Unable to write sticker to backup", e); + throw new FullBackupExporter.InvalidBackupStreamException(); + } + + if (writeStream(in) != size) { + throw new IOException("Size mismatch!"); + } + } + + void writeDatabaseVersion(int version) throws IOException { + write(outputStream, new BackupFrame.Builder() + .version(new DatabaseVersion.Builder().version(version).build()) + .build()); + } + + void writeEnd() throws IOException { + write(outputStream, new BackupFrame.Builder().end(true).build()); + } + + /** + * @return The amount of data written from the provided InputStream. + */ + private long writeStream(@NonNull InputStream inputStream) throws IOException { + try { + Conversions.intToByteArray(iv, 0, counter++); + cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv)); + mac.update(iv); + + byte[] buffer = new byte[8192]; + long total = 0; + + int read; + + while ((read = inputStream.read(buffer)) != -1) { + byte[] ciphertext = cipher.update(buffer, 0, read); + + if (ciphertext != null) { + outputStream.write(ciphertext); + mac.update(ciphertext); + } + + total += read; + } + + byte[] remainder = cipher.doFinal(); + outputStream.write(remainder); + mac.update(remainder); + + byte[] attachmentDigest = mac.doFinal(); + outputStream.write(attachmentDigest, 0, 10); + + return total; + } catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) { + throw new AssertionError(e); + } + } + + private void write(@NonNull OutputStream out, @NonNull BackupFrame frame) throws IOException { + try { + Conversions.intToByteArray(iv, 0, counter++); + cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv)); + + byte[] encodedFrame = frame.encode(); + + // this assumes a stream cipher + byte[] length = Conversions.intToByteArray(encodedFrame.length + 10); + if (BackupVersions.isFrameLengthEncrypted(BackupVersions.CURRENT_VERSION)) { + byte[] encryptedLength = cipher.update(length); + if (encryptedLength.length != length.length) { + throw new IOException("Stream cipher assumption has been violated!"); + } + mac.update(encryptedLength); + length = encryptedLength; + } + + byte[] frameCiphertext = cipher.doFinal(frame.encode()); + if (frameCiphertext.length != encodedFrame.length) { + throw new IOException("Stream cipher assumption has been violated!"); + } + + byte[] frameMac = mac.doFinal(frameCiphertext); + + out.write(length); + out.write(frameCiphertext); + out.write(frameMac, 0, 10); + frames++; + } catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) { + throw new AssertionError(e); + } + } + + public void close() throws IOException { + outputStream.flush(); + outputStream.close(); + } + + public int getFrames() { + return frames; + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/BackupRecordInputStream.java b/app/src/main/java/org/thoughtcrime/securesms/backup/BackupRecordInputStream.java index e90f3169b0..87509aed5a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/BackupRecordInputStream.java +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/BackupRecordInputStream.java @@ -27,6 +27,7 @@ import javax.crypto.spec.SecretKeySpec; class BackupRecordInputStream extends FullBackupBase.BackupStream { + private final int version; private final InputStream in; private final Cipher cipher; private final Mac mac; @@ -55,12 +56,21 @@ class BackupRecordInputStream extends FullBackupBase.BackupStream { Header header = frame.header_; + if (header.iv == null) { + throw new IOException("Missing IV!"); + } + this.iv = header.iv.toByteArray(); if (iv.length != 16) { throw new IOException("Invalid IV length!"); } + this.version = header.version != null ? header.version : 0; + if (!BackupVersions.isCompatible(version)) { + throw new IOException("Invalid backup version: " + version); + } + byte[] key = getBackupKey(passphrase, header.salt != null ? header.salt.toByteArray() : null); byte[] derived = HKDF.deriveSecrets(key, "Backup Export".getBytes(), 64); byte[][] split = ByteUtil.split(derived, 32, 32); @@ -135,7 +145,23 @@ class BackupRecordInputStream extends FullBackupBase.BackupStream { byte[] length = new byte[4]; StreamUtil.readFully(in, length); - byte[] frame = new byte[Conversions.byteArrayToInt(length)]; + Conversions.intToByteArray(iv, 0, counter++); + cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv)); + + int frameLength; + if (BackupVersions.isFrameLengthEncrypted(version)) { + mac.update(length); + // this depends upon cipher being a stream cipher mode in order to get back the length without needing a full AES block-size input + byte[] decryptedLength = cipher.update(length); + if (decryptedLength.length != length.length) { + throw new IOException("Cipher was not a stream cipher!"); + } + frameLength = Conversions.byteArrayToInt(decryptedLength); + } else { + frameLength = Conversions.byteArrayToInt(length); + } + + byte[] frame = new byte[frameLength]; StreamUtil.readFully(in, frame); byte[] theirMac = new byte[10]; @@ -148,9 +174,6 @@ class BackupRecordInputStream extends FullBackupBase.BackupStream { throw new IOException("Bad MAC"); } - Conversions.intToByteArray(iv, 0, counter++); - cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv)); - byte[] plaintext = cipher.doFinal(frame, 0, frame.length - 10); return BackupFrame.ADAPTER.decode(plaintext); diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/BackupVersions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/BackupVersions.kt new file mode 100644 index 0000000000..93275f33d9 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/BackupVersions.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup + +object BackupVersions { + const val CURRENT_VERSION = 1 + const val MINIMUM_VERSION = 0 + + @JvmStatic + fun isCompatible(version: Int): Boolean { + return version in MINIMUM_VERSION..CURRENT_VERSION + } + + @JvmStatic + fun isFrameLengthEncrypted(version: Int): Boolean { + return version >= 1 + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupExporter.java b/app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupExporter.java index 6872558241..e4a7df5da9 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupExporter.java +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupExporter.java @@ -16,24 +16,15 @@ import com.annimon.stream.function.Predicate; import net.zetetic.database.sqlcipher.SQLiteDatabase; import org.greenrobot.eventbus.EventBus; -import org.signal.core.util.Conversions; import org.signal.core.util.CursorUtil; import org.signal.core.util.SetUtil; import org.signal.core.util.SqlUtil; import org.signal.core.util.Stopwatch; import org.signal.core.util.logging.Log; -import org.signal.libsignal.protocol.kdf.HKDF; -import org.signal.libsignal.protocol.util.ByteUtil; import org.thoughtcrime.securesms.attachments.AttachmentId; -import org.thoughtcrime.securesms.backup.proto.Attachment; -import org.thoughtcrime.securesms.backup.proto.Avatar; -import org.thoughtcrime.securesms.backup.proto.BackupFrame; -import org.thoughtcrime.securesms.backup.proto.DatabaseVersion; -import org.thoughtcrime.securesms.backup.proto.Header; import org.thoughtcrime.securesms.backup.proto.KeyValue; import org.thoughtcrime.securesms.backup.proto.SharedPreference; import org.thoughtcrime.securesms.backup.proto.SqlStatement; -import org.thoughtcrime.securesms.backup.proto.Sticker; import org.thoughtcrime.securesms.crypto.AttachmentSecret; import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream; import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream; @@ -59,7 +50,6 @@ import org.thoughtcrime.securesms.keyvalue.KeyValueDataSet; import org.thoughtcrime.securesms.keyvalue.SignalStore; import org.thoughtcrime.securesms.profiles.AvatarHelper; import org.thoughtcrime.securesms.util.TextSecurePreferences; -import org.thoughtcrime.securesms.util.Util; import java.io.File; import java.io.FileNotFoundException; @@ -67,9 +57,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.security.InvalidAlgorithmParameterException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -80,14 +67,6 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import javax.crypto.BadPaddingException; -import javax.crypto.Cipher; -import javax.crypto.IllegalBlockSizeException; -import javax.crypto.Mac; -import javax.crypto.NoSuchPaddingException; -import javax.crypto.spec.IvParameterSpec; -import javax.crypto.spec.SecretKeySpec; - import okio.ByteString; public class FullBackupExporter extends FullBackupBase { @@ -228,7 +207,7 @@ public class FullBackupExporter extends FullBackupBase { outputStream.close(); } } - return new BackupEvent(BackupEvent.Type.FINISHED, outputStream.frames, estimatedCountOutside); + return new BackupEvent(BackupEvent.Type.FINISHED, outputStream.getFrames(), estimatedCountOutside); } private static long calculateCount(@NonNull Context context, @NonNull SQLiteDatabase input, List tables) { @@ -627,190 +606,6 @@ public class FullBackupExporter extends FullBackupBase { return false; } - private static class BackupFrameOutputStream extends BackupStream { - - private final OutputStream outputStream; - private final Cipher cipher; - private final Mac mac; - - private final byte[] cipherKey; - private final byte[] iv; - private int counter; - - private int frames; - - private BackupFrameOutputStream(@NonNull OutputStream output, @NonNull String passphrase) throws IOException { - try { - byte[] salt = Util.getSecretBytes(32); - byte[] key = getBackupKey(passphrase, salt); - byte[] derived = HKDF.deriveSecrets(key, "Backup Export".getBytes(), 64); - byte[][] split = ByteUtil.split(derived, 32, 32); - - this.cipherKey = split[0]; - byte[] macKey = split[1]; - - this.cipher = Cipher.getInstance("AES/CTR/NoPadding"); - this.mac = Mac.getInstance("HmacSHA256"); - this.outputStream = output; - this.iv = Util.getSecretBytes(16); - this.counter = Conversions.byteArrayToInt(iv); - - mac.init(new SecretKeySpec(macKey, "HmacSHA256")); - - byte[] header = new BackupFrame.Builder().header_(new Header.Builder() - .iv(new okio.ByteString(iv)) - .salt(new okio.ByteString(salt)) - .build()) - .build() - .encode(); - - outputStream.write(Conversions.intToByteArray(header.length)); - outputStream.write(header); - } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException e) { - throw new AssertionError(e); - } - } - - public void write(SharedPreference preference) throws IOException { - write(outputStream, new BackupFrame.Builder().preference(preference).build()); - } - - public void write(KeyValue keyValue) throws IOException { - write(outputStream, new BackupFrame.Builder().keyValue(keyValue).build()); - } - - public void write(SqlStatement statement) throws IOException { - write(outputStream, new BackupFrame.Builder().statement(statement).build()); - } - - public void write(@NonNull String avatarName, @NonNull InputStream in, long size) throws IOException { - try { - write(outputStream, new BackupFrame.Builder() - .avatar(new Avatar.Builder() - .recipientId(avatarName) - .length(Util.toIntExact(size)) - .build()) - .build()); - } catch (ArithmeticException e) { - Log.w(TAG, "Unable to write avatar to backup", e); - throw new InvalidBackupStreamException(); - } - - if (writeStream(in) != size) { - throw new IOException("Size mismatch!"); - } - } - - public void write(@NonNull AttachmentId attachmentId, @NonNull InputStream in, long size) throws IOException { - try { - write(outputStream, new BackupFrame.Builder() - .attachment(new Attachment.Builder() - .rowId(attachmentId.getRowId()) - .attachmentId(attachmentId.getUniqueId()) - .length(Util.toIntExact(size)) - .build()) - .build()); - } catch (ArithmeticException e) { - Log.w(TAG, "Unable to write " + attachmentId + " to backup", e); - throw new InvalidBackupStreamException(); - } - - if (writeStream(in) != size) { - throw new IOException("Size mismatch!"); - } - } - - public void writeSticker(long rowId, @NonNull InputStream in, long size) throws IOException { - try { - write(outputStream, new BackupFrame.Builder() - .sticker(new Sticker.Builder() - .rowId(rowId) - .length(Util.toIntExact(size)) - .build()) - .build()); - } catch (ArithmeticException e) { - Log.w(TAG, "Unable to write sticker to backup", e); - throw new InvalidBackupStreamException(); - } - - if (writeStream(in) != size) { - throw new IOException("Size mismatch!"); - } - } - - void writeDatabaseVersion(int version) throws IOException { - write(outputStream, new BackupFrame.Builder() - .version(new DatabaseVersion.Builder().version(version).build()) - .build()); - } - - void writeEnd() throws IOException { - write(outputStream, new BackupFrame.Builder().end(true).build()); - } - - /** - * @return The amount of data written from the provided InputStream. - */ - private long writeStream(@NonNull InputStream inputStream) throws IOException { - try { - Conversions.intToByteArray(iv, 0, counter++); - cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv)); - mac.update(iv); - - byte[] buffer = new byte[8192]; - long total = 0; - - int read; - - while ((read = inputStream.read(buffer)) != -1) { - byte[] ciphertext = cipher.update(buffer, 0, read); - - if (ciphertext != null) { - outputStream.write(ciphertext); - mac.update(ciphertext); - } - - total += read; - } - - byte[] remainder = cipher.doFinal(); - outputStream.write(remainder); - mac.update(remainder); - - byte[] attachmentDigest = mac.doFinal(); - outputStream.write(attachmentDigest, 0, 10); - - return total; - } catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) { - throw new AssertionError(e); - } - } - - private void write(@NonNull OutputStream out, @NonNull BackupFrame frame) throws IOException { - try { - Conversions.intToByteArray(iv, 0, counter++); - cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv)); - - byte[] frameCiphertext = cipher.doFinal(frame.encode()); - byte[] frameMac = mac.doFinal(frameCiphertext); - byte[] length = Conversions.intToByteArray(frameCiphertext.length + 10); - - out.write(length); - out.write(frameCiphertext); - out.write(frameMac, 0, 10); - frames++; - } catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) { - throw new AssertionError(e); - } - } - - - public void close() throws IOException { - outputStream.flush(); - outputStream.close(); - } - } - public interface PostProcessor { int postProcess(@NonNull Cursor cursor, int count) throws IOException; } diff --git a/app/src/main/protowire/Backups.proto b/app/src/main/protowire/Backups.proto index 9632068872..83227de618 100644 --- a/app/src/main/protowire/Backups.proto +++ b/app/src/main/protowire/Backups.proto @@ -54,8 +54,9 @@ message DatabaseVersion { } message Header { - optional bytes iv = 1; - optional bytes salt = 2; + optional bytes iv = 1; + optional bytes salt = 2; + optional uint32 version = 3; } message KeyValue {