Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2026-04-22 09:49:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add initial sms exporter integration behind a feature flag.

Browse Source
This commit is contained in:
Alex Hart
2022-08-30 15:22:40 -03:00
committed by Greyson Parrelli
parent 1cc39fb89b
commit 936212e684
40 changed files with 1218 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
app/src/main/java/org/thoughtcrime/securesms/database/MessageDatabase.java
View File

@@ -25,6 +25,7 @@ import org.thoughtcrime.securesms.database.model.ParentStoryId;
import org.thoughtcrime.securesms.database.model.SmsMessageRecord;
import org.thoughtcrime.securesms.database.model.StoryResult;
import org.thoughtcrime.securesms.database.model.StoryViewState;
import org.thoughtcrime.securesms.database.model.databaseprotos.MessageExportState;
import org.thoughtcrime.securesms.groups.GroupMigrationMembershipChange;
import org.thoughtcrime.securesms.insights.InsightsConstants;
import org.thoughtcrime.securesms.mms.IncomingMediaMessage;
@@ -94,6 +95,9 @@ public abstract class MessageDatabase extends Database implements MmsSmsColumns
public abstract boolean isSent(long messageId);
public abstract List<MessageRecord> getProfileChangeDetailsRecords(long threadId, long afterTimestamp);
public abstract Set<Long> getAllRateLimitedMessageIds();
public abstract Cursor getUnexportedInsecureMessages();
public abstract int getInsecureMessageCount();
public abstract void deleteExportedMessages();
public abstract void markExpireStarted(long messageId);
public abstract void markExpireStarted(long messageId, long startTime);
@@ -353,6 +357,14 @@ public abstract class MessageDatabase extends Database implements MmsSmsColumns
return String.format(Locale.ENGLISH, "(%s OR %s) AND %s", isSent, isReceived, isSecure);
}
protected String getInsecureMessageClause() {
String isSent = "(" + getTypeField() + " & " + Types.BASE_TYPE_MASK + ") = " + Types.BASE_SENT_TYPE;
String isReceived = "(" + getTypeField() + " & " + Types.BASE_TYPE_MASK + ") = " + Types.BASE_INBOX_TYPE;
String isSecure = "(" + getTypeField() + " & " + (Types.SECURE_MESSAGE_BIT | Types.PUSH_MESSAGE_BIT) + ")";
return String.format(Locale.ENGLISH, "(%s OR %s) AND NOT %s", isSent, isReceived, isSecure);
}
public void setReactionsSeen(long threadId, long sinceTimestamp) {
SQLiteDatabase db = databaseHelper.getSignalWritableDatabase();
ContentValues values = new ContentValues();
@@ -803,6 +815,11 @@ public abstract class MessageDatabase extends Database implements MmsSmsColumns
@Deprecated
MessageRecord getCurrent();
/**
* Pulls the export state out of the query, if it is present.
*/
@NonNull MessageExportState getMessageExportStateForCurrentRecord();
/**
* From the {@link Closeable} interface, removing the IOException requirement.
*/

77
app/src/main/java/org/thoughtcrime/securesms/database/MmsDatabase.java
View File

@@ -63,6 +63,7 @@ import org.thoughtcrime.securesms.database.model.StoryType;
import org.thoughtcrime.securesms.database.model.StoryViewState;
import org.thoughtcrime.securesms.database.model.databaseprotos.BodyRangeList;
import org.thoughtcrime.securesms.database.model.databaseprotos.GiftBadge;
import org.thoughtcrime.securesms.database.model.databaseprotos.MessageExportState;
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
import org.thoughtcrime.securesms.groups.GroupMigrationMembershipChange;
import org.thoughtcrime.securesms.jobs.TrimThreadJob;
@@ -189,7 +190,9 @@ public class MmsDatabase extends MessageDatabase {
RECEIPT_TIMESTAMP + " INTEGER DEFAULT -1, " +
MESSAGE_RANGES + " BLOB DEFAULT NULL, " +
STORY_TYPE + " INTEGER DEFAULT 0, " +
PARENT_STORY_ID + " INTEGER DEFAULT 0);";
PARENT_STORY_ID + " INTEGER DEFAULT 0, " +
EXPORT_STATE + " BLOB DEFAULT NULL, " +
EXPORTED + " INTEGER DEFAULT 0);";
public static final String[] CREATE_INDEXS = {
"CREATE INDEX IF NOT EXISTS mms_read_and_notified_and_thread_id_index ON " + TABLE_NAME + "(" + READ + "," + NOTIFIED + "," + THREAD_ID + ");",
@@ -1216,8 +1219,12 @@ public class MmsDatabase extends MessageDatabase {
}
private Cursor rawQuery(@NonNull String where, @Nullable String[] arguments, boolean reverse, long limit) {
return rawQuery(MMS_PROJECTION, where, arguments, reverse, limit);
}
private Cursor rawQuery(@NonNull String[] projection, @NonNull String where, @Nullable String[] arguments, boolean reverse, long limit) {
SQLiteDatabase database = databaseHelper.getSignalReadableDatabase();
String rawQueryString = "SELECT " + Util.join(MMS_PROJECTION, ",") +
String rawQueryString = "SELECT " + Util.join(projection, ",") +
" FROM " + MmsDatabase.TABLE_NAME + " LEFT OUTER JOIN " + AttachmentDatabase.TABLE_NAME +
" ON (" + MmsDatabase.TABLE_NAME + "." + MmsDatabase.ID + " = " + AttachmentDatabase.TABLE_NAME + "." + AttachmentDatabase.MMS_ID + ")" +
" WHERE " + where + " GROUP BY " + MmsDatabase.TABLE_NAME + "." + MmsDatabase.ID;
@@ -2416,6 +2423,53 @@ public class MmsDatabase extends MessageDatabase {
return ids;
}
@Override
public Cursor getUnexportedInsecureMessages() {
return rawQuery(
SqlUtil.appendArg(MMS_PROJECTION, EXPORT_STATE),
getInsecureMessageClause() + " AND NOT " + EXPORTED,
null,
false,
0
);
}
@Override
public int getInsecureMessageCount() {
try (Cursor cursor = getWritableDatabase().query(TABLE_NAME, SqlUtil.COUNT, getInsecureMessageClause(), null, null, null, null)) {
if (cursor.moveToFirst()) {
return cursor.getInt(0);
}
}
return 0;
}
@Override
public void deleteExportedMessages() {
beginTransaction();
try {
List<Long> threadsToUpdate = new LinkedList<>();
try (Cursor cursor = getReadableDatabase().query(TABLE_NAME, THREAD_ID_PROJECTION, EXPORTED + " = ?", SqlUtil.buildArgs(1), THREAD_ID, null, null, null)) {
while (cursor.moveToNext()) {
threadsToUpdate.add(CursorUtil.requireLong(cursor, THREAD_ID));
}
}
getWritableDatabase().delete(TABLE_NAME, EXPORTED + " = ?", SqlUtil.buildArgs(1));
for (final long threadId : threadsToUpdate) {
SignalDatabase.threads().update(threadId, false);
}
SignalDatabase.attachments().deleteAbandonedAttachmentFiles();
setTransactionSuccessful();
} finally {
endTransaction();
}
}
@Override
void deleteThreads(@NonNull Set<Long> threadIds) {
Log.d(TAG, "deleteThreads(count: " + threadIds.size() + ")");
@@ -2664,6 +2718,25 @@ public class MmsDatabase extends MessageDatabase {
}
}
@Override
public @NonNull MessageExportState getMessageExportStateForCurrentRecord() {
byte[] messageExportState = CursorUtil.requireBlob(cursor, MmsDatabase.EXPORT_STATE);
if (messageExportState == null) {
return MessageExportState.getDefaultInstance();
}
try {
return MessageExportState.parseFrom(messageExportState);
} catch (InvalidProtocolBufferException e) {
return MessageExportState.getDefaultInstance();
}
}
public int getCount() {
if (cursor == null) return 0;
else return cursor.getCount();
}
private NotificationMmsMessageRecord getNotificationMmsMessageRecord(Cursor cursor) {
long id = cursor.getLong(cursor.getColumnIndexOrThrow(MmsDatabase.ID));
long dateSent = cursor.getLong(cursor.getColumnIndexOrThrow(MmsDatabase.NORMALIZED_DATE_SENT));

2
app/src/main/java/org/thoughtcrime/securesms/database/MmsSmsColumns.java
View File

@@ -29,6 +29,8 @@ public interface MmsSmsColumns {
public static final String REMOTE_DELETED = "remote_deleted";
public static final String SERVER_GUID = "server_guid";
public static final String RECEIPT_TIMESTAMP = "receipt_timestamp";
public static final String EXPORT_STATE = "export_state";
public static final String EXPORTED = "exported";
/**
* For storage efficiency, all types are stored within a single 64-bit integer column in the

65
app/src/main/java/org/thoughtcrime/securesms/database/MmsSmsDatabase.java
View File

@@ -16,6 +16,7 @@
*/
package org.thoughtcrime.securesms.database;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
@@ -24,21 +25,23 @@ import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import com.annimon.stream.Stream;
import com.google.protobuf.InvalidProtocolBufferException;
import net.zetetic.database.sqlcipher.SQLiteQueryBuilder;
import org.signal.core.util.CursorUtil;
import org.signal.core.util.SqlUtil;
import org.signal.core.util.logging.Log;
import org.signal.libsignal.protocol.util.Pair;
import org.thoughtcrime.securesms.database.MessageDatabase.MessageUpdate;
import org.thoughtcrime.securesms.database.MessageDatabase.SyncMessageId;
import org.thoughtcrime.securesms.database.model.MessageId;
import org.thoughtcrime.securesms.database.model.MessageRecord;
import org.thoughtcrime.securesms.database.model.databaseprotos.MessageExportState;
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
import org.thoughtcrime.securesms.notifications.v2.DefaultMessageNotifier;
import org.thoughtcrime.securesms.recipients.Recipient;
import org.thoughtcrime.securesms.recipients.RecipientId;
import org.signal.core.util.CursorUtil;
import org.signal.core.util.SqlUtil;
import org.whispersystems.signalservice.api.messages.multidevice.ReadMessage;
import java.io.Closeable;
@@ -50,6 +53,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import static org.thoughtcrime.securesms.database.MmsSmsColumns.Types.GROUP_V2_LEAVE_BITS;
@@ -613,6 +617,63 @@ public class MmsSmsDatabase extends Database {
SignalDatabase.mms().updateViewedStories(syncMessageIds);
}
private @NonNull MessageExportState getMessageExportState(@NonNull MessageId messageId) throws NoSuchMessageException {
String table = messageId.isMms() ? MmsDatabase.TABLE_NAME : SmsDatabase.TABLE_NAME;
String[] projection = SqlUtil.buildArgs(MmsSmsColumns.EXPORT_STATE);
String[] args = SqlUtil.buildArgs(messageId.getId());
try (Cursor cursor = getReadableDatabase().query(table, projection, ID_WHERE, args, null, null, null, null)) {
if (cursor.moveToFirst()) {
byte[] bytes = CursorUtil.requireBlob(cursor, MmsSmsColumns.EXPORT_STATE);
if (bytes == null) {
return MessageExportState.getDefaultInstance();
} else {
try {
return MessageExportState.parseFrom(bytes);
} catch (InvalidProtocolBufferException e) {
return MessageExportState.getDefaultInstance();
}
}
} else {
throw new NoSuchMessageException("The requested message does not exist.");
}
}
}
public void updateMessageExportState(@NonNull MessageId messageId, @NonNull Function<MessageExportState, MessageExportState> transform) throws NoSuchMessageException {
SQLiteDatabase database = getWritableDatabase();
database.beginTransaction();
try {
MessageExportState oldState = getMessageExportState(messageId);
MessageExportState newState = transform.apply(oldState);
setMessageExportState(messageId, newState);
database.setTransactionSuccessful();
} finally {
database.endTransaction();
}
}
public void markMessageExported(@NonNull MessageId messageId) {
String table = messageId.isMms() ? MmsDatabase.TABLE_NAME : SmsDatabase.TABLE_NAME;
ContentValues contentValues = new ContentValues(1);
contentValues.put(MmsSmsColumns.EXPORTED, 1);
getWritableDatabase().update(table, contentValues, ID_WHERE, SqlUtil.buildArgs(messageId.getId()));
}
private void setMessageExportState(@NonNull MessageId messageId, @NonNull MessageExportState messageExportState) {
String table = messageId.isMms() ? MmsDatabase.TABLE_NAME : SmsDatabase.TABLE_NAME;
ContentValues contentValues = new ContentValues(1);
contentValues.put(MmsSmsColumns.EXPORT_STATE, messageExportState.toByteArray());
getWritableDatabase().update(table, contentValues, ID_WHERE, SqlUtil.buildArgs(messageId.getId()));
}
/**
* @return Unhandled ids
*/

100
app/src/main/java/org/thoughtcrime/securesms/database/SmsDatabase.java
View File

@@ -29,6 +29,7 @@ import androidx.annotation.VisibleForTesting;
import com.annimon.stream.Stream;
import com.google.android.mms.pdu_alt.NotificationInd;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import net.zetetic.database.sqlcipher.SQLiteStatement;
@@ -47,6 +48,7 @@ import org.thoughtcrime.securesms.database.model.SmsMessageRecord;
import org.thoughtcrime.securesms.database.model.StoryResult;
import org.thoughtcrime.securesms.database.model.StoryViewState;
import org.thoughtcrime.securesms.database.model.databaseprotos.GroupCallUpdateDetails;
import org.thoughtcrime.securesms.database.model.databaseprotos.MessageExportState;
import org.thoughtcrime.securesms.database.model.databaseprotos.ProfileChangeDetails;
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
import org.thoughtcrime.securesms.groups.GroupMigrationMembershipChange;
@@ -66,15 +68,16 @@ import org.thoughtcrime.securesms.util.JsonUtils;
import org.thoughtcrime.securesms.util.TextSecurePreferences;
import org.thoughtcrime.securesms.util.Util;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
@@ -131,7 +134,9 @@ public class SmsDatabase extends MessageDatabase {
REMOTE_DELETED + " INTEGER DEFAULT 0, " +
NOTIFIED_TIMESTAMP + " INTEGER DEFAULT 0, " +
SERVER_GUID + " TEXT DEFAULT NULL, " +
RECEIPT_TIMESTAMP + " INTEGER DEFAULT -1);";
RECEIPT_TIMESTAMP + " INTEGER DEFAULT -1, " +
EXPORT_STATE + " BLOB DEFAULT NULL, " +
EXPORTED + " INTEGER DEFAULT 0);";
public static final String[] CREATE_INDEXS = {
"CREATE INDEX IF NOT EXISTS sms_read_and_notified_and_thread_id_index ON " + TABLE_NAME + "(" + READ + "," + NOTIFIED + "," + THREAD_ID + ");",
@@ -901,6 +906,51 @@ public class SmsDatabase extends MessageDatabase {
return ids;
}
@Override
public Cursor getUnexportedInsecureMessages() {
return queryMessages(
SqlUtil.appendArg(MESSAGE_PROJECTION, EXPORT_STATE),
getInsecureMessageClause() + " AND NOT " + EXPORTED,
null,
false,
-1
);
}
@Override
public int getInsecureMessageCount() {
try (Cursor cursor = getWritableDatabase().query(TABLE_NAME, SqlUtil.COUNT, getInsecureMessageClause(), null, null, null, null)) {
if (cursor.moveToFirst()) {
return cursor.getInt(0);
}
}
return 0;
}
@Override
public void deleteExportedMessages() {
beginTransaction();
try {
List<Long> threadsToUpdate = new LinkedList<>();
try (Cursor cursor = getReadableDatabase().query(TABLE_NAME, THREAD_ID_PROJECTION, EXPORTED + " = ?", SqlUtil.buildArgs(1), THREAD_ID, null, null, null)) {
while (cursor.moveToNext()) {
threadsToUpdate.add(CursorUtil.requireLong(cursor, THREAD_ID));
}
}
getWritableDatabase().delete(TABLE_NAME, EXPORTED + " = ?", SqlUtil.buildArgs(1));
for (final long threadId : threadsToUpdate) {
SignalDatabase.threads().update(threadId, false);
}
setTransactionSuccessful();
} finally {
endTransaction();
}
}
@Override
public List<MessageRecord> getProfileChangeDetailsRecords(long threadId, long afterTimestamp) {
String where = THREAD_ID + " = ? AND " + DATE_RECEIVED + " >= ? AND " + TYPE + " = ?";
@@ -1541,11 +1591,15 @@ public class SmsDatabase extends MessageDatabase {
}
}
private Cursor queryMessages(@NonNull String where, @NonNull String[] args, boolean reverse, long limit) {
private Cursor queryMessages(@NonNull String where, @Nullable String[] args, boolean reverse, long limit) {
return queryMessages(MESSAGE_PROJECTION, where, args, reverse, limit);
}
private Cursor queryMessages(@NonNull String[] projection, @NonNull String where, @Nullable String[] args, boolean reverse, long limit) {
SQLiteDatabase db = databaseHelper.getSignalReadableDatabase();
return db.query(TABLE_NAME,
MESSAGE_PROJECTION,
projection,
where,
args,
null,
@@ -1776,7 +1830,7 @@ public class SmsDatabase extends MessageDatabase {
}
}
public static class Reader implements Closeable {
public static class Reader implements MessageDatabase.Reader {
private final Cursor cursor;
private final Context context;
@@ -1798,6 +1852,20 @@ public class SmsDatabase extends MessageDatabase {
else return cursor.getCount();
}
@Override
public @NonNull MessageExportState getMessageExportStateForCurrentRecord() {
byte[] messageExportState = CursorUtil.requireBlob(cursor, SmsDatabase.EXPORT_STATE);
if (messageExportState == null) {
return MessageExportState.getDefaultInstance();
}
try {
return MessageExportState.parseFrom(messageExportState);
} catch (InvalidProtocolBufferException e) {
return MessageExportState.getDefaultInstance();
}
}
public SmsMessageRecord getCurrent() {
long messageId = cursor.getLong(cursor.getColumnIndexOrThrow(SmsDatabase.ID));
long recipientId = cursor.getLong(cursor.getColumnIndexOrThrow(SmsDatabase.RECIPIENT_ID));
@@ -1853,6 +1921,28 @@ public class SmsDatabase extends MessageDatabase {
public void close() {
cursor.close();
}
@Override
public @NonNull Iterator<MessageRecord> iterator() {
return new ReaderIterator();
}
private class ReaderIterator implements Iterator<MessageRecord> {
@Override
public boolean hasNext() {
return cursor != null && cursor.getCount() != 0 && !cursor.isLast();
}
@Override
public MessageRecord next() {
MessageRecord record = getNext();
if (record == null) {
throw new NoSuchElementException();
}
return record;
}
}
}
@VisibleForTesting

8
app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt
View File

@@ -29,6 +29,7 @@ import org.thoughtcrime.securesms.database.KeyValueDatabase
import org.thoughtcrime.securesms.database.RecipientDatabase
import org.thoughtcrime.securesms.database.helpers.migration.MyStoryMigration
import org.thoughtcrime.securesms.database.helpers.migration.PniSignaturesMigration
import org.thoughtcrime.securesms.database.helpers.migration.SmsExporterMigration
import org.thoughtcrime.securesms.database.helpers.migration.UrgentMslFlagMigration
import org.thoughtcrime.securesms.database.model.databaseprotos.ReactionList
import org.thoughtcrime.securesms.groups.GroupId
@@ -210,8 +211,9 @@ object SignalDatabaseMigrations {
private const val STORY_GROUP_TYPES = 152
private const val MY_STORY_MIGRATION_2 = 153
private const val PNI_SIGNATURES = 154
private const val SMS_EXPORTER = 155
const val DATABASE_VERSION = 154
const val DATABASE_VERSION = 155
@JvmStatic
fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
@@ -2696,6 +2698,10 @@ object SignalDatabaseMigrations {
if (oldVersion < PNI_SIGNATURES) {
PniSignaturesMigration.migrate(context, db, oldVersion, newVersion)
}
if (oldVersion < SMS_EXPORTER) {
SmsExporterMigration.migrate(context, db, oldVersion, newVersion)
}
}
@JvmStatic

16
app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/SmsExporterMigration.kt Normal file
View File

@@ -0,0 +1,16 @@
package org.thoughtcrime.securesms.database.helpers.migration
import android.app.Application
import net.zetetic.database.sqlcipher.SQLiteDatabase
/**
* Adds necessary book-keeping columns to SMS and MMS tables for SMS export.
*/
object SmsExporterMigration : SignalDatabaseMigration {
override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
db.execSQL("ALTER TABLE mms ADD COLUMN export_state BLOB DEFAULT NULL")
db.execSQL("ALTER TABLE mms ADD COLUMN exported INTEGER DEFAULT 0")
db.execSQL("ALTER TABLE sms ADD COLUMN export_state BLOB DEFAULT NULL")
db.execSQL("ALTER TABLE sms ADD COLUMN exported INTEGER DEFAULT 0")
}
}