From 86ef32cd4c050142b20f4963df8f0db09f3fd112 Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Mon, 11 Aug 2025 12:28:21 -0400 Subject: [PATCH] Do not load entire log into memory. --- .../logsubmit/LogSectionRemoteBackups.kt | 26 ++-- .../logsubmit/SubmitDebugLogActivity.java | 67 ++++----- .../logsubmit/SubmitDebugLogViewModel.java | 138 ++++++++++++------ .../signal/debuglogsviewer/DebugLogsViewer.kt | 2 +- 4 files changed, 140 insertions(+), 93 deletions(-) diff --git a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt index c1370b10d1..dfac279b69 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt @@ -13,6 +13,7 @@ import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.keyvalue.protos.ArchiveUploadProgressState class LogSectionRemoteBackups : LogSection { override fun getTitle(): String = "REMOTE BACKUPS" @@ -79,19 +80,22 @@ class LogSectionRemoteBackups : LogSection { output.append("\n -- ArchiveUploadProgress\n") if (SignalStore.backup.archiveUploadState != null) { output.append("State: ${SignalStore.backup.archiveUploadState}\n") - output.append("Pending bytes: ${SignalDatabase.attachments.getPendingArchiveUploadBytes()}\n") - val pendingAttachments = SignalDatabase.attachments.debugGetPendingArchiveUploadAttachments() - if (pendingAttachments.isNotEmpty()) { - output.append("Pending attachments:\n") - output.append(" Count: ${pendingAttachments.size}\n") - output.append(" Sum of Size: ${pendingAttachments.sumOf { it.size }}\n") - output.append(" Content types:\n") - pendingAttachments.groupBy { it.contentType }.forEach { (contentType, attachments) -> - output.append(" $contentType: ${attachments.size}\n") + if (SignalStore.backup.archiveUploadState!!.state !in setOf(ArchiveUploadProgressState.State.None, ArchiveUploadProgressState.State.UserCanceled)) { + output.append("Pending bytes: ${SignalDatabase.attachments.getPendingArchiveUploadBytes()}\n") + + val pendingAttachments = SignalDatabase.attachments.debugGetPendingArchiveUploadAttachments() + if (pendingAttachments.isNotEmpty()) { + output.append("Pending attachments:\n") + output.append(" Count: ${pendingAttachments.size}\n") + output.append(" Sum of Size: ${pendingAttachments.sumOf { it.size }}\n") + output.append(" Content types:\n") + pendingAttachments.groupBy { it.contentType }.forEach { (contentType, attachments) -> + output.append(" $contentType: ${attachments.size}\n") + } + } else { + output.append("Pending attachments: None!\n") } - } else { - output.append("Pending attachments: None!\n") } } else { output.append("None\n") diff --git a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogActivity.java b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogActivity.java index 191b6313bc..e5f9ef6898 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogActivity.java +++ b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogActivity.java @@ -44,6 +44,10 @@ import org.thoughtcrime.securesms.util.views.CircularProgressMaterialButton; import java.util.ArrayList; import java.util.List; +import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers; +import io.reactivex.rxjava3.disposables.CompositeDisposable; +import io.reactivex.rxjava3.disposables.Disposable; + public class SubmitDebugLogActivity extends BaseActivity { private static final int CODE_SAVE = 24601; @@ -82,10 +86,9 @@ public class SubmitDebugLogActivity extends BaseActivity { private boolean isInfo; private boolean isWarning; private boolean isError; - private boolean isWebViewLoaded; - private boolean hasPresentedLines; private final DynamicTheme dynamicTheme = new DynamicTheme(); + private final CompositeDisposable disposables = new CompositeDisposable(); @Override protected void onCreate(Bundle savedInstanceState) { @@ -327,10 +330,7 @@ public class SubmitDebugLogActivity extends BaseActivity { this.scrollToTopButton = findViewById(R.id.debug_log_scroll_to_top); this.progressCard = findViewById(R.id.debug_log_progress_card); - DebugLogsViewer.initWebView(logWebView, this, () -> { - isWebViewLoaded = true; - presentLines((viewModel.getLines().getValue() != null) ? viewModel.getLines().getValue() : new ArrayList<>()); - }); + DebugLogsViewer.initWebView(logWebView, this, this::subscribeToLogLines); submitButton.setOnClickListener(v -> onSubmitClicked()); scrollToTopButton.setOnClickListener(v -> DebugLogsViewer.scrollToTop(logWebView)); @@ -340,46 +340,32 @@ public class SubmitDebugLogActivity extends BaseActivity { } private void initViewModel() { - viewModel.getLines().observe(this, this::presentLines); viewModel.getMode().observe(this, this::presentMode); viewModel.getEvents().observe(this, this::presentEvents); } - private void presentLines(@NonNull List lines) { - if (!isWebViewLoaded || hasPresentedLines) { - return; - } + private void subscribeToLogLines() { + Disposable disposable = viewModel.getLogLinesObservable() + .observeOn(AndroidSchedulers.mainThread()) + .subscribe(this::presentLines, throwable -> { + // Handle error + this.progressCard.setVisibility(View.GONE); + }); + disposables.add(disposable); + } - if (progressCard != null && lines.size() > 0) { - progressCard.setVisibility(View.GONE); - warningBanner.setVisibility(View.VISIBLE); - submitButton.setVisibility(View.VISIBLE); - - hasPresentedLines = true; - } + private void presentLines(@NonNull List lines) { + warningBanner.setVisibility(View.VISIBLE); + submitButton.setVisibility(View.VISIBLE); SignalExecutors.BOUNDED.execute(() -> { - int chunkSize = 1000; - int count = 0; - StringBuilder lineBuilder = new StringBuilder(); - for (LogLine line : lines) { - if (line == null) continue; - - lineBuilder.append(String.format("%s\n", line.getText())); - count++; - - if (count >= chunkSize) { - DebugLogsViewer.presentLines(logWebView, lineBuilder.toString()); - lineBuilder.setLength(0); - count = 0; - } + for (String line : lines) { + lineBuilder.append(line).append("\n"); } - if (lineBuilder.length() > 0) { - DebugLogsViewer.presentLines(logWebView, lineBuilder.toString()); - } + DebugLogsViewer.appendLines(logWebView, lineBuilder.toString()); }); } @@ -389,14 +375,19 @@ public class SubmitDebugLogActivity extends BaseActivity { } switch (mode) { + case LOADING: + progressCard.setVisibility(View.VISIBLE); + break; case NORMAL: searchNav.setVisibility(View.GONE); saveMenuItem.setVisible(true); searchMenuItem.setVisible(true); + progressCard.setVisibility(View.GONE); break; case SUBMITTING: searchMenuItem.setVisible(false); saveMenuItem.setVisible(false); + progressCard.setVisibility(View.VISIBLE); break; } } @@ -466,4 +457,10 @@ public class SubmitDebugLogActivity extends BaseActivity { submitButton.cancelSpinning(); }); } + + @Override + protected void onDestroy() { + super.onDestroy(); + disposables.dispose(); + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogViewModel.java b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogViewModel.java index de0e5aa70e..a5d719a7df 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogViewModel.java +++ b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogViewModel.java @@ -5,11 +5,11 @@ import android.net.Uri; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.lifecycle.LiveData; -import androidx.lifecycle.MediatorLiveData; import androidx.lifecycle.MutableLiveData; import androidx.lifecycle.ViewModel; import androidx.lifecycle.ViewModelProvider; +import org.signal.core.util.Stopwatch; import org.signal.core.util.ThreadUtil; import org.signal.core.util.concurrent.SignalExecutors; import org.signal.core.util.logging.Log; @@ -22,55 +22,98 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import io.reactivex.rxjava3.core.Observable; +import io.reactivex.rxjava3.schedulers.Schedulers; + public class SubmitDebugLogViewModel extends ViewModel { private static final String TAG = Log.tag(SubmitDebugLogViewModel.class); - private final SubmitDebugLogRepository repo; - private final MutableLiveData mode; - private final List staticLines; - private final MediatorLiveData> lines; - private final SingleLiveEvent event; - private final long firstViewTime; - private final byte[] trace; - private final List allLines; + private static final int CHUNK_SIZE = 10_000; + + private final SubmitDebugLogRepository repo; + private final MutableLiveData mode; + private final SingleLiveEvent event; + private final long firstViewTime; + private final byte[] trace; private SubmitDebugLogViewModel() { - this.repo = new SubmitDebugLogRepository(); - this.mode = new MutableLiveData<>(); - this.trace = Tracer.getInstance().serialize(); - this.firstViewTime = System.currentTimeMillis(); - this.staticLines = new ArrayList<>(); - this.lines = new MediatorLiveData<>(); - this.event = new SingleLiveEvent<>(); - this.allLines = new ArrayList<>(); - - repo.getPrefixLogLines(staticLines -> { - this.staticLines.addAll(staticLines); - - Log.blockUntilAllWritesFinished(); - LogDatabase.getInstance(AppDependencies.getApplication()).logs().trimToSize(); - SignalExecutors.UNBOUNDED.execute(() -> { - allLines.clear(); - allLines.addAll(staticLines); - - try (LogDatabase.LogTable.CursorReader logReader = (LogDatabase.LogTable.CursorReader) LogDatabase.getInstance(AppDependencies.getApplication()).logs().getAllBeforeTime(firstViewTime)) { - while (logReader.hasNext()) { - String next = logReader.next(); - allLines.add(new SimpleLogLine(next, LogStyleParser.parseStyle(next), LogLine.Placeholder.NONE)); - } - } - - ThreadUtil.runOnMain(() -> { - lines.setValue(allLines); - mode.setValue(Mode.NORMAL); - }); - }); - }); + this.repo = new SubmitDebugLogRepository(); + this.mode = new MutableLiveData<>(); + this.trace = Tracer.getInstance().serialize(); + this.firstViewTime = System.currentTimeMillis(); + this.event = new SingleLiveEvent<>(); } - @NonNull LiveData> getLines() { - return lines; + @NonNull Observable> getLogLinesObservable() { + return Observable.>create(emitter -> { + Stopwatch stopwatch = new Stopwatch("log-loading"); + try { + mode.postValue(Mode.LOADING); + + repo.getPrefixLogLines(prefixLines -> { + try { + List prefixStrings = new ArrayList<>(); + for (LogLine line : prefixLines) { + prefixStrings.add(line.getText()); + } + stopwatch.split("prefix"); + + Log.blockUntilAllWritesFinished(); + stopwatch.split("flush"); + + LogDatabase.getInstance(AppDependencies.getApplication()).logs().trimToSize(); + stopwatch.split("trim-old"); + + if (!emitter.isDisposed()) { + emitter.onNext(new ArrayList<>(prefixStrings)); + } + + List currentChunk = new ArrayList<>(); + + try (LogDatabase.LogTable.CursorReader logReader = (LogDatabase.LogTable.CursorReader) LogDatabase.getInstance(AppDependencies.getApplication()).logs().getAllBeforeTime(firstViewTime)) { + stopwatch.split("initial-query"); + + int count = 0; + while (logReader.hasNext() && !emitter.isDisposed()) { + String next = logReader.next(); + currentChunk.add(next); + count++; + + if (count >= CHUNK_SIZE) { + emitter.onNext(currentChunk); + count = 0; + currentChunk = new ArrayList<>(); + } + } + + // Send final chunk if any remaining + if (!emitter.isDisposed() && count > 0) { + emitter.onNext(currentChunk); + } + + if (!emitter.isDisposed()) { + mode.postValue(Mode.NORMAL); + emitter.onComplete(); + } + + stopwatch.split("lines"); + stopwatch.stop(TAG); + } + } catch (Exception e) { + if (!emitter.isDisposed()) { + Log.e(TAG, "Error loading log lines", e); + emitter.onError(e); + } + } + }); + } catch (Exception e) { + if (!emitter.isDisposed()) { + Log.e(TAG, "Error creating log lines observable", e); + emitter.onError(e); + } + } + }).subscribeOn(Schedulers.io()); } @NonNull LiveData getMode() { @@ -82,9 +125,12 @@ public class SubmitDebugLogViewModel extends ViewModel { MutableLiveData> result = new MutableLiveData<>(); - repo.submitLogWithPrefixLines(firstViewTime, staticLines, trace, value -> { - mode.postValue(Mode.NORMAL); - result.postValue(value); + // Get prefix lines for submission - this is a quick operation so it's ok to do it here + repo.getPrefixLogLines(prefixLines -> { + repo.submitLogWithPrefixLines(firstViewTime, prefixLines, trace, value -> { + mode.postValue(Mode.NORMAL); + result.postValue(value); + }); }); return result; @@ -115,7 +161,7 @@ public class SubmitDebugLogViewModel extends ViewModel { } enum Mode { - NORMAL, SUBMITTING + NORMAL, LOADING, SUBMITTING } enum Event { diff --git a/debuglogs-viewer/lib/src/main/java/org/signal/debuglogsviewer/DebugLogsViewer.kt b/debuglogs-viewer/lib/src/main/java/org/signal/debuglogsviewer/DebugLogsViewer.kt index 91e154401c..2ab95c04c9 100644 --- a/debuglogs-viewer/lib/src/main/java/org/signal/debuglogsviewer/DebugLogsViewer.kt +++ b/debuglogs-viewer/lib/src/main/java/org/signal/debuglogsviewer/DebugLogsViewer.kt @@ -41,7 +41,7 @@ object DebugLogsViewer { } @JvmStatic - fun presentLines(webview: WebView, lines: String) { + fun appendLines(webview: WebView, lines: String) { // Set the debug log lines val escaped = JSONObject.quote(lines) ThreadUtil.runOnMain { webview.evaluateJavascript("editor.insert($escaped); logLines+=$escaped;", null) }