Move logging into a database.

app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogDataSource.kt

@@ -0,0 +1,42 @@
+package org.thoughtcrime.securesms.logsubmit
+
+import android.app.Application
+import org.signal.paging.PagedDataSource
+import org.thoughtcrime.securesms.database.LogDatabase
+import org.thoughtcrime.securesms.logsubmit.util.Scrubber
+
+/**
+ * Retrieves logs to show in the [SubmitDebugLogActivity].
+ *
+ * @param prefixLines A static list of lines to show before all of the lines retrieved from [LogDatabase]
+ * @param untilTime Only show logs before this time. This is our way of making sure the set of logs we show on this screen doesn't grow.
+ */
+class LogDataSource(
+  application: Application,
+  private val prefixLines: List<LogLine>,
+  private val untilTime: Long
+) :
+  PagedDataSource<LogLine> {
+
+  val logDatabase = LogDatabase.getInstance(application)
+
+  override fun size(): Int {
+    return prefixLines.size + logDatabase.getLogCountBeforeTime(untilTime)
+  }
+
+  override fun load(start: Int, length: Int, cancellationSignal: PagedDataSource.CancellationSignal): List<LogLine> {
+    if (start + length < prefixLines.size) {
+      return prefixLines.subList(start, start + length)
+    } else if (start < prefixLines.size) {
+      return prefixLines.subList(start, prefixLines.size) +
+        logDatabase.getRangeBeforeTime(0, length - (prefixLines.size - start), untilTime).map { convertToLogLine(it) }
+    } else {
+      return logDatabase.getRangeBeforeTime(start - prefixLines.size, length, untilTime).map { convertToLogLine(it) }
+    }
+  }
+
+  private fun convertToLogLine(raw: String): LogLine {
+    val scrubbed: String = Scrubber.scrub(raw).toString()
+    return SimpleLogLine(scrubbed, LogStyleParser.parseStyle(scrubbed), LogLine.Placeholder.NONE)
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionLoggerHeader.java

@@ -4,9 +4,10 @@ import android.content.Context;
 
 import androidx.annotation.NonNull;
 
-import org.thoughtcrime.securesms.ApplicationContext;
-
-public class LogSectionLogger implements LogSection {
+/**
+ * Because the actual contents of this section are paged from the database, this class just has a header and no content.
+ */
+public class LogSectionLoggerHeader implements LogSection {
 
   @Override
   public @NonNull String getTitle() {
@@ -15,7 +16,6 @@ public class LogSectionLogger implements LogSection {
 
   @Override
   public @NonNull CharSequence getContent(@NonNull Context context) {
-    CharSequence logs = ApplicationContext.getInstance(context).getPersistentLogger().getLogs();
-    return logs != null ? logs : "Unable to retrieve logs.";
+    return "";
   }
 }

app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogActivity.java

@@ -60,6 +60,8 @@ public class SubmitDebugLogActivity extends BaseActivity implements SubmitDebugL
     getSupportActionBar().setDisplayHomeAsUpEnabled(true);
     getSupportActionBar().setTitle(R.string.HelpSettingsFragment__debug_log);
 
+    this.viewModel = ViewModelProviders.of(this, new SubmitDebugLogViewModel.Factory()).get(SubmitDebugLogViewModel.class);
+
     initView();
     initViewModel();
   }
@@ -115,16 +117,13 @@ public class SubmitDebugLogActivity extends BaseActivity implements SubmitDebugL
   public boolean onOptionsItemSelected(MenuItem item) {
     super.onOptionsItemSelected(item);
 
-    switch (item.getItemId()) {
-      case android.R.id.home:
-        finish();
-        return true;
-      case R.id.menu_edit_log:
-        viewModel.onEditButtonPressed();
-        break;
-      case R.id.menu_done_editing_log:
-        viewModel.onDoneEditingButtonPressed();
-        break;
+    if (item.getItemId() == android.R.id.home) {
+      finish();
+      return true;
+    } else if (item.getItemId() == R.id.menu_edit_log) {
+      viewModel.onEditButtonPressed();
+    } else if (item.getItemId() == R.id.menu_done_editing_log) {
+      viewModel.onDoneEditingButtonPressed();
     }
 
     return false;
@@ -150,10 +149,11 @@ public class SubmitDebugLogActivity extends BaseActivity implements SubmitDebugL
     this.scrollToBottomButton = findViewById(R.id.debug_log_scroll_to_bottom);
     this.scrollToTopButton    = findViewById(R.id.debug_log_scroll_to_top);
 
-    this.adapter = new SubmitDebugLogAdapter(this);
+    this.adapter = new SubmitDebugLogAdapter(this, viewModel.getPagingController());
 
     this.lineList.setLayoutManager(new LinearLayoutManager(this));
     this.lineList.setAdapter(adapter);
+    this.lineList.setItemAnimator(null);
 
     submitButton.setOnClickListener(v -> onSubmitClicked());
 
@@ -181,8 +181,6 @@ public class SubmitDebugLogActivity extends BaseActivity implements SubmitDebugL
   }
 
   private void initViewModel() {
-    this.viewModel = ViewModelProviders.of(this, new SubmitDebugLogViewModel.Factory()).get(SubmitDebugLogViewModel.class);
-
     viewModel.getLines().observe(this, this::presentLines);
     viewModel.getMode().observe(this, this::presentMode);
   }
@@ -196,7 +194,7 @@ public class SubmitDebugLogActivity extends BaseActivity implements SubmitDebugL
       submitButton.setVisibility(View.VISIBLE);
     }
 
-    adapter.setLines(lines);
+    adapter.submitList(lines);
   }
 
   private void presentMode(@NonNull SubmitDebugLogViewModel.Mode mode) {
@@ -204,9 +202,10 @@ public class SubmitDebugLogActivity extends BaseActivity implements SubmitDebugL
       case NORMAL:
         editBanner.setVisibility(View.GONE);
         adapter.setEditing(false);
-        editMenuItem.setVisible(true);
-        doneMenuItem.setVisible(false);
-        searchMenuItem.setVisible(true);
+        // TODO [greyson][log] Not yet implemented
+//        editMenuItem.setVisible(true);
+//        doneMenuItem.setVisible(false);
+//        searchMenuItem.setVisible(true);
         break;
       case SUBMITTING:
         editBanner.setVisibility(View.GONE);

app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogAdapter.java

@@ -10,8 +10,7 @@ import androidx.annotation.NonNull;
 import androidx.core.content.ContextCompat;
 import androidx.recyclerview.widget.RecyclerView;
 
-import com.annimon.stream.Stream;
-
+import org.signal.paging.PagingController;
 import org.thoughtcrime.securesms.R;
 import org.thoughtcrime.securesms.components.ListenableHorizontalScrollView;
 
@@ -21,26 +20,52 @@ import java.util.concurrent.CopyOnWriteArrayList;
 
 public class SubmitDebugLogAdapter extends RecyclerView.Adapter<SubmitDebugLogAdapter.LineViewHolder> {
 
-  private static final int MAX_LINE_LENGTH = 1000;
+  private static final int LINE_LENGTH = 500;
 
-  private final List<LogLine> lines;
-  private final ScrollManager scrollManager;
-  private final Listener      listener;
+  private static final int TYPE_LOG         = 1;
+  private static final int TYPE_PLACEHOLDER = 2;
+
+  private final ScrollManager    scrollManager;
+  private final Listener         listener;
+  private final PagingController pagingController;
+  private final List<LogLine>    lines;
 
   private boolean editing;
-  private int     longestLine;
 
-  public SubmitDebugLogAdapter(@NonNull Listener listener) {
-    this.listener      = listener;
-    this.lines         = new ArrayList<>();
-    this.scrollManager = new ScrollManager();
+  public SubmitDebugLogAdapter(@NonNull Listener listener, @NonNull PagingController pagingController) {
+    this.listener         = listener;
+    this.pagingController = pagingController;
+    this.scrollManager    = new ScrollManager();
+    this.lines            = new ArrayList<>();
 
     setHasStableIds(true);
   }
 
   @Override
   public long getItemId(int position) {
-    return lines.get(position).getId();
+    LogLine item = getItem(position);
+    return item != null ? getItem(position).getId() : -1;
+  }
+
+  @Override
+  public int getItemViewType(int position) {
+    return getItem(position) == null ? TYPE_PLACEHOLDER : TYPE_LOG;
+  }
+
+  protected LogLine getItem(int position) {
+    pagingController.onDataNeededAroundIndex(position);
+    return lines.get(position);
+  }
+
+  public void submitList(@NonNull List<LogLine> list) {
+    this.lines.clear();
+    this.lines.addAll(list);
+    notifyDataSetChanged();
+  }
+
+  @Override
+  public int getItemCount() {
+    return lines.size();
   }
 
   @Override
@@ -50,7 +75,13 @@ public class SubmitDebugLogAdapter extends RecyclerView.Adapter<SubmitDebugLogAd
 
   @Override
   public void onBindViewHolder(@NonNull LineViewHolder holder, int position) {
-    holder.bind(lines.get(position), longestLine, editing, scrollManager, listener);
+    LogLine item = getItem(position);
+
+    if (item == null) {
+      item = SimpleLogLine.EMPTY;
+    }
+
+    holder.bind(item, LINE_LENGTH, editing, scrollManager, listener);
   }
 
   @Override
@@ -58,21 +89,6 @@ public class SubmitDebugLogAdapter extends RecyclerView.Adapter<SubmitDebugLogAd
     holder.unbind(scrollManager);
   }
 
-  @Override
-  public int getItemCount() {
-    return lines.size();
-  }
-
-  public void setLines(@NonNull List<LogLine> lines) {
-    this.lines.clear();
-    this.lines.addAll(lines);
-
-    this.longestLine = Stream.of(lines).reduce(0, (currentMax, line) -> Math.max(currentMax, line.getText().length()));
-    this.longestLine = Math.min(longestLine, MAX_LINE_LENGTH);
-
-    notifyDataSetChanged();
-  }
-
   public void setEditing(boolean editing) {
     this.editing = editing;
     notifyDataSetChanged();

app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogRepository.java

@@ -1,7 +1,10 @@
 package org.thoughtcrime.securesms.logsubmit;
 
+import android.app.Application;
 import android.content.Context;
+import android.net.Uri;
 import android.os.Build;
+import android.os.ParcelFileDescriptor;
 
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
@@ -11,20 +14,26 @@ import com.annimon.stream.Stream;
 
 import org.json.JSONException;
 import org.json.JSONObject;
+import org.signal.core.util.StreamUtil;
 import org.signal.core.util.concurrent.SignalExecutors;
 import org.signal.core.util.logging.Log;
+import org.signal.core.util.tracing.Tracer;
+import org.thoughtcrime.securesms.database.LogDatabase;
 import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
 import org.thoughtcrime.securesms.logsubmit.util.Scrubber;
 import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
+import org.thoughtcrime.securesms.providers.BlobProvider;
 import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess;
 import org.whispersystems.libsignal.util.guava.Optional;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
 
 import okhttp3.MediaType;
 import okhttp3.MultipartBody;
@@ -33,6 +42,9 @@ import okhttp3.Request;
 import okhttp3.RequestBody;
 import okhttp3.Response;
 import okhttp3.ResponseBody;
+import okio.BufferedSink;
+import okio.Okio;
+import okio.Source;
 
 /**
  * Handles retrieving, scrubbing, and uploading of all debug logs.
@@ -68,10 +80,10 @@ public class SubmitDebugLogRepository {
     add(new LogSectionThreads());
     add(new LogSectionBlockedThreads());
     add(new LogSectionLogcat());
-    add(new LogSectionLogger());
+    add(new LogSectionLoggerHeader());
   }};
 
-  private final Context         context;
+  private final Application     context;
   private final ExecutorService executor;
 
   public SubmitDebugLogRepository() {
@@ -79,44 +91,88 @@ public class SubmitDebugLogRepository {
     this.executor = SignalExecutors.SERIAL;
   }
 
-  public void getLogLines(@NonNull Callback<List<LogLine>> callback) {
-    executor.execute(() -> callback.onResult(getLogLinesInternal()));
+  public void getPrefixLogLines(@NonNull Callback<List<LogLine>> callback) {
+    executor.execute(() -> callback.onResult(getPrefixLogLinesInternal()));
   }
 
-  public void submitLog(@NonNull List<LogLine> lines, Callback<Optional<String>> callback) {
-    SignalExecutors.UNBOUNDED.execute(() -> callback.onResult(submitLogInternal(lines, null)));
+  public void buildAndSubmitLog(@NonNull Callback<Optional<String>> callback) {
+    SignalExecutors.UNBOUNDED.execute(() -> callback.onResult(submitLogInternal(System.currentTimeMillis(), getPrefixLogLinesInternal(), Tracer.getInstance().serialize())));
   }
 
-  public void submitLog(@NonNull List<LogLine> lines, @Nullable byte[] trace, Callback<Optional<String>> callback) {
-    SignalExecutors.UNBOUNDED.execute(() -> callback.onResult(submitLogInternal(lines, trace)));
+  /**
+   * Submits a log with the provided prefix lines.
+   *
+   * @param untilTime Only submit logs from {@link LogDatabase} if they were created before this time. This is our way of making sure that the logs we submit
+   *                  only include the logs that we've already shown the user. It's possible some old logs may have been trimmed off in the meantime, but no
+   *                  new ones could pop up.
+   */
+  public void submitLogWithPrefixLines(long untilTime, @NonNull List<LogLine> prefixLines, @Nullable byte[] trace, Callback<Optional<String>> callback) {
+    SignalExecutors.UNBOUNDED.execute(() -> callback.onResult(submitLogInternal(untilTime, prefixLines, trace)));
   }
 
   @WorkerThread
-  private @NonNull Optional<String> submitLogInternal(@NonNull List<LogLine> lines, @Nullable byte[] trace) {
+  private @NonNull Optional<String> submitLogInternal(long untilTime, @NonNull List<LogLine> prefixLines, @Nullable byte[] trace) {
     String traceUrl = null;
     if (trace != null) {
       try {
-        traceUrl = uploadContent("application/octet-stream", trace);
+        traceUrl = uploadContent("application/octet-stream", RequestBody.create(MediaType.get("application/octet-stream"), trace));
       } catch (IOException e) {
         Log.w(TAG, "Error during trace upload.", e);
         return Optional.absent();
       }
     }
 
-    StringBuilder bodyBuilder = new StringBuilder();
-    for (LogLine line : lines) {
+    StringBuilder prefixStringBuilder = new StringBuilder();
+    for (LogLine line : prefixLines) {
       switch (line.getPlaceholderType()) {
         case NONE:
-          bodyBuilder.append(line.getText()).append('\n');
+          prefixStringBuilder.append(line.getText()).append('\n');
           break;
         case TRACE:
-          bodyBuilder.append(traceUrl).append('\n');
+          prefixStringBuilder.append(traceUrl).append('\n');
           break;
       }
     }
 
     try {
-      String logUrl = uploadContent("text/plain", bodyBuilder.toString().getBytes());
+      ParcelFileDescriptor[] fds     = ParcelFileDescriptor.createPipe();
+      Uri                    gzipUri = BlobProvider.getInstance()
+                                                   .forData(new ParcelFileDescriptor.AutoCloseInputStream(fds[0]), 0)
+                                                   .withMimeType("application/gzip")
+                                                   .createForSingleSessionOnDiskAsync(context, null, null);
+
+      OutputStream gzipOutput = new GZIPOutputStream(new ParcelFileDescriptor.AutoCloseOutputStream(fds[1]));
+
+      gzipOutput.write(prefixStringBuilder.toString().getBytes());
+
+      try (LogDatabase.Reader reader = LogDatabase.getInstance(context).getAllBeforeTime(untilTime)) {
+        while (reader.hasNext()) {
+          gzipOutput.write(Scrubber.scrub(reader.next()).toString().getBytes());
+          gzipOutput.write("\n".getBytes());
+        }
+      }
+
+      StreamUtil.close(gzipOutput);
+
+      String logUrl = uploadContent("application/gzip", new RequestBody() {
+        @Override
+        public @NonNull MediaType contentType() {
+          return MediaType.get("application/gzip");
+        }
+
+        @Override public long contentLength() {
+          return BlobProvider.getInstance().calculateFileSize(context, gzipUri);
+        }
+
+        @Override
+        public void writeTo(@NonNull BufferedSink sink) throws IOException {
+          Source source = Okio.source(BlobProvider.getInstance().getStream(context, gzipUri));
+          sink.writeAll(source);
+        }
+      });
+
+      BlobProvider.getInstance().delete(context, gzipUri);
+
       return Optional.of(logUrl);
     } catch (IOException e) {
       Log.w(TAG, "Error during log upload.", e);
@@ -125,7 +181,7 @@ public class SubmitDebugLogRepository {
   }
 
   @WorkerThread
-  private @NonNull String uploadContent(@NonNull String contentType, @NonNull byte[] content) throws IOException {
+  private @NonNull String uploadContent(@NonNull String contentType, @NonNull RequestBody requestBody) throws IOException {
     try {
       OkHttpClient client   = new OkHttpClient.Builder().addInterceptor(new StandardUserAgentInterceptor()).dns(SignalServiceNetworkAccess.DNS).build();
       Response     response = client.newCall(new Request.Builder().url(API_ENDPOINT).get().build()).execute();
@@ -149,7 +205,7 @@ public class SubmitDebugLogRepository {
         post.addFormDataPart(key, fields.getString(key));
       }
 
-      post.addFormDataPart("file", "file", RequestBody.create(MediaType.parse(contentType), content));
+      post.addFormDataPart("file", "file", requestBody);
 
       Response postResponse = client.newCall(new Request.Builder().url(url).post(post.build()).build()).execute();
 
@@ -165,7 +221,7 @@ public class SubmitDebugLogRepository {
   }
 
   @WorkerThread
-  private @NonNull List<LogLine> getLogLinesInternal() {
+  private @NonNull List<LogLine> getPrefixLogLinesInternal() {
     long startTime = System.currentTimeMillis();
 
     int maxTitleLength = Stream.of(SECTIONS).reduce(0, (max, section) -> Math.max(max, section.getTitle().length()));

app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogViewModel.java

@@ -1,42 +1,60 @@
 package org.thoughtcrime.securesms.logsubmit;
 
-import android.text.TextUtils;
-
 import androidx.annotation.NonNull;
 import androidx.lifecycle.LiveData;
+import androidx.lifecycle.MediatorLiveData;
 import androidx.lifecycle.MutableLiveData;
 import androidx.lifecycle.ViewModel;
 import androidx.lifecycle.ViewModelProvider;
 
-import com.annimon.stream.Stream;
-
+import org.signal.core.util.ThreadUtil;
 import org.signal.core.util.tracing.Tracer;
+import org.signal.paging.PagedData;
+import org.signal.paging.PagingConfig;
+import org.signal.paging.PagingController;
+import org.signal.paging.ProxyPagingController;
 import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
-import org.thoughtcrime.securesms.util.DefaultValueLiveData;
 import org.whispersystems.libsignal.util.guava.Optional;
 
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.List;
 
 public class SubmitDebugLogViewModel extends ViewModel {
 
-  private final SubmitDebugLogRepository            repo;
-  private final DefaultValueLiveData<List<LogLine>> lines;
-  private final MutableLiveData<Mode>               mode;
+  private final SubmitDebugLogRepository        repo;
+  private final MutableLiveData<Mode>           mode;
+  private final ProxyPagingController           pagingController;
+  private final List<LogLine>                   staticLines;
+  private final MediatorLiveData<List<LogLine>> lines;
+  private final long                            firstViewTime;
+  private final byte[]                          trace;
 
-  private List<LogLine> sourceLines;
-  private byte[]        trace;
 
   private SubmitDebugLogViewModel() {
-    this.repo  = new SubmitDebugLogRepository();
-    this.lines = new DefaultValueLiveData<>(Collections.emptyList());
-    this.mode  = new MutableLiveData<>();
-    this.trace = Tracer.getInstance().serialize();
+    this.repo             = new SubmitDebugLogRepository();
+    this.mode             = new MutableLiveData<>();
+    this.trace            = Tracer.getInstance().serialize();
+    this.pagingController = new ProxyPagingController();
+    this.firstViewTime    = System.currentTimeMillis();
+    this.staticLines      = new ArrayList<>();
+    this.lines            = new MediatorLiveData<>();
 
-    repo.getLogLines(result -> {
-      sourceLines = result;
-      mode.postValue(Mode.NORMAL);
-      lines.postValue(sourceLines);
+    repo.getPrefixLogLines(staticLines -> {
+      this.staticLines.addAll(staticLines);
+
+      LogDataSource dataSource = new LogDataSource(ApplicationDependencies.getApplication(), staticLines, firstViewTime);
+      PagingConfig  config     = new PagingConfig.Builder().setPageSize(100)
+                                                           .setBufferPages(3)
+                                                           .setStartIndex(0)
+                                                           .build();
+
+      PagedData<LogLine> pagedData = PagedData.create(dataSource, config);
+
+      ThreadUtil.runOnMain(() -> {
+        pagingController.set(pagedData.getController());
+        lines.addSource(pagedData.getData(), lines::setValue);
+        mode.setValue(Mode.NORMAL);
+      });
     });
   }
 
@@ -44,6 +62,10 @@ public class SubmitDebugLogViewModel extends ViewModel {
     return lines;
   }
 
+  @NonNull PagingController getPagingController() {
+    return pagingController;
+  }
+
   @NonNull LiveData<Mode> getMode() {
     return mode;
   }
@@ -53,7 +75,7 @@ public class SubmitDebugLogViewModel extends ViewModel {
 
     MutableLiveData<Optional<String>> result = new MutableLiveData<>();
 
-    repo.submitLog(lines.getValue(), trace, value -> {
+    repo.submitLogWithPrefixLines(firstViewTime, staticLines, trace, value -> {
       mode.postValue(Mode.NORMAL);
       result.postValue(value);
     });
@@ -62,35 +84,23 @@ public class SubmitDebugLogViewModel extends ViewModel {
   }
 
   void onQueryUpdated(@NonNull String query) {
-    if (TextUtils.isEmpty(query)) {
-      lines.postValue(sourceLines);
-    } else {
-      List<LogLine> filtered = Stream.of(sourceLines)
-                                     .filter(l -> l.getText().toLowerCase().contains(query.toLowerCase()))
-                                     .toList();
-      lines.postValue(filtered);
-    }
+    throw new UnsupportedOperationException("Not yet implemented.");
   }
 
   void onSearchClosed() {
-    lines.postValue(sourceLines);
+    throw new UnsupportedOperationException("Not yet implemented.");
   }
 
   void onEditButtonPressed() {
-    mode.setValue(Mode.EDIT);
+    throw new UnsupportedOperationException("Not yet implemented.");
   }
 
   void onDoneEditingButtonPressed() {
-    mode.setValue(Mode.NORMAL);
+    throw new UnsupportedOperationException("Not yet implemented.");
   }
 
   void onLogDeleted(@NonNull LogLine line) {
-    sourceLines.remove(line);
-
-    List<LogLine> logs = lines.getValue();
-    logs.remove(line);
-
-    lines.postValue(logs);
+    throw new UnsupportedOperationException("Not yet implemented.");
   }
 
   boolean onBackPressed() {