Eliminate orphaned external message files on startup

Attachments, visual attachment thumbnails, video attachment screenshots
Quote thumbnails
Contact avatars

app/sql.js

@@ -4,7 +4,7 @@ const rimraf = require('rimraf');
 const sql = require('@journeyapps/sqlcipher');
 const pify = require('pify');
 const uuidv4 = require('uuid/v4');
-const { map, isString } = require('lodash');
+const { map, isString, fromPairs, forEach, last } = require('lodash');
 
 // To get long stack traces
 //   https://github.com/mapbox/node-sqlite3/wiki/API#sqlite3verbose
@@ -15,6 +15,7 @@ module.exports = {
   close,
   removeDB,
 
+  getMessageCount,
   saveMessage,
   saveMessages,
   removeMessage,
@@ -39,6 +40,8 @@ module.exports = {
   getMessagesNeedingUpgrade,
   getMessagesWithVisualMediaAttachments,
   getMessagesWithFileAttachments,
+
+  removeKnownAttachments,
 };
 
 function generateUUID() {
@@ -260,6 +263,16 @@ async function removeDB() {
   rimraf.sync(filePath);
 }
 
+async function getMessageCount() {
+  const row = await db.get('SELECT count(*) from messages;');
+
+  if (!row) {
+    throw new Error('getMessageCount: Unable to get count of messages');
+  }
+
+  return row['count(*)'];
+}
+
 async function saveMessage(data, { forceSave } = {}) {
   const {
     conversationId,
@@ -710,3 +723,92 @@ async function getMessagesWithFileAttachments(conversationId, { limit }) {
 
   return map(rows, row => jsonToObject(row.json));
 }
+
+function getExternalFilesForMessage(message) {
+  const { attachments, contact, quote } = message;
+  const files = [];
+
+  forEach(attachments, attachment => {
+    const { path: file, thumbnail, screenshot } = attachment;
+    if (file) {
+      files.push(file);
+    }
+
+    if (thumbnail && thumbnail.path) {
+      files.push(thumbnail.path);
+    }
+
+    if (screenshot && screenshot.path) {
+      files.push(screenshot.path);
+    }
+  });
+
+  if (quote && quote.attachments && quote.attachments.length) {
+    forEach(quote.attachments, attachment => {
+      const { thumbnail } = attachment;
+
+      if (thumbnail && thumbnail.path) {
+        files.push(thumbnail.path);
+      }
+    });
+  }
+
+  if (contact && contact.length) {
+    forEach(contact, item => {
+      const { avatar } = item;
+
+      if (avatar && avatar.avatar && avatar.avatar.path) {
+        files.push(avatar.avatar.path);
+      }
+    });
+  }
+
+  return files;
+}
+
+async function removeKnownAttachments(allAttachments) {
+  const lookup = fromPairs(map(allAttachments, file => [file, true]));
+  const chunkSize = 50;
+
+  const total = await getMessageCount();
+  console.log(
+    `removeKnownAttachments: About to iterate through ${total} messages`
+  );
+
+  let count = 0;
+  let complete = false;
+  let id = '';
+
+  while (!complete) {
+    // eslint-disable-next-line no-await-in-loop
+    const rows = await db.all(
+      `SELECT json FROM messages
+       WHERE id > $id
+       ORDER BY id ASC
+       LIMIT $chunkSize;`,
+      {
+        $id: id,
+        $chunkSize: chunkSize,
+      }
+    );
+
+    const messages = map(rows, row => jsonToObject(row.json));
+    forEach(messages, message => {
+      const externalFiles = getExternalFilesForMessage(message);
+      forEach(externalFiles, file => {
+        delete lookup[file];
+      });
+    });
+
+    const lastMessage = last(messages);
+    if (lastMessage) {
+      ({ id } = lastMessage);
+    }
+    complete = messages.length < chunkSize;
+    count += messages.length;
+  }
+
+  console.log(`removeKnownAttachments: Done processing ${count} messages`);
+
+  return Object.keys(lookup);
+}