Multi-recipient message views

This adds support for storing multi-recipient message payloads and recipient views in Redis, and only fanning out on delivery or persistence. Phase 1: confirm storage and retrieval correctness.

service/src/main/resources/lua/apn/get.lua

@@ -46,7 +46,7 @@ local getNextInterval = function(interval)
 end
 
 
-local results  = redis.call("ZRANGEBYSCORE", pendingNotificationQueue, 0, maxTime, "LIMIT", 0, limit)
+local results  = redis.call("ZRANGE", pendingNotificationQueue, 0, maxTime, "BYSCORE", "LIMIT", 0, limit)
 local collated = {}
 
 if results and next(results) then

service/src/main/resources/lua/get_items.lua

@@ -1,7 +1,10 @@
-local queueKey     = KEYS[1]
-local queueLockKey = KEYS[2]
-local limit        = ARGV[1]
-local afterMessageId = ARGV[2]
+-- gets messages from a device's queue, up to a given limit
+-- returns a list of all envelopes and their queue-local IDs
+
+local queueKey       = KEYS[1] -- sorted set of all Envelopes for a device, scored by queue-local ID
+local queueLockKey   = KEYS[2] -- a key whose presence indicates that the queue is being persistent and must not be read
+local limit          = ARGV[1] -- [number] the maximum number of messages to return
+local afterMessageId = ARGV[2] -- [number] a queue-local ID to exclusively start after, to support pagination. Use -1 to start at the beginning
 
 local locked = redis.call("GET", queueLockKey)
 
@@ -9,17 +12,8 @@ if locked then
     return {}
 end
 
-if afterMessageId == "null" then
-    -- An index range is inclusive
-    local min = 0
-    local max = limit - 1
-
-    if max < 0 then
-        return {}
-    end
-
-    return redis.call("ZRANGE", queueKey, min, max, "WITHSCORES")
-else
-    -- note: this is deprecated in Redis 6.2, and should be migrated to zrange after the cluster is updated
-    return redis.call("ZRANGEBYSCORE", queueKey, "("..afterMessageId, "+inf", "WITHSCORES", "LIMIT", 0, limit)
+if afterMessageId == "null" or afterMessageId == nil then
+    return redis.error_reply("ERR afterMessageId is required")
 end
+
+return redis.call("ZRANGE", queueKey, "("..afterMessageId, "+inf", "BYSCORE", "LIMIT", 0, limit, "WITHSCORES")

service/src/main/resources/lua/get_queues_to_persist.lua

@@ -1,8 +1,10 @@
-local queueTotalIndexKey = KEYS[1]
-local maxTime            = ARGV[1]
-local limit              = ARGV[2]
+-- returns a list of queues that meet persistence criteria
 
-local results = redis.call("ZRANGEBYSCORE", queueTotalIndexKey, 0, maxTime, "LIMIT", 0, limit)
+local queueTotalIndexKey = KEYS[1] -- sorted set of all queues in the shard, by timestamp of oldest message
+local maxTime            = ARGV[1] -- [number] the most recent queue timestamp that may be fetched
+local limit              = ARGV[2] -- [number] the maximum number of queues to fetch
+
+local results = redis.call("ZRANGE", queueTotalIndexKey, 0, maxTime, "BYSCORE", "LIMIT", 0, limit)
 
 if results and next(results) then
     redis.call("ZREM", queueTotalIndexKey, unpack(results))

service/src/main/resources/lua/insert_item.lua

@@ -1,9 +1,12 @@
-local queueKey           = KEYS[1]
-local queueMetadataKey   = KEYS[2]
-local queueTotalIndexKey = KEYS[3]
-local message            = ARGV[1]
-local currentTime        = ARGV[2]
-local guid               = ARGV[3]
+-- inserts a message into a device's queue, and updates relevant associated data
+-- returns a number, the queue-local message ID (useful for testing)
+
+local queueKey           = KEYS[1] -- sorted set of Envelopes for a device, by queue-local ID
+local queueMetadataKey   = KEYS[2] -- hash of message GUID to queue-local IDs
+local queueTotalIndexKey = KEYS[3] -- sorted set of all queues in the shard, by timestamp of oldest message
+local message            = ARGV[1] -- [bytes] the Envelope to insert
+local currentTime        = ARGV[2] -- [number] the message timestamp, to sort the queue in the queueTotalIndex
+local guid               = ARGV[3] -- [string] the message GUID
 
 if redis.call("HEXISTS", queueMetadataKey, guid) == 1 then
     return tonumber(redis.call("HGET", queueMetadataKey, guid))
@@ -14,9 +17,8 @@ local messageId = redis.call("HINCRBY", queueMetadataKey, "counter", 1)
 redis.call("ZADD", queueKey, "NX", messageId, message)
 
 redis.call("HSET", queueMetadataKey, guid, messageId)
-
-redis.call("EXPIRE", queueKey, 7776000)         -- 90 days
-redis.call("EXPIRE", queueMetadataKey, 7776000) -- 90 days
+redis.call("EXPIRE", queueKey, 2678400) -- 31 days
+redis.call("EXPIRE", queueMetadataKey, 2678400) -- 31 days
 
 redis.call("ZADD", queueTotalIndexKey, "NX", currentTime, queueKey)
 return messageId

service/src/main/resources/lua/insert_shared_multirecipient_message_data.lua

@@ -0,0 +1,13 @@
+-- inserts shared multi-recipient message data
+
+local sharedMrmKey = KEYS[1] -- [string] the key containing the shared MRM data
+local mrmData      = ARGV[1] -- [bytes] the serialized multi-recipient message data
+-- the remainder of ARGV is list of recipient keys and view data
+
+redis.call("HSET", sharedMrmKey, "data", mrmData);
+redis.call("EXPIRE", sharedMrmKey, 604800) -- 7 days
+
+-- unpack() fails with "too many results" at very large table sizes, so we loop
+for i = 2, #ARGV, 2 do
+    redis.call("HSET", sharedMrmKey, ARGV[i], ARGV[i + 1])
+end

service/src/main/resources/lua/remove_item_by_guid.lua

@@ -1,20 +1,26 @@
-local queueKey           = KEYS[1]
-local queueMetadataKey   = KEYS[2]
-local queueTotalIndexKey = KEYS[3]
+-- removes a list of messages by ID from the cluster, returning the deleted messages
+-- returns a list of removed envelopes
+--   Note: content may be absent for MRM messages, and for these messages, the caller must update the sharedMrmKey
+--         to remove the recipient's reference
+
+local queueKey           = KEYS[1] -- sorted set of Envelopes for a device, by queue-local ID
+local queueMetadataKey   = KEYS[2] -- hash of message GUID to queue-local IDs
+local queueTotalIndexKey = KEYS[3] -- sorted set of all queues in the shard, by timestamp of oldest message
+local messageGuids       = ARGV    -- [list[string]] message GUIDs
 
 local removedMessages = {}
 
-for _, guid in ipairs(ARGV) do
+for _, guid in ipairs(messageGuids) do
     local messageId = redis.call("HGET", queueMetadataKey, guid)
 
     if messageId then
-        local envelope = redis.call("ZRANGEBYSCORE", queueKey, messageId, messageId, "LIMIT", 0, 1)
+        local envelope = redis.call("ZRANGE", queueKey, messageId, messageId, "BYSCORE", "LIMIT", 0, 1)
 
         redis.call("ZREMRANGEBYSCORE", queueKey, messageId, messageId)
         redis.call("HDEL", queueMetadataKey, guid)
 
         if envelope and next(envelope) then
-            removedMessages[#removedMessages + 1] = envelope[1]
+            table.insert(removedMessages, envelope[1])
         end
     end
 end

service/src/main/resources/lua/remove_queue.lua

@@ -1,7 +1,29 @@
-local queueKey           = KEYS[1]
-local queueMetadataKey   = KEYS[2]
-local queueTotalIndexKey = KEYS[3]
+-- incrementally removes a given device's queue and associated data
+-- returns: a page of messages and scores.
+--    The messages must be checked for mrmKeys to update. After updating MRM keys, this script must be called again
+--    with processedMessageGuids. If the returned table is empty, then
+--    the queue has been fully deleted.
 
-redis.call("DEL", queueKey)
-redis.call("DEL", queueMetadataKey)
-redis.call("ZREM", queueTotalIndexKey, queueKey)
+local queueKey              = KEYS[1] -- sorted set of Envelopes for a device, by queue-local ID
+local queueMetadataKey      = KEYS[2] -- hash of message GUID to queue-local IDs
+local queueTotalIndexKey    = KEYS[3] -- sorted set of all queues in the shard, by timestamp of oldest message
+local limit                 = ARGV[1] -- the maximum number of messages to return
+local processedMessageGuids = { unpack(ARGV, 2) }
+
+for _, guid in ipairs(processedMessageGuids) do
+    local messageId = redis.call("HGET", queueMetadataKey, guid)
+    if messageId then
+        redis.call("ZREMRANGEBYSCORE", queueKey, messageId, messageId)
+        redis.call("HDEL", queueMetadataKey, guid)
+    end
+end
+
+local messages = redis.call("ZRANGE", queueKey, 0, limit-1)
+
+if #messages == 0 then
+    redis.call("DEL", queueKey)
+    redis.call("DEL", queueMetadataKey)
+    redis.call("ZREM", queueTotalIndexKey, queueKey)
+end
+
+return messages

service/src/main/resources/lua/remove_recipient_view_from_mrm_data.lua

@@ -0,0 +1,17 @@
+-- Removes the given recipient view from the shared MRM data. If the only field remaining after the removal is the
+-- `data` field, then the key will be deleted
+
+local sharedMrmKeys         = KEYS    -- KEYS: list of all keys in a single slot to update
+local recipientViewToRemove = ARGV[1] -- the recipient view to remove from the hash
+
+local keysDeleted = 0
+
+for _, sharedMrmKey in ipairs(sharedMrmKeys) do
+    redis.call("HDEL", sharedMrmKey, recipientViewToRemove)
+    if redis.call("HLEN", sharedMrmKey) == 1 then
+        redis.call("DEL", sharedMrmKey)
+        keysDeleted = keysDeleted + 1
+    end
+end
+
+return keysDeleted