From ef5f4717258940042fa11f980622034cde765860 Mon Sep 17 00:00:00 2001
From: Eric Mc Sween <eric.mcsween@gmail.com>
Date: Sun, 20 Mar 2022 21:22:49 -0400
Subject: [PATCH] perf: speed up clean operation (#2326)

The clean operation on sets backed by lists (wait, active, paused)
quickly gets very slow when the list is large. This is because each job
deletion scans the whole list in a LREM call, resulting in O(N * M)
complexity where N is the number of jobs in the list and M is the number
of jobs to delete.

With this change, the deletion is done in two passes. The first pass
sets each item that should be deleted to a special value. The second
pass deletes all items with that special value in a single LREM call.
This results in O(N) complexity.
---
 lib/commands/cleanJobsInSet-3.lua | 12 ++++++++++--
 test/test_queue.js                | 10 ++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/lib/commands/cleanJobsInSet-3.lua b/lib/commands/cleanJobsInSet-3.lua
index 830105946..a8298d1a1 100644
--- a/lib/commands/cleanJobsInSet-3.lua
+++ b/lib/commands/cleanJobsInSet-3.lua
@@ -53,7 +53,8 @@ local jobTS
 -- - Once, if limit is -1 or 0
 -- - As many times as needed if limit is positive
 while ((limit <= 0 or deletedCount < limit) and next(jobIds, nil) ~= nil) do
-  for _, jobId in ipairs(jobIds) do
+  local jobIdsLen = #jobIds
+  for i, jobId in ipairs(jobIds) do
     if limit > 0 and deletedCount >= limit then
       break
     end
@@ -73,7 +74,10 @@ while ((limit <= 0 or deletedCount < limit) and next(jobIds, nil) ~= nil) do
       end
       if (not jobTS or jobTS < maxTimestamp) then
         if isList then
-          rcall("LREM", setKey, 0, jobId)
+          -- Job ids can't be the empty string. Use the empty string as a
+          -- deletion marker. The actual deletion will occur at the end of the
+          -- script.
+          rcall("LSET", setKey, rangeEnd - jobIdsLen + i, "")
         else
           rcall("ZREM", setKey, jobId)
         end
@@ -111,4 +115,8 @@ while ((limit <= 0 or deletedCount < limit) and next(jobIds, nil) ~= nil) do
   end
 end
 
+if isList then
+  rcall("LREM", setKey, 0, "")
+end
+
 return deleted
diff --git a/test/test_queue.js b/test/test_queue.js
index 4b50194c4..0706515f8 100644
--- a/test/test_queue.js
+++ b/test/test_queue.js
@@ -3008,6 +3008,16 @@ describe('Queue', () => {
         });
     });
 
+    it('should succeed when the limit is higher than the actual number of jobs', async () => {
+      await queue.add({ some: 'data' });
+      await queue.add({ some: 'data' });
+      await delay(100);
+      const deletedJobs = await queue.clean(0, 'wait', 100);
+      expect(deletedJobs).to.have.length(2);
+      const remainingJobsCount = await queue.count();
+      expect(remainingJobsCount).to.be.eql(0);
+    });
+
     it("should clean the number of jobs requested even if first jobs timestamp doesn't match", async () => {
       // This job shouldn't get deleted due to the 5000 grace
       await queue.add({ some: 'data' });