Skip to content

Commit

Permalink
feat(Client): add global sweepers (#6825)
Browse files Browse the repository at this point in the history 
Co-authored-by: Antonio Román <kyradiscord@gmail.com>
Co-authored-by: Vlad Frangu <kingdgrizzle@gmail.com>
Co-authored-by: Rodry <38259440+ImRodry@users.noreply.github.com>
Co-authored-by: Almeida <almeidx@pm.me>
  • Loading branch information
@ckohen @kyranet
@vladfrangu @ImRodry @almeidx
5 people committed Dec 15, 2021
1 parent bc6a6c5 commit d1ef2f5
Show file tree
Hide file tree
Showing 7 changed files with 660 additions and 63 deletions.
33 changes: 14 additions & 19 deletions src/client/Client.js
View file
Expand Up @@ -25,6 +25,7 @@ const DataResolver = require('../util/DataResolver');
const Intents = require('../util/Intents');
const Options = require('../util/Options');
const Permissions = require('../util/Permissions');
const Sweepers = require('../util/Sweepers');

/**
* The main hub for interacting with the Discord API, and the starting point for any bot.
Expand Down Expand Up @@ -135,6 +136,12 @@ class Client extends BaseClient {
*/
this.channels = new ChannelManager(this);

/**
* The sweeping functions and their intervals used to periodically sweep caches
* @type {Sweepers}
*/
this.sweepers = new Sweepers(this, this.options.sweepers);

/**
* The presence of the Client
* @private
Expand Down Expand Up @@ -176,7 +183,7 @@ class Client extends BaseClient {

if (this.options.messageSweepInterval > 0) {
process.emitWarning(
'The message sweeping client options are deprecated, use the makeCache option with LimitedCollection instead.',
'The message sweeping client options are deprecated, use the global sweepers instead.',
'DeprecationWarning',
);
this.sweepMessageInterval = setInterval(
Expand Down Expand Up @@ -271,6 +278,7 @@ class Client extends BaseClient {

if (this.sweepMessageInterval) clearInterval(this.sweepMessageInterval);

this.sweepers.destroy();
this.ws.destroy();
this.token = null;
}
Expand Down Expand Up @@ -401,24 +409,8 @@ class Client extends BaseClient {
return -1;
}

const lifetimeMs = lifetime * 1_000;
const now = Date.now();
let channels = 0;
let messages = 0;

for (const channel of this.channels.cache.values()) {
if (!channel.messages) continue;
channels++;

messages += channel.messages.cache.sweep(
message => now - (message.editedTimestamp ?? message.createdTimestamp) > lifetimeMs,
);
}

this.emit(
Events.DEBUG,
`Swept ${messages} messages older than ${lifetime} seconds in ${channels} text-based channels`,
);
const messages = this.sweepers.sweepMessages(Sweepers.outdatedMessageSweepFilter(lifetime)());
this.emit(Events.DEBUG, `Swept ${messages} messages older than ${lifetime} seconds`);
return messages;
}

Expand Down Expand Up @@ -561,6 +553,9 @@ class Client extends BaseClient {
if (typeof options.messageSweepInterval !== 'number' || isNaN(options.messageSweepInterval)) {
throw new TypeError('CLIENT_INVALID_OPTION', 'messageSweepInterval', 'a number');
}
if (typeof options.sweepers !== 'object' || options.sweepers === null) {
throw new TypeError('CLIENT_INVALID_OPTION', 'sweepers', 'an object');
}
if (typeof options.invalidRequestWarningInterval !== 'number' || isNaN(options.invalidRequestWarningInterval)) {
throw new TypeError('CLIENT_INVALID_OPTION', 'invalidRequestWarningInterval', 'a number');
}
Expand Down
36 changes: 36 additions & 0 deletions src/util/Constants.js
View file
Expand Up @@ -188,6 +188,7 @@ exports.Events = {
ERROR: 'error',
WARN: 'warn',
DEBUG: 'debug',
CACHE_SWEEP: 'cacheSweep',
SHARD_DISCONNECT: 'shardDisconnect',
SHARD_ERROR: 'shardError',
SHARD_RECONNECTING: 'shardReconnecting',
Expand Down Expand Up @@ -431,6 +432,41 @@ exports.MessageTypes = [
'CONTEXT_MENU_COMMAND',
];

/**
* The name of an item to be swept in Sweepers
* * `applicationCommands` - both global and guild commands
* * `bans`
* * `emojis`
* * `invites` - accepts the `lifetime` property, using it will sweep based on expires timestamp
* * `guildMembers`
* * `messages` - accepts the `lifetime` property, using it will sweep based on edited or created timestamp
* * `presences`
* * `reactions`
* * `stageInstances`
* * `stickers`
* * `threadMembers`
* * `threads` - accepts the `lifetime` property, using it will sweep archived threads based on archived timestamp
* * `users`
* * `voiceStates`
* @typedef {string} SweeperKey
*/
exports.SweeperKeys = [
'applicationCommands',
'bans',
'emojis',
'invites',
'guildMembers',
'messages',
'presences',
'reactions',
'stageInstances',
'stickers',
'threadMembers',
'threads',
'users',
'voiceStates',
];

/**
* The types of messages that are `System`. The available types are `MessageTypes` excluding:
* * DEFAULT
Expand Down
46 changes: 11 additions & 35 deletions src/util/LimitedCollection.js
View file
Expand Up @@ -2,6 +2,7 @@

const { Collection } = require('@discordjs/collection');
const { _cleanupSymbol } = require('./Constants.js');
const Sweepers = require('./Sweepers.js');
const { TypeError } = require('../errors/DJSError.js');

/**
Expand All @@ -18,8 +19,12 @@ const { TypeError } = require('../errors/DJSError.js');
* @property {?number} [maxSize=Infinity] The maximum size of the Collection
* @property {?Function} [keepOverLimit=null] A function, which is passed the value and key of an entry, ran to decide
* to keep an entry past the maximum size
* @property {?SweepFilter} [sweepFilter=null] A function ran every `sweepInterval` to determine how to sweep
* @property {?number} [sweepInterval=0] How frequently, in seconds, to sweep the collection.
* @property {?SweepFilter} [sweepFilter=null] DEPRECATED: There is no direct alternative to this,
* however most of its purpose is fulfilled by {@link Client#sweepers}
* A function ran every `sweepInterval` to determine how to sweep
* @property {?number} [sweepInterval=0] DEPRECATED: There is no direct alternative to this,
* however most of its purpose is fulfilled by {@link Client#sweepers}
* How frequently, in seconds, to sweep the collection.
*/

/**
Expand Down Expand Up @@ -64,12 +69,14 @@ class LimitedCollection extends Collection {

/**
* A function called every sweep interval that returns a function passed to `sweep`.
* @deprecated in favor of {@link Client#sweepers}
* @type {?SweepFilter}
*/
this.sweepFilter = sweepFilter;

/**
* The id of the interval being used to sweep.
* @deprecated in favor of {@link Client#sweepers}
* @type {?Timeout}
*/
this.interval =
Expand Down Expand Up @@ -97,49 +104,18 @@ class LimitedCollection extends Collection {
return super.set(key, value);
}

/**
* Options for generating a filter function based on lifetime
* @typedef {Object} LifetimeFilterOptions
* @property {number} [lifetime=14400] How long, in seconds, an entry should stay in the collection
* before it is considered sweepable.
* @property {Function} [getComparisonTimestamp=e => e?.createdTimestamp] A function that takes an entry, key,
* and the collection and returns a timestamp to compare against in order to determine the lifetime of the entry.
* @property {Function} [excludeFromSweep=() => false] A function that takes an entry, key, and the collection
* and returns a boolean, `true` when the entry should not be checked for sweepability.
*/

/**
* Create a sweepFilter function that uses a lifetime to determine sweepability.
* @param {LifetimeFilterOptions} [options={}] The options used to generate the filter function
* @deprecated Use {@link Sweepers.filterByLifetime} instead
* @returns {SweepFilter}
*/
static filterByLifetime({
lifetime = 14400,
getComparisonTimestamp = e => e?.createdTimestamp,
excludeFromSweep = () => false,
} = {}) {
if (typeof lifetime !== 'number') {
throw new TypeError('INVALID_TYPE', 'lifetime', 'number');
}
if (typeof getComparisonTimestamp !== 'function') {
throw new TypeError('INVALID_TYPE', 'getComparisonTimestamp', 'function');
}
if (typeof excludeFromSweep !== 'function') {
throw new TypeError('INVALID_TYPE', 'excludeFromSweep', 'function');
}
return () => {
if (lifetime <= 0) return null;
const lifetimeMs = lifetime * 1_000;
const now = Date.now();
return (entry, key, coll) => {
if (excludeFromSweep(entry, key, coll)) {
return false;
}
const comparisonTimestamp = getComparisonTimestamp(entry, key, coll);
if (!comparisonTimestamp || typeof comparisonTimestamp !== 'number') return false;
return now - comparisonTimestamp > lifetimeMs;
};
};
return Sweepers.filterByLifetime({ lifetime, getComparisonTimestamp, excludeFromSweep });
}

[_cleanupSymbol]() {
Expand Down
37 changes: 35 additions & 2 deletions src/util/Options.js
View file
Expand Up @@ -37,9 +37,9 @@
* You can use your own function, or the {@link Options} class to customize the Collection used for the cache.
* <warn>Overriding the cache used in `GuildManager`, `ChannelManager`, `GuildChannelManager`, `RoleManager`,
* and `PermissionOverwriteManager` is unsupported and **will** break functionality</warn>
* @property {number} [messageCacheLifetime=0] DEPRECATED: Use `makeCache` with a `LimitedCollection` instead.
* @property {number} [messageCacheLifetime=0] DEPRECATED: Pass `lifetime` to `sweepers.messages` instead.
* How long a message should stay in the cache until it is considered sweepable (in seconds, 0 for forever)
* @property {number} [messageSweepInterval=0] DEPRECATED: Use `makeCache` with a `LimitedCollection` instead.
* @property {number} [messageSweepInterval=0] DEPRECATED: Pass `interval` to `sweepers.messages` instead.
* How frequently to remove messages from the cache that are older than the message cache lifetime
* (in seconds, 0 for never)
* @property {MessageMentionOptions} [allowedMentions] Default value for {@link MessageOptions#allowedMentions}
Expand Down Expand Up @@ -70,10 +70,27 @@
* [User Agent](https://discord.com/developers/docs/reference#user-agent) header
* @property {PresenceData} [presence={}] Presence data to use upon login
* @property {IntentsResolvable} intents Intents to enable for this connection
* @property {SweeperOptions} [sweepers={}] Options for cache sweeping
* @property {WebsocketOptions} [ws] Options for the WebSocket
* @property {HTTPOptions} [http] HTTP options
*/

/**
* Options for {@link Sweepers} defining the behavior of cache sweeping
* @typedef {Object<SweeperKey, SweepOptions>} SweeperOptions
*/

/**
* Options for sweeping a single type of item from cache
* @typedef {Object} SweepOptions
* @property {number} interval The interval (in seconds) at which to perform sweeping of the item
* @property {number} [lifetime] How long an item should stay in cache until it is considered sweepable.
* <warn>This property is only valid for the `invites`, `messages`, and `threads` keys. The `filter` property
* is mutually exclusive to this property and takes priority</warn>
* @property {GlobalSweepFilter} filter The function used to determine the function passed to the sweep method
* <info>This property is optional when the key is `invites`, `messages`, or `threads` and `lifetime` is set</info>
*/

/**
* WebSocket options (these are left as snake_case to match the API)
* @typedef {Object} WebsocketOptions
Expand Down Expand Up @@ -125,6 +142,7 @@ class Options extends null {
failIfNotExists: true,
userAgentSuffix: [],
presence: {},
sweepers: {},
ws: {
large_threshold: 50,
compress: false,
Expand Down Expand Up @@ -251,4 +269,19 @@ class Options extends null {
}
}

/**
* The default settings passed to {@link Options.sweepers} (for v14).
* The sweepers that this changes are:
* * `threads` - Sweep archived threads every hour, removing those archived more than 4 hours ago
* <info>If you want to keep default behavior and add on top of it you can use this object and add on to it, e.g.
* `sweepers: { ...Options.defaultSweeperSettings, messages: { interval: 300, lifetime: 600 } })`</info>
* @type {SweeperOptions}
*/
Options.defaultSweeperSettings = {
threads: {
interval: 3600,
lifetime: 14400,
},
};

module.exports = Options;

0 comments on commit d1ef2f5

Please sign in to comment.