Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: replace statsd-client with hot-shots (#4806)
* fix: replace statsd-client with hot-shots * chore: comment * chore: fix test * chore: refactor code * chore: jsdoc * chore: better framework check * chore: variable name
- Loading branch information
Showing
8 changed files
with
226 additions
and
238 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,40 +1,69 @@ | ||
| import { promisify } from 'util' | ||
|
|
||
| import slugify from '@sindresorhus/slugify' | ||
| import StatsdClient from 'statsd-client' | ||
| import { StatsD } from 'hot-shots' | ||
|
|
||
| // TODO: replace with `timers/promises` after dropping Node < 15.0.0 | ||
| const pSetTimeout = promisify(setTimeout) | ||
| export interface InputStatsDOptions { | ||
| host?: string | ||
| port?: number | ||
| } | ||
|
|
||
| // See https://github.com/msiebuhr/node-statsd-client/blob/45a93ee4c94ca72f244a40b06cb542d4bd7c3766/lib/EphemeralSocket.js#L81 | ||
| const CLOSE_TIMEOUT = 11 | ||
| export type StatsDOptions = Required<InputStatsDOptions> | ||
|
|
||
| // The socket creation is delayed until the first packet is sent. In our | ||
| // case, this happens just before `client.close()` is called, which is too | ||
| // late and make it not send anything. We need to manually create it using | ||
| // the internal API. | ||
| export const startClient = async function (host: string, port: number): Promise<StatsdClient> { | ||
| const client = new StatsdClient({ host, port, socketTimeout: 0 }) | ||
| export const validateStatsDOptions = function (statsdOpts: InputStatsDOptions): statsdOpts is StatsDOptions { | ||
| return !!(statsdOpts && statsdOpts.host && statsdOpts.port) | ||
| } | ||
|
|
||
| // @ts-expect-error using internals :D | ||
| await promisify(client._socket._createSocket.bind(client._socket))() | ||
| /** | ||
| * Start a new StatsD Client and a new UDP socket | ||
| */ | ||
| export const startClient = async function (statsdOpts: StatsDOptions): Promise<StatsD> { | ||
| const { host, port } = statsdOpts | ||
|
|
||
| return client | ||
| return new StatsD({ | ||
| host, | ||
| port, | ||
| // This caches the dns resolution for subsequent sends of metrics for this instance | ||
| // Because we only try to send the metrics on close, this comes only into effect if `bufferFlushInterval` time is exceeded | ||
| cacheDns: true, | ||
| // set the maxBufferSize to infinite and the bufferFlushInterval very high, so that we only | ||
| // send the metrics on close or if more than 10 seconds past by | ||
| maxBufferSize: Infinity, | ||
| bufferFlushInterval: 10_000, | ||
| }) | ||
| } | ||
|
|
||
| // UDP packets are buffered and flushed at regular intervals by statsd-client. | ||
| // Closing force flushing all of them. | ||
| export const closeClient = async function (client: StatsdClient): Promise<void> { | ||
| client.close() | ||
|
|
||
| // statsd-client does not provide a way of knowing when the socket is done | ||
| // closing, so we need to use the following hack. | ||
| await pSetTimeout(CLOSE_TIMEOUT) | ||
| await pSetTimeout(CLOSE_TIMEOUT) | ||
| /** | ||
| * Close the StatsD Client | ||
| * | ||
| * UDP packets are buffered and flushed every 10 seconds and | ||
| * closing the client forces all buffered metrics to be flushed. | ||
| */ | ||
| export const closeClient = async function (client: StatsD): Promise<void> { | ||
| await promisify(client.close.bind(client))() | ||
| } | ||
|
|
||
| // Make sure the timer name does not include special characters. | ||
| // For example, the `packageName` of local plugins includes dots. | ||
| /** | ||
| * Make sure the timer name does not include special characters. | ||
| * For example, the `packageName` of local plugins includes dots. | ||
| */ | ||
| export const normalizeTagName = function (name: string): string { | ||
| return slugify(name, { separator: '_' }) | ||
| } | ||
|
|
||
| /** | ||
| * Formats and flattens the tags as array | ||
| * We do this because we might send the same tag with different values `{ tag: ['a', 'b'] }` | ||
| * which cannot be represented in an flattened object and `hot-shots` also supports tags as array in the format `['tag:a', 'tag:b']` | ||
| */ | ||
| export const formatTags = function (tags: Record<string, string | string[]>): string[] { | ||
| return Object.entries(tags) | ||
| .map(([key, value]) => { | ||
| if (Array.isArray(value)) { | ||
| return value.map((subValue) => `${key}:${subValue}`) | ||
| } else { | ||
| return `${key}:${value}` | ||
| } | ||
| }) | ||
| .flat() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,29 +1,58 @@ | ||
| import { closeClient, startClient } from '../report/statsd.js' | ||
| import type { Tags, StatsD } from 'hot-shots' | ||
|
|
||
| import { | ||
| closeClient, | ||
| formatTags, | ||
| InputStatsDOptions, | ||
| startClient, | ||
| StatsDOptions, | ||
| validateStatsDOptions, | ||
| } from '../report/statsd.js' | ||
|
|
||
| import { addAggregatedTimers } from './aggregate.js' | ||
| import { roundTimerToMillisecs } from './measure.js' | ||
|
|
||
| // Record the duration of a build phase, for monitoring. | ||
| // Sends to statsd daemon. | ||
| export const reportTimers = async function ({ timers, statsdOpts: { host, port }, framework }) { | ||
| if (host === undefined) { | ||
| interface Timer { | ||
| metricName: string | ||
| stageTag: string | ||
| parentTag: string | ||
| durationNs: number | ||
| tags: Record<string, string | string[]> | ||
| } | ||
|
|
||
| /** | ||
| * Record the duration of a build phase, for monitoring. | ||
| * Sends to statsd daemon. | ||
| */ | ||
| export const reportTimers = async function ( | ||
| timers: Timer[], | ||
| statsdOpts: InputStatsDOptions, | ||
| framework?: string, | ||
| ): Promise<void> { | ||
| if (!validateStatsDOptions(statsdOpts)) { | ||
| return | ||
| } | ||
|
|
||
| const timersA = addAggregatedTimers(timers) | ||
| await sendTimers({ timers: timersA, host, port, framework }) | ||
| await sendTimers(addAggregatedTimers(timers), statsdOpts, framework) | ||
| } | ||
|
|
||
| const sendTimers = async function ({ timers, host, port, framework }) { | ||
| const client = await startClient(host, port) | ||
| const sendTimers = async function (timers: Timer[], statsdOpts: StatsDOptions, framework?: string): Promise<void> { | ||
| const client = await startClient(statsdOpts) | ||
| timers.forEach((timer) => { | ||
| sendTimer({ timer, client, framework }) | ||
| sendTimer(timer, client, framework) | ||
| }) | ||
| await closeClient(client) | ||
| } | ||
|
|
||
| const sendTimer = function ({ timer: { metricName, stageTag, parentTag, durationNs, tags }, client, framework }) { | ||
| const sendTimer = function (timer: Timer, client: StatsD, framework?: string): void { | ||
| const { metricName, stageTag, parentTag, durationNs, tags } = timer | ||
| const durationMs = roundTimerToMillisecs(durationNs) | ||
| const frameworkTag = framework === undefined ? {} : { framework } | ||
| client.distribution(metricName, durationMs, { stage: stageTag, parent: parentTag, ...tags, ...frameworkTag }) | ||
| const statsDTags: Tags = { stage: stageTag, parent: parentTag, ...tags } | ||
|
|
||
| // Do not add a framework tag if empty string or null/undefined | ||
| if (framework) { | ||
| statsDTags.framework = framework | ||
| } | ||
|
|
||
| client.distribution(metricName, durationMs, formatTags(statsDTags)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters