From 96165fb31cce650e08f530b2ce3e1bc8e85f7bf7 Mon Sep 17 00:00:00 2001
From: Stefan Bueringer <buringerst@vmware.com>
Date: Wed, 1 Mar 2023 03:14:38 +0100
Subject: [PATCH] Add rest client request duration and size metrics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stefan Büringer buringerst@vmware.com
---
 pkg/metrics/client_go_adapter.go | 59 ++++++++++++++++++++++++++++----
 1 file changed, 53 insertions(+), 6 deletions(-)

diff --git a/pkg/metrics/client_go_adapter.go b/pkg/metrics/client_go_adapter.go
index dc805a9d04..a8b43ea0a4 100644
--- a/pkg/metrics/client_go_adapter.go
+++ b/pkg/metrics/client_go_adapter.go
@@ -62,11 +62,44 @@ var (
 		Buckets:   prometheus.ExponentialBuckets(0.001, 2, 10),
 	}, []string{"verb", "url"})
 
-	requestResult = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Subsystem: RestClientSubsystem,
-		Name:      ResultKey,
-		Help:      "Number of HTTP requests, partitioned by status code, method, and host.",
-	}, []string{"code", "method", "host"})
+	// requestLatency is a Prometheus Histogram metric type partitioned by
+	// "verb", and "host" labels. It is used for the rest client latency metrics.
+	requestLatency = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "rest_client_request_duration_seconds",
+			Help:    "Request latency in seconds. Broken down by verb, and host.",
+			Buckets: []float64{0.005, 0.025, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 15.0, 30.0, 60.0},
+		},
+		[]string{"verb", "host"},
+	)
+
+	requestSize = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "rest_client_request_size_bytes",
+			Help: "Request size in bytes. Broken down by verb and host.",
+			// 64 bytes to 16MB
+			Buckets: []float64{64, 256, 512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216},
+		},
+		[]string{"verb", "host"},
+	)
+
+	responseSize = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "rest_client_response_size_bytes",
+			Help: "Response size in bytes. Broken down by verb and host.",
+			// 64 bytes to 16MB
+			Buckets: []float64{64, 256, 512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216},
+		},
+		[]string{"verb", "host"},
+	)
+
+	requestResult = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "rest_client_requests_total",
+			Help: "Number of HTTP requests, partitioned by status code, method, and host.",
+		},
+		[]string{"code", "method", "host"},
+	)
 )
 
 func init() {
@@ -76,11 +109,17 @@ func init() {
 // registerClientMetrics sets up the client latency metrics from client-go.
 func registerClientMetrics() {
 	// register the metrics with our registry
+	Registry.MustRegister(requestLatency)
+	Registry.MustRegister(requestSize)
+	Registry.MustRegister(responseSize)
 	Registry.MustRegister(requestResult)
 
 	// register the metrics with client-go
 	clientmetrics.Register(clientmetrics.RegisterOpts{
-		RequestResult: &resultAdapter{metric: requestResult},
+		RequestLatency: &LatencyAdapter{metric: requestLatency},
+		RequestSize:    &sizeAdapter{metric: requestSize},
+		ResponseSize:   &sizeAdapter{metric: responseSize},
+		RequestResult:  &resultAdapter{metric: requestResult},
 	})
 }
 
@@ -102,6 +141,14 @@ func (l *LatencyAdapter) Observe(_ context.Context, verb string, u url.URL, late
 	l.metric.WithLabelValues(verb, u.String()).Observe(latency.Seconds())
 }
 
+type sizeAdapter struct {
+	metric *prometheus.HistogramVec
+}
+
+func (s *sizeAdapter) Observe(ctx context.Context, verb string, host string, size float64) {
+	s.metric.WithLabelValues(verb, host).Observe(size)
+}
+
 type resultAdapter struct {
 	metric *prometheus.CounterVec
 }