ISPN-14164 Fix metrics produced by the Infinispan server for Prometheus scraping

client/rest-client/src/main/java/org/infinispan/client/rest/impl/okhttp/RestMetricsClientOkHttp.java

@@ -1,7 +1,7 @@
 package org.infinispan.client.rest.impl.okhttp;
 
-import static org.infinispan.commons.dataconversion.MediaType.APPLICATION_JSON_TYPE;
 import static org.infinispan.commons.dataconversion.MediaType.APPLICATION_OPENMETRICS_TYPE;
+import static org.infinispan.commons.dataconversion.MediaType.TEXT_PLAIN_TYPE;
 
 import java.util.concurrent.CompletionStage;
 
@@ -38,7 +38,7 @@ public CompletionStage<RestResponse> metricsMetadata() {
 
    private CompletionStage<RestResponse> metricsGet(boolean openMetricsFormat) {
       Request.Builder builder = new Request.Builder()
-            .addHeader("ACCEPT", openMetricsFormat ? APPLICATION_OPENMETRICS_TYPE : APPLICATION_JSON_TYPE);
+            .addHeader("ACCEPT", openMetricsFormat ? APPLICATION_OPENMETRICS_TYPE : TEXT_PLAIN_TYPE);
       return client.execute(builder, baseMetricsURL);
    }
 

core/src/main/java/org/infinispan/metrics/impl/BaseAdditionalMetrics.java

@@ -18,6 +18,8 @@
 @NonNullFields
 class BaseAdditionalMetrics implements MeterBinder {
 
+   static final String PREFIX = "base.";
+
    @Override
    public void bindTo(MeterRegistry registry) {
       bindClassLoaderMetrics(registry);
@@ -31,27 +33,27 @@ public void bindTo(MeterRegistry registry) {
    private void bindClassLoaderMetrics(MeterRegistry registry) {
       ClassLoadingMXBean classLoadingBean = ManagementFactory.getClassLoadingMXBean();
 
-      Gauge.builder("classloader.loadedClasses.count", classLoadingBean, ClassLoadingMXBean::getLoadedClassCount)
+      Gauge.builder(PREFIX + "classloader.loadedClasses.count", classLoadingBean, ClassLoadingMXBean::getLoadedClassCount)
             .description("Displays the number of classes that are currently loaded in the Java virtual machine.")
             .register(registry);
 
-      FunctionCounter.builder("classloader.loadedClasses.total", classLoadingBean, ClassLoadingMXBean::getTotalLoadedClassCount)
+      FunctionCounter.builder(PREFIX + "classloader.loadedClasses.total", classLoadingBean, ClassLoadingMXBean::getTotalLoadedClassCount)
             .description("Displays the total number of classes that have been loaded since the Java virtual machine has started execution.")
             .register(registry);
 
-      FunctionCounter.builder("classloader.unloadedClasses.total", classLoadingBean, ClassLoadingMXBean::getUnloadedClassCount)
+      FunctionCounter.builder(PREFIX + "classloader.unloadedClasses.total", classLoadingBean, ClassLoadingMXBean::getUnloadedClassCount)
             .description("Displays the total number of classes unloaded since the Java virtual machine has started execution.")
             .register(registry);
    }
 
    private void bindGarbageCollectionMetrics(MeterRegistry registry) {
       for (GarbageCollectorMXBean garbageCollectorBean : ManagementFactory.getGarbageCollectorMXBeans()) {
-         FunctionCounter.builder("gc.total", garbageCollectorBean, GarbageCollectorMXBean::getCollectionCount)
+         FunctionCounter.builder(PREFIX + "gc.total", garbageCollectorBean, GarbageCollectorMXBean::getCollectionCount)
                .tags(Tags.of("name", garbageCollectorBean.getName()))
                .description("Displays the total number of collections that have occurred. This attribute lists -1 if the collection count is undefined for this collector.")
                .register(registry);
 
-         FunctionCounter.builder("gc.time", garbageCollectorBean, GarbageCollectorMXBean::getCollectionTime)
+         FunctionCounter.builder(PREFIX + "gc.time", garbageCollectorBean, GarbageCollectorMXBean::getCollectionTime)
                .tags(Tags.of("name", garbageCollectorBean.getName()))
                .description("Displays the approximate accumulated collection elapsed time in milliseconds. This attribute displays -1 if the collection elapsed time is undefined for this collector. The Java virtual machine implementation may use a high resolution timer to measure the elapsed time. This attribute might display the same value even if the collection count has been incremented if the collection elapsed time is very short.")
                .register(registry);
@@ -61,27 +63,27 @@ private void bindGarbageCollectionMetrics(MeterRegistry registry) {
    private void bindRuntimeMetrics(MeterRegistry registry) {
       RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
 
-      Gauge.builder("jvm.uptime", runtimeBean, RuntimeMXBean::getUptime)
+      Gauge.builder(PREFIX + "jvm.uptime", runtimeBean, RuntimeMXBean::getUptime)
             .description("Displays the uptime of the Java virtual machine.")
             .register(registry);
    }
 
    private void bindThreadingMetrics(MeterRegistry registry) {
       ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
 
-      Gauge.builder("thread.count", threadBean, ThreadMXBean::getThreadCount)
+      Gauge.builder(PREFIX + "thread.count", threadBean, ThreadMXBean::getThreadCount)
             .description("Displays the current thread count.")
             .register(registry);
 
-      Gauge.builder("thread.daemon.count", threadBean, ThreadMXBean::getDaemonThreadCount)
+      Gauge.builder(PREFIX + "thread.daemon.count", threadBean, ThreadMXBean::getDaemonThreadCount)
             .description("Displays the current number of live daemon threads.")
             .register(registry);
 
-      Gauge.builder("thread.max.count", threadBean, ThreadMXBean::getPeakThreadCount)
+      Gauge.builder(PREFIX + "thread.max.count", threadBean, ThreadMXBean::getPeakThreadCount)
             .description("Displays the peak live thread count since the Java virtual machine started or peak was reset. This includes daemon and non-daemon threads.")
             .register(registry);
 
-      Gauge.builder("thread.totalStarted", threadBean, ThreadMXBean::getTotalStartedThreadCount)
+      Gauge.builder(PREFIX + "thread.totalStarted", threadBean, ThreadMXBean::getTotalStartedThreadCount)
             .description("Displays the total number of started threads.")
             .register(registry);
    }

core/src/main/java/org/infinispan/metrics/impl/BaseMemoryAdditionalMetrics.java

@@ -1,5 +1,7 @@
 package org.infinispan.metrics.impl;
 
+import static org.infinispan.metrics.impl.BaseAdditionalMetrics.PREFIX;
+
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
 
@@ -14,42 +16,42 @@ public class BaseMemoryAdditionalMetrics implements MeterBinder {
    public void bindTo(MeterRegistry registry) {
       MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean();
 
-      Gauge.builder("memory.committedHeap", memoryBean, (mem) -> mem.getHeapMemoryUsage().getCommitted())
+      Gauge.builder(PREFIX + "memory.committedHeap", memoryBean, (mem) -> mem.getHeapMemoryUsage().getCommitted())
             .baseUnit(BaseUnits.BYTES)
             .description("Displays the amount of memory that is committed for the Java virtual machine to use.")
             .register(registry);
 
-      Gauge.builder("memory.maxHeap", memoryBean, (mem) -> mem.getHeapMemoryUsage().getMax())
+      Gauge.builder(PREFIX + "memory.maxHeap", memoryBean, (mem) -> mem.getHeapMemoryUsage().getMax())
             .baseUnit(BaseUnits.BYTES)
             .description("Displays the maximum amount of memory, in bytes, that can be used for memory management.")
             .register(registry);
 
-      Gauge.builder("memory.usedHeap", memoryBean, (mem) -> mem.getHeapMemoryUsage().getUsed())
+      Gauge.builder(PREFIX + "memory.usedHeap", memoryBean, (mem) -> mem.getHeapMemoryUsage().getUsed())
             .baseUnit(BaseUnits.BYTES)
             .description("Displays the amount of used memory.")
             .register(registry);
 
-      Gauge.builder("memory.initHeap", memoryBean, (mem) -> mem.getHeapMemoryUsage().getInit())
+      Gauge.builder(PREFIX + "memory.initHeap", memoryBean, (mem) -> mem.getHeapMemoryUsage().getInit())
             .baseUnit(BaseUnits.BYTES)
             .description("Displays the initial amount of allocated heap memory in bytes.")
             .register(registry);
 
-      Gauge.builder("memory.committedNonHeap", memoryBean, (mem) -> mem.getNonHeapMemoryUsage().getCommitted())
+      Gauge.builder(PREFIX + "memory.committedNonHeap", memoryBean, (mem) -> mem.getNonHeapMemoryUsage().getCommitted())
             .baseUnit(BaseUnits.BYTES)
             .description("Displays the amount of memory that is committed for the Java virtual machine to use.")
             .register(registry);
 
-      Gauge.builder("memory.maxNonHeap", memoryBean, (mem) -> mem.getNonHeapMemoryUsage().getMax())
+      Gauge.builder(PREFIX + "memory.maxNonHeap", memoryBean, (mem) -> mem.getNonHeapMemoryUsage().getMax())
             .baseUnit(BaseUnits.BYTES)
             .description("Displays the maximum amount of memory in bytes that can be used for memory management.")
             .register(registry);
 
-      Gauge.builder("memory.usedNonHeap", memoryBean, (mem) -> mem.getNonHeapMemoryUsage().getUsed())
+      Gauge.builder(PREFIX + "memory.usedNonHeap", memoryBean, (mem) -> mem.getNonHeapMemoryUsage().getUsed())
             .baseUnit(BaseUnits.BYTES)
             .description("Displays the amount of used memory.")
             .register(registry);
 
-      Gauge.builder("memory.initNonHeap", memoryBean, (mem) -> mem.getNonHeapMemoryUsage().getInit())
+      Gauge.builder(PREFIX + "memory.initNonHeap", memoryBean, (mem) -> mem.getNonHeapMemoryUsage().getInit())
             .baseUnit(BaseUnits.BYTES)
             .description("Displays the initial amount of allocated memory, in bytes, for off-heap storage.")
             .register(registry);

core/src/main/java/org/infinispan/metrics/impl/BaseOperatingSystemAdditionalMetrics.java

@@ -1,5 +1,7 @@
 package org.infinispan.metrics.impl;
 
+import static org.infinispan.metrics.impl.BaseAdditionalMetrics.PREFIX;
+
 import java.lang.management.ManagementFactory;
 import java.lang.management.OperatingSystemMXBean;
 import java.lang.reflect.InvocationTargetException;
@@ -41,22 +43,22 @@ public class BaseOperatingSystemAdditionalMetrics implements MeterBinder {
 
    @Override
    public void bindTo(MeterRegistry registry) {
-      Gauge.builder("cpu.availableProcessors", operatingSystemBean, OperatingSystemMXBean::getAvailableProcessors)
+      Gauge.builder(PREFIX + "cpu.availableProcessors", operatingSystemBean, OperatingSystemMXBean::getAvailableProcessors)
             .description("Displays the number of processors available to the Java virtual machine. This value may change during a particular invocation of the virtual machine.")
             .register(registry);
 
-      Gauge.builder("cpu.systemLoadAverage", operatingSystemBean, OperatingSystemMXBean::getSystemLoadAverage)
+      Gauge.builder(PREFIX + "cpu.systemLoadAverage", operatingSystemBean, OperatingSystemMXBean::getSystemLoadAverage)
             .description("Displays the system load average for the last minute. The system load average is the sum of the number of runnable entities queued to the available processors and the number of runnable entities running on the available processors averaged over a period of time. The way in which the load average is calculated is operating system specific but is typically a damped time-dependent average. If the load average is not available, a negative value is displayed. This attribute is designed to provide a hint about the system load and may be queried frequently. The load average might be unavailable on some platforms where it is expensive to implement this method.")
             .register(registry);
 
       if (processCpuLoadMethod != null) {
-         Gauge.builder("cpu.processCpuLoad", () -> invoke(processCpuLoadMethod))
+         Gauge.builder(PREFIX + "cpu.processCpuLoad", () -> invoke(processCpuLoadMethod))
                .description("Displays the \"recent cpu usage\" for the Java virtual machine process.")
                .register(registry);
       }
 
       if (processCpuTimeMethod != null) {
-         Gauge.builder("cpu.processCpuTime", () -> invoke(processCpuTimeMethod))
+         Gauge.builder(PREFIX + "cpu.processCpuTime", () -> invoke(processCpuTimeMethod))
                .description("Displays the CPU time, in nanoseconds, used by the process on which the Java virtual machine is running.")
                .register(registry);
       }

core/src/main/java/org/infinispan/metrics/impl/MetricsCollector.java

@@ -32,7 +32,6 @@
 import io.micrometer.core.instrument.Meter;
 import io.micrometer.core.instrument.Tag;
 import io.micrometer.core.instrument.Timer;
-import io.micrometer.core.instrument.config.MeterFilter;
 import io.micrometer.prometheus.PrometheusConfig;
 import io.micrometer.prometheus.PrometheusMeterRegistry;
 
@@ -49,8 +48,7 @@ public class MetricsCollector implements Constants {
 
    private static final Log log = LogFactory.getLog(MetricsCollector.class);
 
-   private PrometheusMeterRegistry baseRegistry;
-   private PrometheusMeterRegistry vendorRegistry;
+   private PrometheusMeterRegistry registry;
 
    private Tag nodeTag;
 
@@ -65,23 +63,15 @@ public class MetricsCollector implements Constants {
    protected MetricsCollector() {
    }
 
-   public PrometheusMeterRegistry getBaseRegistry() {
-      return baseRegistry;
-   }
-
-   public PrometheusMeterRegistry getVendorRegistry() {
-      return vendorRegistry;
+   public PrometheusMeterRegistry registry() {
+      return registry;
    }
 
    @Start
    protected void start() {
-      baseRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
-      baseRegistry.config().meterFilter(new BaseFilter());
-      new BaseAdditionalMetrics().bindTo(baseRegistry);
-
-      vendorRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
-      vendorRegistry.config().meterFilter(new VendorFilter());
-      new VendorAdditionalMetrics().bindTo(vendorRegistry);
+      registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
+      new BaseAdditionalMetrics().bindTo(registry);
+      new VendorAdditionalMetrics().bindTo(registry);
 
       Transport transport = transportRef.running();
       String nodeName = transport != null ? transport.getAddress().toString() : globalConfig.transport().nodeName();
@@ -99,16 +89,9 @@ protected void start() {
 
    @Stop
    protected void stop() {
-      try {
-         if (baseRegistry != null) {
-            baseRegistry.close();
-            baseRegistry = null;
-         }
-      } finally {
-         if (vendorRegistry != null) {
-            vendorRegistry.close();
-            vendorRegistry = null;
-         }
+      if (registry != null) {
+         registry.close();
+         registry = null;
       }
    }
 
@@ -159,15 +142,15 @@ private Set<Object> registerMetrics(Object instance, Collection<MBeanMetadata.At
          Consumer<TimerTracker> setter = (Consumer<TimerTracker>) attr.setter(instance);
 
          if (getter != null || setter != null) {
-            String metricName = namePrefix + NameUtils.decamelize(attr.getName());
+            String metricName = VendorAdditionalMetrics.PREFIX + namePrefix + NameUtils.decamelize(attr.getName());
 
             if (getter != null) {
                if (metricsCfg.gauges()) {
                   Gauge gauge = Gauge.builder(metricName, getter)
                         .tags(tags)
                         .strongReference(true)
                         .description(attr.getDescription())
-                        .register(vendorRegistry);
+                        .register(registry);
 
                   Meter.Id id = gauge.getId();
 
@@ -181,7 +164,7 @@ private Set<Object> registerMetrics(Object instance, Collection<MBeanMetadata.At
                   Timer timer = Timer.builder(metricName)
                         .tags(tags)
                         .description(attr.getDescription())
-                        .register(vendorRegistry);
+                        .register(registry);
 
                   Meter.Id id = timer.getId();
 
@@ -197,7 +180,7 @@ private Set<Object> registerMetrics(Object instance, Collection<MBeanMetadata.At
 
       if (log.isTraceEnabled()) {
          log.tracef("Registered %d metrics. Metric registry @%x contains %d metrics.",
-               metricIds.size(), System.identityHashCode(vendorRegistry), vendorRegistry.getMeters().size());
+               metricIds.size(), System.identityHashCode(registry), registry.getMeters().size());
       }
 
       return metricIds;
@@ -217,37 +200,19 @@ private Tag asTag(String key, String value) {
    }
 
    public void unregisterMetric(Object metricId) {
-      if (vendorRegistry == null) {
+      if (registry == null) {
          return;
       }
 
-      Meter removed = vendorRegistry.remove((Meter.Id) metricId);
+      Meter removed = registry.remove((Meter.Id) metricId);
       if (log.isTraceEnabled()) {
          if (removed != null) {
             log.tracef("Unregistered metric \"%s\". Metric registry @%x contains %d metrics.",
-                  metricId, System.identityHashCode(vendorRegistry), vendorRegistry.getMeters().size());
+                  metricId, System.identityHashCode(registry), registry.getMeters().size());
          } else {
             log.tracef("Could not remove unexisting metric \"%s\". Metric registry @%x contains %d metrics.",
-                  metricId, System.identityHashCode(vendorRegistry), vendorRegistry.getMeters().size());
+                  metricId, System.identityHashCode(registry), registry.getMeters().size());
          }
       }
    }
-
-   private static class BaseFilter implements MeterFilter {
-      private static final String PREFIX = "base.";
-
-      @Override
-      public Meter.Id map(Meter.Id id) {
-         return id.withName(PREFIX + id.getName());
-      }
-   }
-
-   private static class VendorFilter implements MeterFilter {
-      private static final String PREFIX = "vendor.";
-
-      @Override
-      public Meter.Id map(Meter.Id id) {
-         return id.withName(PREFIX + id.getName());
-      }
-   }
 }

core/src/main/java/org/infinispan/metrics/impl/VendorAdditionalMetrics.java

@@ -16,12 +16,14 @@
 @NonNullFields
 class VendorAdditionalMetrics implements MeterBinder {
 
+   static final String PREFIX = "vendor.";
+
    @Override
    public void bindTo(MeterRegistry registry) {
       List<BufferPoolMXBean> bufferPoolBeans = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class);
       for (BufferPoolMXBean bufferPoolBean : bufferPoolBeans) {
          String name = bufferPoolBean.getName();
-         Gauge.builder("BufferPool.used.memory." + name, bufferPoolBean, BufferPoolMXBean::getMemoryUsed)
+         Gauge.builder(PREFIX + "BufferPool.used.memory." + name, bufferPoolBean, BufferPoolMXBean::getMemoryUsed)
                .baseUnit(BaseUnits.BYTES)
                .description("The memory used by the NIO pool:" + name)
                .register(registry);
@@ -30,12 +32,12 @@ public void bindTo(MeterRegistry registry) {
       List<MemoryPoolMXBean> memoryPoolBeans = ManagementFactory.getMemoryPoolMXBeans();
       for (MemoryPoolMXBean memoryPoolBean : memoryPoolBeans) {
          String name = memoryPoolBean.getName();
-         Gauge.builder("memoryPool." + name + ".usage", memoryPoolBean, (mem) -> mem.getUsage().getUsed())
+         Gauge.builder(PREFIX + "memoryPool." + name + ".usage", memoryPoolBean, (mem) -> mem.getUsage().getUsed())
                .baseUnit(BaseUnits.BYTES)
                .description("Current usage of the " + name + " memory pool")
                .register(registry);
 
-         Gauge.builder("memoryPool." + name + ".usage.max", memoryPoolBean, (mem) -> mem.getPeakUsage().getUsed())
+         Gauge.builder(PREFIX + "memoryPool." + name + ".usage.max", memoryPoolBean, (mem) -> mem.getPeakUsage().getUsed())
                .baseUnit(BaseUnits.BYTES)
                .description("Peak usage of the " + name + " memory pool")
                .register(registry);