JMX Metrics to OTLP & Breeze

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiConfigCustomizer.java

@@ -25,6 +25,10 @@ public Map<String, String> apply(ConfigProperties otelConfig) {
         "applicationinsights.internal.micrometer.step.millis",
         Long.toString(SECONDS.toMillis(configuration.metricIntervalSeconds)));
 
+    properties.put(
+        "otel.metric.export.interval",
+        Long.toString(SECONDS.toMillis(configuration.metricIntervalSeconds)));
+
     enableInstrumentations(otelConfig, configuration, properties);
 
     // enable "io.opentelemetry.sdk.autoconfigure.internal.EnvironmentResourceProvider" only. It

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/PerformanceCounterInitializer.java

@@ -12,12 +12,16 @@
 import com.microsoft.applicationinsights.agent.internal.perfcounter.FreeMemoryPerformanceCounter;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.GcPerformanceCounter;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.JmxAttributeData;
-import com.microsoft.applicationinsights.agent.internal.perfcounter.JmxMetricPerformanceCounter;
+import com.microsoft.applicationinsights.agent.internal.perfcounter.JmxDataFetcher;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.JvmHeapMemoryUsedPerformanceCounter;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.OshiPerformanceCounter;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.PerformanceCounterContainer;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.ProcessCpuPerformanceCounter;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.ProcessMemoryPerformanceCounter;
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.ObservableDoubleMeasurement;
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadMXBean;
 import java.util.ArrayList;
@@ -32,6 +36,8 @@
 public class PerformanceCounterInitializer {
 
   private static final Logger logger = LoggerFactory.getLogger(PerformanceCounterInitializer.class);
+  private static final String METRIC_NAME_REGEXP = "[a-zA-z0-9_.-/]+";
+  private static final String INVALID_CHARACTER_REGEXP = "[^a-zA-z0-9_.-/]";
 
   public static void initialize(Configuration configuration) {
 
@@ -73,59 +79,119 @@ private static boolean isAgentRunningInSandboxEnvWindows() {
    * a map where the key is the Jmx object name and the value is a list of requested attributes. 2.
    * Go through all the requested Jmx counters: a. If the object name is not in the map, add it with
    * an empty list Else get the list b. Add the attribute to the list. 3. Go through the map For
-   * every entry (object name and attributes) Build a {@link JmxMetricPerformanceCounter} Register
-   * the Performance Counter in the {@link PerformanceCounterContainer}
+   * every entry (object name and attributes) to build a meter per attribute & for each meter
+   * register a callback to report the metric value.
    */
   private static void loadCustomJmxPerfCounters(List<Configuration.JmxMetric> jmxXmlElements) {
-    try {
-      HashMap<String, Collection<JmxAttributeData>> data = new HashMap<>();
+    HashMap<String, Collection<JmxAttributeData>> data = new HashMap<>();
 
-      // Build a map of object name to its requested attributes
-      for (Configuration.JmxMetric jmxElement : jmxXmlElements) {
-        Collection<JmxAttributeData> collection =
-            data.computeIfAbsent(jmxElement.objectName, k -> new ArrayList<>());
+    // Build a map of object name to its requested attributes
+    for (Configuration.JmxMetric jmxElement : jmxXmlElements) {
+      Collection<JmxAttributeData> collection =
+          data.computeIfAbsent(jmxElement.objectName, k -> new ArrayList<>());
 
-        if (Strings.isNullOrEmpty(jmxElement.objectName)) {
-          try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
-            logger.error("JMX object name is empty, will be ignored");
-          }
-          continue;
+      if (Strings.isNullOrEmpty(jmxElement.objectName)) {
+        try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
+          logger.error("JMX object name is empty, will be ignored");
         }
+        continue;
+      }
 
-        if (Strings.isNullOrEmpty(jmxElement.attribute)) {
-          try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
-            logger.error("JMX attribute is empty for '{}'", jmxElement.objectName);
-          }
-          continue;
+      if (Strings.isNullOrEmpty(jmxElement.attribute)) {
+        try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
+          logger.error("JMX attribute is empty for '{}'", jmxElement.objectName);
         }
+        continue;
+      }
 
-        if (Strings.isNullOrEmpty(jmxElement.name)) {
-          try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
-            logger.error("JMX name is empty for '{}', will be ignored", jmxElement.objectName);
-          }
-          continue;
+      if (Strings.isNullOrEmpty(jmxElement.name)) {
+        try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
+          logger.error("JMX name is empty for '{}', will be ignored", jmxElement.objectName);
         }
+        continue;
+      }
+
+      collection.add(new JmxAttributeData(jmxElement.name, jmxElement.attribute));
+    }
+
+    createMeterPerAttribute(data);
+  }
 
-        collection.add(new JmxAttributeData(jmxElement.name, jmxElement.attribute));
+  // Create a meter for each attribute & declare the callback that reports the metric in the meter.
+  private static void createMeterPerAttribute(
+      Map<String, Collection<JmxAttributeData>> objectAndAttributesMap) {
+    for (Map.Entry<String, Collection<JmxAttributeData>> entry :
+        objectAndAttributesMap.entrySet()) {
+      String objectName = entry.getKey();
+
+      for (JmxAttributeData jmxAttributeData : entry.getValue()) {
+
+        String otelMetricName;
+        if (jmxAttributeData.metricName.matches(METRIC_NAME_REGEXP)) {
+          otelMetricName = jmxAttributeData.metricName;
+        } else {
+          otelMetricName = jmxAttributeData.metricName.replaceAll(INVALID_CHARACTER_REGEXP, "_");
+        }
+
+        GlobalOpenTelemetry.getMeter("com.microsoft.applicationinsights.jmx")
+            .gaugeBuilder(otelMetricName)
+            .buildWithCallback(
+                observableDoubleMeasurement -> {
+                  calculateAndRecordValueForAttribute(
+                      observableDoubleMeasurement, objectName, jmxAttributeData);
+                });
       }
+    }
+  }
 
-      // Register each entry in the performance container
-      for (Map.Entry<String, Collection<JmxAttributeData>> entry : data.entrySet()) {
+  private static void calculateAndRecordValueForAttribute(
+      ObservableDoubleMeasurement observableDoubleMeasurement,
+      String objectName,
+      JmxAttributeData jmxAttributeData) {
+    try {
+      List<Object> result =
+          JmxDataFetcher.fetch(
+              objectName, jmxAttributeData.attribute); // should return the [val, ...] here
+
+      logger.trace(
+          "Size of the JmxDataFetcher.fetch result: {}, for objectName:{} and metricName:{}",
+          result.size(),
+          objectName,
+          jmxAttributeData.metricName);
+
+      boolean ok = true;
+      double value = 0.0;
+      for (Object obj : result) {
         try {
-          PerformanceCounterContainer.INSTANCE.register(
-              new JmxMetricPerformanceCounter(entry.getKey(), entry.getValue()));
-        } catch (RuntimeException e) {
-          try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
-            logger.error(
-                "Failed to register JMX performance counter: '{}' : '{}'",
-                entry.getKey(),
-                e.toString());
+          if (obj instanceof Boolean) {
+            value = ((Boolean) obj).booleanValue() ? 1 : 0;
+          } else {
+            value += Double.parseDouble(String.valueOf(obj));
           }
+        } catch (RuntimeException e) {
+          ok = false;
+          break;
         }
       }
-    } catch (RuntimeException e) {
+      if (ok) {
+        logger.trace(
+            "value {} for objectName:{} and metricName{}",
+            value,
+            objectName,
+            jmxAttributeData.metricName);
+        observableDoubleMeasurement.record(
+            value,
+            Attributes.of(
+                AttributeKey.stringKey("applicationinsights.internal.metric_name"),
+                jmxAttributeData.metricName));
+      }
+    } catch (Exception e) {
       try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
-        logger.error("Failed to register JMX performance counters: '{}'", e.toString());
+        logger.error(
+            "Failed to calculate the metric value for objectName {} and metric name {}",
+            objectName,
+            jmxAttributeData.metricName);
+        logger.error("Exception: {}", e.toString());
       }
     }
   }

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/perfcounter/JmxDataFetcher.java

@@ -66,6 +66,37 @@ public static Map<String, Collection<Object>> fetch(
     return result;
   }
 
+  /**
+   * Gets an object name and an attribute to fetch and will return the data.
+   *
+   * @param objectName The object name to search.
+   * @param attribute An attribute that belongs to the object name
+   * @return A List of values found for that attribute
+   * @throws Exception In case the object name is not found.
+   */
+  public static List<Object> fetch(String objectName, String attribute) throws Exception {
+    List<Object> resultForAttribute = new ArrayList<>();
+
+    MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+    Set<ObjectName> objects = server.queryNames(new ObjectName(objectName), null);
+    logger.trace("Matching object names for pattern {}: {}", objectName, objects.toString());
+    if (objects.isEmpty()) {
+      String errorMsg = String.format("Cannot find object name '%s'", objectName);
+      throw new IllegalArgumentException(errorMsg);
+    }
+
+    try {
+      resultForAttribute = fetch(server, objects, attribute);
+    } catch (Exception e) {
+      try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
+        logger.warn("Failed to fetch JMX object '{}' with attribute '{}': ", objectName, attribute);
+      }
+      throw e;
+    }
+
+    return resultForAttribute;
+  }
+
   private static List<Object> fetch(
       MBeanServer server, Set<ObjectName> objects, String attributeName)
       throws AttributeNotFoundException,

agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/perfcounter/JmxDataFetcherTest.java

@@ -68,6 +68,13 @@ void testBadAttributeName() throws Exception {
 
     assertThatThrownBy(() -> JmxDataFetcher.fetch("JSDKTests:type=TestStub3", attributes))
         .isInstanceOf(Exception.class);
+
+    // Test the other fetch method
+    for (JmxAttributeData attribute : attributes) {
+      assertThatThrownBy(
+              () -> JmxDataFetcher.fetch("JSDKTests:type=TestStub3", attribute.attribute))
+          .isInstanceOf(Exception.class);
+    }
   }
 
   @Test
@@ -81,6 +88,11 @@ void testBadName() throws Exception {
 
     assertThatThrownBy(() -> JmxDataFetcher.fetch("JSDKTests:type=TestStub", attributes))
         .isInstanceOf(IllegalArgumentException.class);
+
+    // Test the other fetch method
+    assertThatThrownBy(
+            () -> JmxDataFetcher.fetch("JSDKTests:type=TestStub", attributes.get(0).attribute))
+        .isInstanceOf(IllegalArgumentException.class);
   }
 
   @Test
@@ -119,6 +131,27 @@ private static void performTest(
     verify(result, "Int", expectedInt);
     verify(result, "Double", expectedDouble);
     verify(result, "Long", expectedLong);
+
+    // verify the other fetch
+    for (int i = 0; i < attributes.size(); i++) {
+      List<Object> singleAttributeResult =
+          JmxDataFetcher.fetch("JSDKTests:type=TestStub", attributes.get(i).attribute);
+      assertThat(singleAttributeResult).isNotNull();
+      assertThat(singleAttributeResult.size()).isEqualTo(1);
+
+      double value = 0.0;
+      for (Object obj : singleAttributeResult) {
+        value += Double.parseDouble(String.valueOf(obj));
+      }
+
+      if (i == 0) {
+        assertThat(value).isEqualTo(expectedInt);
+      } else if (i == 1) {
+        assertThat(value).isEqualTo(expectedDouble);
+      } else {
+        assertThat(value).isEqualTo(expectedLong);
+      }
+    }
   }
 
   private static void verify(

settings.gradle.kts

@@ -98,7 +98,7 @@ hideFromDependabot(":smoke-tests:apps:Diagnostics")
 hideFromDependabot(":smoke-tests:apps:Diagnostics:JfrFileReader")
 hideFromDependabot(":smoke-tests:apps:DiagnosticExtension:MockExtension")
 hideFromDependabot(":smoke-tests:apps:DiagnosticExtension")
-hideFromDependabot(":smoke-tests:apps:DotInJmxMetric")
+hideFromDependabot(":smoke-tests:apps:JmxMetric")
 hideFromDependabot(":smoke-tests:apps:gRPC")
 hideFromDependabot(":smoke-tests:apps:HeartBeat")
 hideFromDependabot(":smoke-tests:apps:HttpClients")

smoke-tests/apps/DetectUnexpectedOtelMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/DetectUnexpectedOtelMetricsTest.java

@@ -14,6 +14,8 @@
 import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.TimeoutException;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -23,14 +25,22 @@ abstract class DetectUnexpectedOtelMetricsTest {
 
   @RegisterExtension static final SmokeTestExtension testing = SmokeTestExtension.create();
 
+  private static final List<String> EXPECTED_METRIC_NAMES = new ArrayList<>();
+
+  static {
+    EXPECTED_METRIC_NAMES.add("_OTELRESOURCE_");
+    EXPECTED_METRIC_NAMES.add("Current Thread Count");
+    EXPECTED_METRIC_NAMES.add("Loaded Class Count");
+  }
+
   @Test
   @TargetUri("/app")
   void testApp() throws Exception {
     // verify no unexpected otel metrics, expect an TimeoutException being thrown
     assertThatThrownBy(
             () ->
                 testing.mockedIngestion.waitForItemsUnexpectedOtelMetric(
-                    "MetricData", envelope -> true))
+                    "MetricData", envelope -> true, EXPECTED_METRIC_NAMES))
         .isInstanceOf(TimeoutException.class);
   }