JMX Metrics to OTLP & Breeze

agent/agent-tooling/build.gradle.kts

@@ -21,7 +21,7 @@ dependencies {
   implementation(project(":agent:agent-profiler:agent-diagnostics"))
   implementation(project(":etw:java"))
 
-  implementation("com.azure:azure-monitor-opentelemetry-exporter:1.0.0-beta.14")
+  implementation("com.azure:azure-monitor-opentelemetry-exporter:1.0.0-beta.15")
   compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-bootstrap")
   compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-tooling")
   compileOnly("io.opentelemetry.javaagent.instrumentation:opentelemetry-javaagent-servlet-common-bootstrap")

agent/agent-tooling/gradle.lockfile

@@ -9,7 +9,7 @@ com.azure:azure-core-http-netty:1.13.11=runtimeClasspath
 com.azure:azure-core:1.45.1=runtimeClasspath
 com.azure:azure-identity:1.11.1=runtimeClasspath
 com.azure:azure-json:1.1.0=runtimeClasspath
-com.azure:azure-monitor-opentelemetry-exporter:1.0.0-beta.14=runtimeClasspath
+com.azure:azure-monitor-opentelemetry-exporter:1.0.0-beta.15=runtimeClasspath
 com.azure:azure-sdk-bom:1.2.19=runtimeClasspath
 com.azure:azure-storage-blob:12.25.1=runtimeClasspath
 com.azure:azure-storage-common:12.24.1=runtimeClasspath

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiConfigCustomizer.java

@@ -25,6 +25,10 @@ public Map<String, String> apply(ConfigProperties otelConfig) {
         "applicationinsights.internal.micrometer.step.millis",
         Long.toString(SECONDS.toMillis(configuration.metricIntervalSeconds)));
 
+    properties.put(
+        "otel.metric.export.interval",
+        Long.toString(SECONDS.toMillis(configuration.metricIntervalSeconds)));
+
     enableInstrumentations(otelConfig, configuration, properties);
 
     // enable "io.opentelemetry.sdk.autoconfigure.internal.EnvironmentResourceProvider" only. It

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/PerformanceCounterInitializer.java

@@ -12,12 +12,16 @@
 import com.microsoft.applicationinsights.agent.internal.perfcounter.FreeMemoryPerformanceCounter;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.GcPerformanceCounter;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.JmxAttributeData;
-import com.microsoft.applicationinsights.agent.internal.perfcounter.JmxMetricPerformanceCounter;
+import com.microsoft.applicationinsights.agent.internal.perfcounter.JmxDataFetcher;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.JvmHeapMemoryUsedPerformanceCounter;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.OshiPerformanceCounter;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.PerformanceCounterContainer;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.ProcessCpuPerformanceCounter;
 import com.microsoft.applicationinsights.agent.internal.perfcounter.ProcessMemoryPerformanceCounter;
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.ObservableDoubleMeasurement;
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadMXBean;
 import java.util.ArrayList;
@@ -73,8 +77,8 @@ private static boolean isAgentRunningInSandboxEnvWindows() {
    * a map where the key is the Jmx object name and the value is a list of requested attributes. 2.
    * Go through all the requested Jmx counters: a. If the object name is not in the map, add it with
    * an empty list Else get the list b. Add the attribute to the list. 3. Go through the map For
-   * every entry (object name and attributes) Build a {@link JmxMetricPerformanceCounter} Register
-   * the Performance Counter in the {@link PerformanceCounterContainer}
+   * every entry (object name and attributes) to build a meter per attribute & for each meter
+   * register a callback to report the metric value.
    */
   private static void loadCustomJmxPerfCounters(List<Configuration.JmxMetric> jmxXmlElements) {
     try {
@@ -109,23 +113,93 @@ private static void loadCustomJmxPerfCounters(List<Configuration.JmxMetric> jmxX
         collection.add(new JmxAttributeData(jmxElement.name, jmxElement.attribute));
       }
 
-      // Register each entry in the performance container
-      for (Map.Entry<String, Collection<JmxAttributeData>> entry : data.entrySet()) {
+      createMeterPerAttribute(data);
+
+    } catch (RuntimeException e) {
+      try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
+        logger.error("Failed to register JMX performance counters: '{}'", e.toString());
+      }
+    }
+  }
+
+  // Create a meter for each attribute & declare the callback that reports the metric in the meter.
+  private static void createMeterPerAttribute(
+      Map<String, Collection<JmxAttributeData>> objectAndAttributesMap) {
+    for (Map.Entry<String, Collection<JmxAttributeData>> entry :
+        objectAndAttributesMap.entrySet()) {
+      String objectName = entry.getKey();
+
+      for (JmxAttributeData jmxAttributeData : entry.getValue()) {
+
+        String otelMetricName;
+        if (jmxAttributeData.metricName.matches("[a-zA-z0-9_.-/]+")) {
+          otelMetricName = jmxAttributeData.metricName;
+        } else {
+          otelMetricName = jmxAttributeData.metricName.replaceAll("[^a-zA-z0-9_.-/]", "_");
+        }
+
+        JmxAttributeData newJmxAttributeData =
+            new JmxAttributeData(otelMetricName, jmxAttributeData.attribute);
+
+        GlobalOpenTelemetry.getMeter("com.microsoft.applicationinsights.jmx")
+            .gaugeBuilder(otelMetricName)
+            .buildWithCallback(
+                observableDoubleMeasurement -> {
+                  calculateAndRecordValueForAttribute(
+                      observableDoubleMeasurement, objectName, newJmxAttributeData);
+                });
+      }
+    }
+  }
+
+  private static void calculateAndRecordValueForAttribute(
+      ObservableDoubleMeasurement observableDoubleMeasurement,
+      String objectName,
+      JmxAttributeData jmxAttributeData) {
+    try {
+      List<Object> result =
+          JmxDataFetcher.fetch(
+              objectName, jmxAttributeData.attribute); // should return the [val, ...] here
+
+      logger.trace(
+          "Size of the JmxDataFetcher.fetch result: {}, for objectName:{} and metricName:{}",
+          result.size(),
+          objectName,
+          jmxAttributeData.metricName);
+
+      boolean ok = true;
+      double value = 0.0;
+      for (Object obj : result) {
         try {
-          PerformanceCounterContainer.INSTANCE.register(
-              new JmxMetricPerformanceCounter(entry.getKey(), entry.getValue()));
-        } catch (RuntimeException e) {
-          try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
-            logger.error(
-                "Failed to register JMX performance counter: '{}' : '{}'",
-                entry.getKey(),
-                e.toString());
+          if (obj instanceof Boolean) {
+            value = ((Boolean) obj).booleanValue() ? 1 : 0;
+          } else {
+            value += Double.parseDouble(String.valueOf(obj));
           }
+        } catch (RuntimeException e) {
+          ok = false;
+          break;
         }
       }
-    } catch (RuntimeException e) {
+      if (ok) {
+        logger.trace(
+            "value {} for objectName:{} and metricName{}",
+            value,
+            objectName,
+            jmxAttributeData.metricName);
+        observableDoubleMeasurement.record(
+            value,
+            Attributes.of(
+                AttributeKey.stringKey("applicationinsights.internal.metric_name"),
+                jmxAttributeData.metricName));
+      }
+    } catch (Exception e) {
       try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
-        logger.error("Failed to register JMX performance counters: '{}'", e.toString());
+        logger.error(
+            "Failed to calculate the metric value for objectName {} and metric name {}",
+            objectName,
+            jmxAttributeData.metricName);
+        logger.error("Exception: {}", e.toString());
       }
     }
   }

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/perfcounter/JmxDataFetcher.java

@@ -66,6 +66,37 @@ public static Map<String, Collection<Object>> fetch(
     return result;
   }
 
+  /**
+   * Gets an object name and an attribute to fetch and will return the data.
+   *
+   * @param objectName The object name to search.
+   * @param attribute An attribute that belongs to the object name
+   * @return A List of values found for that attribute
+   * @throws Exception In case the object name is not found.
+   */
+  public static List<Object> fetch(String objectName, String attribute) throws Exception {
+    List<Object> resultForAttribute = new ArrayList<>();
+
+    MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+    Set<ObjectName> objects = server.queryNames(new ObjectName(objectName), null);
+    logger.trace("Matching object names for pattern {}: {}", objectName, objects.toString());
+    if (objects.isEmpty()) {
+      String errorMsg = String.format("Cannot find object name '%s'", objectName);
+      throw new IllegalArgumentException(errorMsg);
+    }
+
+    try {
+      resultForAttribute = fetch(server, objects, attribute);
+    } catch (Exception e) {
+      try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
+        logger.warn("Failed to fetch JMX object '{}' with attribute '{}': ", objectName, attribute);
+      }
+      throw e;
+    }
+
+    return resultForAttribute;
+  }
+
   private static List<Object> fetch(
       MBeanServer server, Set<ObjectName> objects, String attributeName)
       throws AttributeNotFoundException,

agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/perfcounter/JmxDataFetcherTest.java

@@ -68,6 +68,13 @@ void testBadAttributeName() throws Exception {
 
     assertThatThrownBy(() -> JmxDataFetcher.fetch("JSDKTests:type=TestStub3", attributes))
         .isInstanceOf(Exception.class);
+
+    // Test the other fetch method
+    for (JmxAttributeData attribute : attributes) {
+      assertThatThrownBy(
+              () -> JmxDataFetcher.fetch("JSDKTests:type=TestStub3", attribute.attribute))
+          .isInstanceOf(Exception.class);
+    }
   }
 
   @Test
@@ -81,6 +88,11 @@ void testBadName() throws Exception {
 
     assertThatThrownBy(() -> JmxDataFetcher.fetch("JSDKTests:type=TestStub", attributes))
         .isInstanceOf(IllegalArgumentException.class);
+
+    // Test the other fetch method
+    assertThatThrownBy(
+            () -> JmxDataFetcher.fetch("JSDKTests:type=TestStub", attributes.get(0).attribute))
+        .isInstanceOf(IllegalArgumentException.class);
   }
 
   @Test
@@ -119,6 +131,27 @@ private static void performTest(
     verify(result, "Int", expectedInt);
     verify(result, "Double", expectedDouble);
     verify(result, "Long", expectedLong);
+
+    // verify the other fetch
+    for (int i = 0; i < attributes.size(); i++) {
+      List<Object> singleAttributeResult =
+          JmxDataFetcher.fetch("JSDKTests:type=TestStub", attributes.get(i).attribute);
+      assertThat(singleAttributeResult).isNotNull();
+      assertThat(singleAttributeResult.size()).isEqualTo(1);
+
+      double value = 0.0;
+      for (Object obj : singleAttributeResult) {
+        value += Double.parseDouble(String.valueOf(obj));
+      }
+
+      if (i == 0) {
+        assertThat(value).isEqualTo(expectedInt);
+      } else if (i == 1) {
+        assertThat(value).isEqualTo(expectedDouble);
+      } else {
+        assertThat(value).isEqualTo(expectedLong);
+      }
+    }
   }
 
   private static void verify(

licenses/more-licenses.md

@@ -1,7 +1,7 @@
 
 # agent
 ## Dependency License Report
-_2023-12-14 03:16:39 UTC_
+_2023-12-19 12:59:44 PST_
 ## Apache License, Version 2.0
 
 **1** **Group:** `com.fasterxml.jackson.core` **Name:** `jackson-annotations` **Version:** `2.16.0` 
@@ -363,7 +363,7 @@ _2023-12-14 03:16:39 UTC_
 > - **POM Project URL**: [https://github.com/Azure/azure-sdk-for-java](https://github.com/Azure/azure-sdk-for-java)
 > - **POM License**: MIT License - [https://opensource.org/licenses/MIT](https://opensource.org/licenses/MIT)
 
-**70** **Group:** `com.azure` **Name:** `azure-monitor-opentelemetry-exporter` **Version:** `1.0.0-beta.14` 
+**70** **Group:** `com.azure` **Name:** `azure-monitor-opentelemetry-exporter` **Version:** `1.0.0-beta.15` 
 > - **POM License**: MIT License - [https://opensource.org/licenses/MIT](https://opensource.org/licenses/MIT)
 
 **71** **Group:** `com.azure` **Name:** `azure-storage-blob` **Version:** `12.25.1` 

settings.gradle.kts

@@ -98,7 +98,7 @@ hideFromDependabot(":smoke-tests:apps:Diagnostics")
 hideFromDependabot(":smoke-tests:apps:Diagnostics:JfrFileReader")
 hideFromDependabot(":smoke-tests:apps:DiagnosticExtension:MockExtension")
 hideFromDependabot(":smoke-tests:apps:DiagnosticExtension")
-hideFromDependabot(":smoke-tests:apps:DotInJmxMetric")
+hideFromDependabot(":smoke-tests:apps:JmxMetric")
 hideFromDependabot(":smoke-tests:apps:gRPC")
 hideFromDependabot(":smoke-tests:apps:HeartBeat")
 hideFromDependabot(":smoke-tests:apps:HttpClients")

smoke-tests/apps/DetectUnexpectedOtelMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/DetectUnexpectedOtelMetricsTest.java

@@ -14,6 +14,8 @@
 import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.TimeoutException;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -23,14 +25,22 @@ abstract class DetectUnexpectedOtelMetricsTest {
 
   @RegisterExtension static final SmokeTestExtension testing = SmokeTestExtension.create();
 
+  private static final List<String> EXPECTED_METRIC_NAMES = new ArrayList<>();
+
+  static {
+    EXPECTED_METRIC_NAMES.add("_OTELRESOURCE_");
+    EXPECTED_METRIC_NAMES.add("Current_Thread_Count");
+    EXPECTED_METRIC_NAMES.add("Loaded_Class_Count");
+  }
+
   @Test
   @TargetUri("/app")
   void testApp() throws Exception {
     // verify no unexpected otel metrics, expect an TimeoutException being thrown
     assertThatThrownBy(
             () ->
                 testing.mockedIngestion.waitForItemsUnexpectedOtelMetric(
-                    "MetricData", envelope -> true))
+                    "MetricData", envelope -> true, EXPECTED_METRIC_NAMES))
         .isInstanceOf(TimeoutException.class);
   }