Skip to content

Commit

Permalink
JMX Metrics to OTLP & Breeze (#3406)
Browse files Browse the repository at this point in the history 
Co-authored-by: heyams <heya@microsoft.com>
Co-authored-by: Helen <56097766+heyams@users.noreply.github.com>
  • Loading branch information
@harsimar @heyams
3 people committed Jan 25, 2024
1 parent 0b18ef9 commit 1deaee2
Show file tree
Hide file tree
Showing 16 changed files with 471 additions and 127 deletions.
4 changes: 4 additions & 0 deletions ...c/main/java/com/microsoft/applicationinsights/agent/internal/init/AiConfigCustomizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ public Map<String, String> apply(ConfigProperties otelConfig) {
"applicationinsights.internal.micrometer.step.millis",
Long.toString(SECONDS.toMillis(configuration.metricIntervalSeconds)));

properties.put(
"otel.metric.export.interval",
Long.toString(SECONDS.toMillis(configuration.metricIntervalSeconds)));

enableInstrumentations(otelConfig, configuration, properties);

// enable "io.opentelemetry.sdk.autoconfigure.internal.EnvironmentResourceProvider" only. It
Expand Down
140 changes: 103 additions & 37 deletions .../com/microsoft/applicationinsights/agent/internal/init/PerformanceCounterInitializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,16 @@
import com.microsoft.applicationinsights.agent.internal.perfcounter.FreeMemoryPerformanceCounter;
import com.microsoft.applicationinsights.agent.internal.perfcounter.GcPerformanceCounter;
import com.microsoft.applicationinsights.agent.internal.perfcounter.JmxAttributeData;
import com.microsoft.applicationinsights.agent.internal.perfcounter.JmxMetricPerformanceCounter;
import com.microsoft.applicationinsights.agent.internal.perfcounter.JmxDataFetcher;
import com.microsoft.applicationinsights.agent.internal.perfcounter.JvmHeapMemoryUsedPerformanceCounter;
import com.microsoft.applicationinsights.agent.internal.perfcounter.OshiPerformanceCounter;
import com.microsoft.applicationinsights.agent.internal.perfcounter.PerformanceCounterContainer;
import com.microsoft.applicationinsights.agent.internal.perfcounter.ProcessCpuPerformanceCounter;
import com.microsoft.applicationinsights.agent.internal.perfcounter.ProcessMemoryPerformanceCounter;
import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.metrics.ObservableDoubleMeasurement;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.util.ArrayList;
Expand All @@ -32,6 +36,8 @@
public class PerformanceCounterInitializer {

private static final Logger logger = LoggerFactory.getLogger(PerformanceCounterInitializer.class);
private static final String METRIC_NAME_REGEXP = "[a-zA-z0-9_.-/]+";
private static final String INVALID_CHARACTER_REGEXP = "[^a-zA-z0-9_.-/]";

public static void initialize(Configuration configuration) {

Expand Down Expand Up @@ -73,59 +79,119 @@ private static boolean isAgentRunningInSandboxEnvWindows() {
* a map where the key is the Jmx object name and the value is a list of requested attributes. 2.
* Go through all the requested Jmx counters: a. If the object name is not in the map, add it with
* an empty list Else get the list b. Add the attribute to the list. 3. Go through the map For
* every entry (object name and attributes) Build a {@link JmxMetricPerformanceCounter} Register
* the Performance Counter in the {@link PerformanceCounterContainer}
* every entry (object name and attributes) to build a meter per attribute & for each meter
* register a callback to report the metric value.
*/
private static void loadCustomJmxPerfCounters(List<Configuration.JmxMetric> jmxXmlElements) {
try {
HashMap<String, Collection<JmxAttributeData>> data = new HashMap<>();
HashMap<String, Collection<JmxAttributeData>> data = new HashMap<>();

// Build a map of object name to its requested attributes
for (Configuration.JmxMetric jmxElement : jmxXmlElements) {
Collection<JmxAttributeData> collection =
data.computeIfAbsent(jmxElement.objectName, k -> new ArrayList<>());
// Build a map of object name to its requested attributes
for (Configuration.JmxMetric jmxElement : jmxXmlElements) {
Collection<JmxAttributeData> collection =
data.computeIfAbsent(jmxElement.objectName, k -> new ArrayList<>());

if (Strings.isNullOrEmpty(jmxElement.objectName)) {
try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
logger.error("JMX object name is empty, will be ignored");
}
continue;
if (Strings.isNullOrEmpty(jmxElement.objectName)) {
try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
logger.error("JMX object name is empty, will be ignored");
}
continue;
}

if (Strings.isNullOrEmpty(jmxElement.attribute)) {
try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
logger.error("JMX attribute is empty for '{}'", jmxElement.objectName);
}
continue;
if (Strings.isNullOrEmpty(jmxElement.attribute)) {
try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
logger.error("JMX attribute is empty for '{}'", jmxElement.objectName);
}
continue;
}

if (Strings.isNullOrEmpty(jmxElement.name)) {
try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
logger.error("JMX name is empty for '{}', will be ignored", jmxElement.objectName);
}
continue;
if (Strings.isNullOrEmpty(jmxElement.name)) {
try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
logger.error("JMX name is empty for '{}', will be ignored", jmxElement.objectName);
}
continue;
}

collection.add(new JmxAttributeData(jmxElement.name, jmxElement.attribute));
}

createMeterPerAttribute(data);
}

collection.add(new JmxAttributeData(jmxElement.name, jmxElement.attribute));
// Create a meter for each attribute & declare the callback that reports the metric in the meter.
private static void createMeterPerAttribute(
Map<String, Collection<JmxAttributeData>> objectAndAttributesMap) {
for (Map.Entry<String, Collection<JmxAttributeData>> entry :
objectAndAttributesMap.entrySet()) {
String objectName = entry.getKey();

for (JmxAttributeData jmxAttributeData : entry.getValue()) {

String otelMetricName;
if (jmxAttributeData.metricName.matches(METRIC_NAME_REGEXP)) {
otelMetricName = jmxAttributeData.metricName;
} else {
otelMetricName = jmxAttributeData.metricName.replaceAll(INVALID_CHARACTER_REGEXP, "_");
}

GlobalOpenTelemetry.getMeter("com.microsoft.applicationinsights.jmx")
.gaugeBuilder(otelMetricName)
.buildWithCallback(
observableDoubleMeasurement -> {
calculateAndRecordValueForAttribute(
observableDoubleMeasurement, objectName, jmxAttributeData);
});
}
}
}

// Register each entry in the performance container
for (Map.Entry<String, Collection<JmxAttributeData>> entry : data.entrySet()) {
private static void calculateAndRecordValueForAttribute(
ObservableDoubleMeasurement observableDoubleMeasurement,
String objectName,
JmxAttributeData jmxAttributeData) {
try {
List<Object> result =
JmxDataFetcher.fetch(
objectName, jmxAttributeData.attribute); // should return the [val, ...] here

logger.trace(
"Size of the JmxDataFetcher.fetch result: {}, for objectName:{} and metricName:{}",
result.size(),
objectName,
jmxAttributeData.metricName);

boolean ok = true;
double value = 0.0;
for (Object obj : result) {
try {
PerformanceCounterContainer.INSTANCE.register(
new JmxMetricPerformanceCounter(entry.getKey(), entry.getValue()));
} catch (RuntimeException e) {
try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
logger.error(
"Failed to register JMX performance counter: '{}' : '{}'",
entry.getKey(),
e.toString());
if (obj instanceof Boolean) {
value = ((Boolean) obj).booleanValue() ? 1 : 0;
} else {
value += Double.parseDouble(String.valueOf(obj));
}
} catch (RuntimeException e) {
ok = false;
break;
}
}
} catch (RuntimeException e) {
if (ok) {
logger.trace(
"value {} for objectName:{} and metricName{}",
value,
objectName,
jmxAttributeData.metricName);
observableDoubleMeasurement.record(
value,
Attributes.of(
AttributeKey.stringKey("applicationinsights.internal.metric_name"),
jmxAttributeData.metricName));
}
} catch (Exception e) {
try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
logger.error("Failed to register JMX performance counters: '{}'", e.toString());
logger.error(
"Failed to calculate the metric value for objectName {} and metric name {}",
objectName,
jmxAttributeData.metricName);
logger.error("Exception: {}", e.toString());
}
}
}
Expand Down
31 changes: 31 additions & 0 deletions ...ain/java/com/microsoft/applicationinsights/agent/internal/perfcounter/JmxDataFetcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,37 @@ public static Map<String, Collection<Object>> fetch(
return result;
}

/**
* Gets an object name and an attribute to fetch and will return the data.
*
* @param objectName The object name to search.
* @param attribute An attribute that belongs to the object name
* @return A List of values found for that attribute
* @throws Exception In case the object name is not found.
*/
public static List<Object> fetch(String objectName, String attribute) throws Exception {
List<Object> resultForAttribute = new ArrayList<>();

MBeanServer server = ManagementFactory.getPlatformMBeanServer();
Set<ObjectName> objects = server.queryNames(new ObjectName(objectName), null);
logger.trace("Matching object names for pattern {}: {}", objectName, objects.toString());
if (objects.isEmpty()) {
String errorMsg = String.format("Cannot find object name '%s'", objectName);
throw new IllegalArgumentException(errorMsg);
}

try {
resultForAttribute = fetch(server, objects, attribute);
} catch (Exception e) {
try (MDC.MDCCloseable ignored = CUSTOM_JMX_METRIC_ERROR.makeActive()) {
logger.warn("Failed to fetch JMX object '{}' with attribute '{}': ", objectName, attribute);
}
throw e;
}

return resultForAttribute;
}

private static List<Object> fetch(
MBeanServer server, Set<ObjectName> objects, String attributeName)
throws AttributeNotFoundException,
Expand Down
33 changes: 33 additions & 0 deletions ...java/com/microsoft/applicationinsights/agent/internal/perfcounter/JmxDataFetcherTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,13 @@ void testBadAttributeName() throws Exception {

assertThatThrownBy(() -> JmxDataFetcher.fetch("JSDKTests:type=TestStub3", attributes))
.isInstanceOf(Exception.class);

// Test the other fetch method
for (JmxAttributeData attribute : attributes) {
assertThatThrownBy(
() -> JmxDataFetcher.fetch("JSDKTests:type=TestStub3", attribute.attribute))
.isInstanceOf(Exception.class);
}
}

@Test
Expand All @@ -81,6 +88,11 @@ void testBadName() throws Exception {

assertThatThrownBy(() -> JmxDataFetcher.fetch("JSDKTests:type=TestStub", attributes))
.isInstanceOf(IllegalArgumentException.class);

// Test the other fetch method
assertThatThrownBy(
() -> JmxDataFetcher.fetch("JSDKTests:type=TestStub", attributes.get(0).attribute))
.isInstanceOf(IllegalArgumentException.class);
}

@Test
Expand Down Expand Up @@ -119,6 +131,27 @@ private static void performTest(
verify(result, "Int", expectedInt);
verify(result, "Double", expectedDouble);
verify(result, "Long", expectedLong);

// verify the other fetch
for (int i = 0; i < attributes.size(); i++) {
List<Object> singleAttributeResult =
JmxDataFetcher.fetch("JSDKTests:type=TestStub", attributes.get(i).attribute);
assertThat(singleAttributeResult).isNotNull();
assertThat(singleAttributeResult.size()).isEqualTo(1);

double value = 0.0;
for (Object obj : singleAttributeResult) {
value += Double.parseDouble(String.valueOf(obj));
}

if (i == 0) {
assertThat(value).isEqualTo(expectedInt);
} else if (i == 1) {
assertThat(value).isEqualTo(expectedDouble);
} else {
assertThat(value).isEqualTo(expectedLong);
}
}
}

private static void verify(
Expand Down
2 changes: 1 addition & 1 deletion settings.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ hideFromDependabot(":smoke-tests:apps:Diagnostics")
hideFromDependabot(":smoke-tests:apps:Diagnostics:JfrFileReader")
hideFromDependabot(":smoke-tests:apps:DiagnosticExtension:MockExtension")
hideFromDependabot(":smoke-tests:apps:DiagnosticExtension")
hideFromDependabot(":smoke-tests:apps:DotInJmxMetric")
hideFromDependabot(":smoke-tests:apps:JmxMetric")
hideFromDependabot(":smoke-tests:apps:gRPC")
hideFromDependabot(":smoke-tests:apps:HeartBeat")
hideFromDependabot(":smoke-tests:apps:HttpClients")
Expand Down
12 changes: 11 additions & 1 deletion ...est/java/com/microsoft/applicationinsights/smoketest/DetectUnexpectedOtelMetricsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeoutException;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
Expand All @@ -23,14 +25,22 @@ abstract class DetectUnexpectedOtelMetricsTest {

@RegisterExtension static final SmokeTestExtension testing = SmokeTestExtension.create();

private static final List<String> EXPECTED_METRIC_NAMES = new ArrayList<>();

static {
EXPECTED_METRIC_NAMES.add("_OTELRESOURCE_");
EXPECTED_METRIC_NAMES.add("Current Thread Count");
EXPECTED_METRIC_NAMES.add("Loaded Class Count");
}

@Test
@TargetUri("/app")
void testApp() throws Exception {
// verify no unexpected otel metrics, expect an TimeoutException being thrown
assertThatThrownBy(
() ->
testing.mockedIngestion.waitForItemsUnexpectedOtelMetric(
"MetricData", envelope -> true))
"MetricData", envelope -> true, EXPECTED_METRIC_NAMES))
.isInstanceOf(TimeoutException.class);
}

Expand Down
68 changes: 0 additions & 68 deletions ...ic/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/DotInJmxMetricTest.java
View file

This file was deleted.

0 comments on commit 1deaee2

Please sign in to comment.