Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Apply sampling override to Exceptions #3022

Merged
merged 13 commits into from
May 1, 2023
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,24 @@
package com.microsoft.applicationinsights.agent.internal.exporter;

import static com.azure.monitor.opentelemetry.exporter.implementation.utils.AzureMonitorMsgId.EXPORTER_MAPPING_ERROR;
import static com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils.shouldSample;

import com.azure.monitor.opentelemetry.exporter.implementation.LogDataMapper;
import com.azure.monitor.opentelemetry.exporter.implementation.logging.OperationLogger;
import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem;
import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulse;
import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride;
import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler;
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient;
import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.opentelemetry.api.trace.SpanContext;
import io.opentelemetry.sdk.common.CompletableResultCode;
import io.opentelemetry.sdk.logs.data.LogRecordData;
import io.opentelemetry.sdk.logs.export.LogRecordExporter;
import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Consumer;
import javax.annotation.Nullable;
import org.slf4j.Logger;
Expand Down Expand Up @@ -134,23 +132,4 @@ public CompletableResultCode flush() {
public CompletableResultCode shutdown() {
return CompletableResultCode.ofSuccess();
}

@SuppressFBWarnings(
value = "SECPR", // Predictable pseudorandom number generator
justification = "Predictable random is ok for sampling decision")
private static boolean shouldSample(SpanContext spanContext, double percentage) {
if (percentage == 100) {
// optimization, no need to calculate score
return true;
}
if (percentage == 0) {
// optimization, no need to calculate score
return false;
}
if (spanContext.isValid()) {
return AiSampler.shouldRecordAndSample(spanContext.getTraceId(), percentage);
}
// this is a standalone log (not part of a trace), so randomly sample at the given percentage
return ThreadLocalRandom.current().nextDouble() < percentage / 100;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package com.microsoft.applicationinsights.agent.internal.exporter;

import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.opentelemetry.api.trace.SpanContext;
import java.util.concurrent.ThreadLocalRandom;

public final class ExporterUtils {

@SuppressFBWarnings(
value = "SECPR", // Predictable pseudorandom number generator
justification = "Predictable random is ok for sampling decision")
public static boolean shouldSample(SpanContext spanContext, double percentage) {
if (percentage == 100) {
// optimization, no need to calculate score
return true;
}
if (percentage == 0) {
// optimization, no need to calculate score
return false;
}
if (spanContext.isValid()) {
return AiSampler.shouldRecordAndSample(spanContext.getTraceId(), percentage);
}
// this is a standalone log (not part of a trace), so randomly sample at the given percentage
return ThreadLocalRandom.current().nextDouble() < percentage / 100;
}

private ExporterUtils() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.microsoft.applicationinsights.agent.internal.exporter.AgentLogExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.AgentMetricExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.AgentSpanExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils;
import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient;
import com.microsoft.applicationinsights.agent.internal.legacyheaders.AiLegacyHeaderSpanProcessor;
import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithLogProcessor;
Expand All @@ -46,6 +47,7 @@
import com.microsoft.applicationinsights.agent.internal.processors.SpanExporterWithAttributeProcessor;
import com.microsoft.applicationinsights.agent.internal.profiler.ProfilingInitializer;
import com.microsoft.applicationinsights.agent.internal.profiler.triggers.AlertTriggerSpanProcessor;
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
import com.microsoft.applicationinsights.agent.internal.telemetry.MetricFilter;
import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient;
Expand Down Expand Up @@ -491,9 +493,16 @@ private static SdkTracerProviderBuilder configureTracing(

String tracesExporter = otelConfig.getString("otel.traces.exporter");
if ("none".equals(tracesExporter)) { // "none" is the default set in AiConfigCustomizer
List<Configuration.SamplingOverride> exceptionSamplingOverrides =
configuration.preview.sampling.overrides.stream()
.filter(override -> override.telemetryType == SamplingTelemetryType.EXCEPTION)
.collect(Collectors.toList());
SpanExporter spanExporter =
createSpanExporter(
telemetryClient, quickPulse, configuration.preview.captureHttpServer4xxAsError);
telemetryClient,
quickPulse,
configuration.preview.captureHttpServer4xxAsError,
exceptionSamplingOverrides);

spanExporter = wrapSpanExporter(spanExporter, configuration);

Expand All @@ -512,7 +521,8 @@ private static SdkTracerProviderBuilder configureTracing(
private static SpanExporter createSpanExporter(
TelemetryClient telemetryClient,
@Nullable QuickPulse quickPulse,
boolean captureHttpServer4xxAsError) {
boolean captureHttpServer4xxAsError,
List<Configuration.SamplingOverride> exceptionSamplingOverrides) {

SpanDataMapper mapper =
new SpanDataMapper(
Expand All @@ -533,6 +543,13 @@ private static SpanExporter createSpanExporter(
return true;
}
return false;
},
(span, event) -> {
Double samplingPercentage =
new SamplingOverrides(exceptionSamplingOverrides)
heyams marked this conversation as resolved.
Show resolved Hide resolved
.getOverridePercentage(event.getAttributes());
return samplingPercentage != null
&& !ExporterUtils.shouldSample(span.getSpanContext(), samplingPercentage);
});

BatchItemProcessor batchItemProcessor = telemetryClient.getGeneralBatchItemProcessor();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,11 @@ public AzureMonitorExporterBuilder credential(TokenCredential credential) {
*/
public SpanExporter buildTraceExporter() {
SpanDataMapper mapper =
new SpanDataMapper(true, this::populateDefaults, (event, instrumentationName) -> false);
new SpanDataMapper(
true,
this::populateDefaults,
(event, instrumentationName) -> false,
(span, event) -> false);

return new AzureMonitorTraceExporter(mapper, initExporterBuilder());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,17 @@ public final class SpanDataMapper {
private final boolean captureHttpServer4xxAsError;
private final BiConsumer<AbstractTelemetryBuilder, Resource> telemetryInitializer;
private final BiPredicate<EventData, String> eventSuppressor;
private final BiPredicate<SpanData, EventData> shouldSuppress;

public SpanDataMapper(
boolean captureHttpServer4xxAsError,
BiConsumer<AbstractTelemetryBuilder, Resource> telemetryInitializer,
BiPredicate<EventData, String> eventSuppressor) {
BiPredicate<EventData, String> eventSuppressor,
BiPredicate<SpanData, EventData> shouldSuppress) {
this.captureHttpServer4xxAsError = captureHttpServer4xxAsError;
this.telemetryInitializer = telemetryInitializer;
this.eventSuppressor = eventSuppressor;
this.shouldSuppress = shouldSuppress;
}

public TelemetryItem map(SpanData span) {
Expand Down Expand Up @@ -704,7 +707,7 @@ private void exportEvents(
if (!parentSpanContext.isValid() || parentSpanContext.isRemote()) {
// TODO (trask) map OpenTelemetry exception to Application Insights exception better
String stacktrace = event.getAttributes().get(SemanticAttributes.EXCEPTION_STACKTRACE);
if (stacktrace != null) {
if (stacktrace != null && !shouldSuppress.test(span, event)) {
consumer.accept(
createExceptionTelemetryItem(stacktrace, span, operationName, itemCount));
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package com.microsoft.applicationinsights.smoketestapp;

import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

@WebServlet("/trackException")
public class ExceptionSamplingOverridesServlet extends HttpServlet {

protected void doGet(HttpServletRequest request, HttpServletResponse response) {
throw new RuntimeException("this is an expected exception");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package com.microsoft.applicationinsights.smoketest;

import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11_OPENJ9;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_17;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_19;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_20;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8_OPENJ9;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9;
import static org.assertj.core.api.Assertions.assertThat;

import com.microsoft.applicationinsights.smoketest.schemav2.Envelope;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;

@UseAgent("applicationinsights3.json")
abstract class ExceptionSamplingOverridesTest {

@RegisterExtension static final SmokeTestExtension testing = SmokeTestExtension.create();

@Test
@TargetUri(value = "/trackException")
void testExceptionSamplingOverrides() throws Exception {
List<Envelope> rdList = testing.mockedIngestion.waitForItems("RequestData", 1);
Envelope rdEnvelope = rdList.get(0);
assertThat(rdEnvelope.getTags().get("ai.operation.name"))
.isEqualTo("GET /SamplingOverrides/trackException");
assertThat(testing.mockedIngestion.getCountForType("ExceptionData")).isZero();
}

@Environment(TOMCAT_8_JAVA_8)
static class Tomcat8Java8Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_8_OPENJ9)
static class Tomcat8Java8OpenJ9Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_11)
static class Tomcat8Java11Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_11_OPENJ9)
static class Tomcat8Java11OpenJ9Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_17)
static class Tomcat8Java17Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_19)
static class Tomcat8Java19Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_20)
static class Tomcat8Java20Test extends ExceptionSamplingOverridesTest {}

@Environment(WILDFLY_13_JAVA_8)
static class Wildfly13Java8Test extends ExceptionSamplingOverridesTest {}

@Environment(WILDFLY_13_JAVA_8_OPENJ9)
static class Wildfly13Java8OpenJ9Test extends ExceptionSamplingOverridesTest {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
{
"role": {
"name": "testrolename",
"instance": "testroleinstance"
},
"sampling": {
"percentage": 100
},
"preview": {
"sampling": {
"overrides": [
{
"telemetryType": "exception",
"attributes": [
{
"key": "exception.message",
"value": "this is an expected exception",
"matchType": "strict"
},
{
"key": "exception.type",
"value": "java.lang.RuntimeException",
"matchType": "strict"
}
],
"percentage": 0
}
]
}
}
}