Skip to content

Commit

Permalink
Apply sampling override to Exceptions (#3022)
Browse files Browse the repository at this point in the history
Fix #2941 
Replace #2970
  • Loading branch information
heyams committed May 1, 2023
1 parent b652112 commit 93f6ccb
Show file tree
Hide file tree
Showing 8 changed files with 173 additions and 27 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,24 @@
package com.microsoft.applicationinsights.agent.internal.exporter;

import static com.azure.monitor.opentelemetry.exporter.implementation.utils.AzureMonitorMsgId.EXPORTER_MAPPING_ERROR;
import static com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils.shouldSample;

import com.azure.monitor.opentelemetry.exporter.implementation.LogDataMapper;
import com.azure.monitor.opentelemetry.exporter.implementation.logging.OperationLogger;
import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem;
import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulse;
import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride;
import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler;
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient;
import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.opentelemetry.api.trace.SpanContext;
import io.opentelemetry.sdk.common.CompletableResultCode;
import io.opentelemetry.sdk.logs.data.LogRecordData;
import io.opentelemetry.sdk.logs.export.LogRecordExporter;
import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Consumer;
import javax.annotation.Nullable;
import org.slf4j.Logger;
Expand Down Expand Up @@ -134,23 +132,4 @@ public CompletableResultCode flush() {
public CompletableResultCode shutdown() {
return CompletableResultCode.ofSuccess();
}

@SuppressFBWarnings(
value = "SECPR", // Predictable pseudorandom number generator
justification = "Predictable random is ok for sampling decision")
private static boolean shouldSample(SpanContext spanContext, double percentage) {
if (percentage == 100) {
// optimization, no need to calculate score
return true;
}
if (percentage == 0) {
// optimization, no need to calculate score
return false;
}
if (spanContext.isValid()) {
return AiSampler.shouldRecordAndSample(spanContext.getTraceId(), percentage);
}
// this is a standalone log (not part of a trace), so randomly sample at the given percentage
return ThreadLocalRandom.current().nextDouble() < percentage / 100;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package com.microsoft.applicationinsights.agent.internal.exporter;

import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.opentelemetry.api.trace.SpanContext;
import java.util.concurrent.ThreadLocalRandom;

public final class ExporterUtils {

@SuppressFBWarnings(
value = "SECPR", // Predictable pseudorandom number generator
justification = "Predictable random is ok for sampling decision")
public static boolean shouldSample(SpanContext spanContext, double percentage) {
if (percentage == 100) {
// optimization, no need to calculate score
return true;
}
if (percentage == 0) {
// optimization, no need to calculate score
return false;
}
if (spanContext.isValid()) {
return AiSampler.shouldRecordAndSample(spanContext.getTraceId(), percentage);
}
// this is a standalone log (not part of a trace), so randomly sample at the given percentage
return ThreadLocalRandom.current().nextDouble() < percentage / 100;
}

private ExporterUtils() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.microsoft.applicationinsights.agent.internal.exporter.AgentLogExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.AgentMetricExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.AgentSpanExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils;
import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient;
import com.microsoft.applicationinsights.agent.internal.legacyheaders.AiLegacyHeaderSpanProcessor;
import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithLogProcessor;
Expand All @@ -46,6 +47,7 @@
import com.microsoft.applicationinsights.agent.internal.processors.SpanExporterWithAttributeProcessor;
import com.microsoft.applicationinsights.agent.internal.profiler.ProfilingInitializer;
import com.microsoft.applicationinsights.agent.internal.profiler.triggers.AlertTriggerSpanProcessor;
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
import com.microsoft.applicationinsights.agent.internal.telemetry.MetricFilter;
import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient;
Expand Down Expand Up @@ -491,9 +493,16 @@ private static SdkTracerProviderBuilder configureTracing(

String tracesExporter = otelConfig.getString("otel.traces.exporter");
if ("none".equals(tracesExporter)) { // "none" is the default set in AiConfigCustomizer
List<Configuration.SamplingOverride> exceptionSamplingOverrides =
configuration.preview.sampling.overrides.stream()
.filter(override -> override.telemetryType == SamplingTelemetryType.EXCEPTION)
.collect(Collectors.toList());
SpanExporter spanExporter =
createSpanExporter(
telemetryClient, quickPulse, configuration.preview.captureHttpServer4xxAsError);
telemetryClient,
quickPulse,
configuration.preview.captureHttpServer4xxAsError,
new SamplingOverrides(exceptionSamplingOverrides));

spanExporter = wrapSpanExporter(spanExporter, configuration);

Expand All @@ -512,7 +521,8 @@ private static SdkTracerProviderBuilder configureTracing(
private static SpanExporter createSpanExporter(
TelemetryClient telemetryClient,
@Nullable QuickPulse quickPulse,
boolean captureHttpServer4xxAsError) {
boolean captureHttpServer4xxAsError,
SamplingOverrides exceptionSamplingOverrides) {

SpanDataMapper mapper =
new SpanDataMapper(
Expand All @@ -533,6 +543,12 @@ private static SpanExporter createSpanExporter(
return true;
}
return false;
},
(span, event) -> {
Double samplingPercentage =
exceptionSamplingOverrides.getOverridePercentage(event.getAttributes());
return samplingPercentage != null
&& !ExporterUtils.shouldSample(span.getSpanContext(), samplingPercentage);
});

BatchItemProcessor batchItemProcessor = telemetryClient.getGeneralBatchItemProcessor();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,11 @@ public AzureMonitorExporterBuilder credential(TokenCredential credential) {
*/
public SpanExporter buildTraceExporter() {
SpanDataMapper mapper =
new SpanDataMapper(true, this::populateDefaults, (event, instrumentationName) -> false);
new SpanDataMapper(
true,
this::populateDefaults,
(event, instrumentationName) -> false,
(span, event) -> false);

return new AzureMonitorTraceExporter(mapper, initExporterBuilder());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,17 @@ public final class SpanDataMapper {
private final boolean captureHttpServer4xxAsError;
private final BiConsumer<AbstractTelemetryBuilder, Resource> telemetryInitializer;
private final BiPredicate<EventData, String> eventSuppressor;
private final BiPredicate<SpanData, EventData> shouldSuppress;

public SpanDataMapper(
boolean captureHttpServer4xxAsError,
BiConsumer<AbstractTelemetryBuilder, Resource> telemetryInitializer,
BiPredicate<EventData, String> eventSuppressor) {
BiPredicate<EventData, String> eventSuppressor,
BiPredicate<SpanData, EventData> shouldSuppress) {
this.captureHttpServer4xxAsError = captureHttpServer4xxAsError;
this.telemetryInitializer = telemetryInitializer;
this.eventSuppressor = eventSuppressor;
this.shouldSuppress = shouldSuppress;
}

public TelemetryItem map(SpanData span) {
Expand Down Expand Up @@ -704,7 +707,7 @@ private void exportEvents(
if (!parentSpanContext.isValid() || parentSpanContext.isRemote()) {
// TODO (trask) map OpenTelemetry exception to Application Insights exception better
String stacktrace = event.getAttributes().get(SemanticAttributes.EXCEPTION_STACKTRACE);
if (stacktrace != null) {
if (stacktrace != null && !shouldSuppress.test(span, event)) {
consumer.accept(
createExceptionTelemetryItem(stacktrace, span, operationName, itemCount));
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package com.microsoft.applicationinsights.smoketestapp;

import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

@WebServlet("/trackException")
public class ExceptionSamplingOverridesServlet extends HttpServlet {

protected void doGet(HttpServletRequest request, HttpServletResponse response) {
throw new RuntimeException("this is an expected exception");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package com.microsoft.applicationinsights.smoketest;

import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11_OPENJ9;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_17;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_19;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_20;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8_OPENJ9;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8;
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9;
import static org.assertj.core.api.Assertions.assertThat;

import com.microsoft.applicationinsights.smoketest.schemav2.Envelope;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;

@UseAgent("applicationinsights3.json")
abstract class ExceptionSamplingOverridesTest {

@RegisterExtension static final SmokeTestExtension testing = SmokeTestExtension.create();

@Test
@TargetUri(value = "/trackException")
void testExceptionSamplingOverrides() throws Exception {
List<Envelope> rdList = testing.mockedIngestion.waitForItems("RequestData", 1);
Envelope rdEnvelope = rdList.get(0);
assertThat(rdEnvelope.getTags().get("ai.operation.name"))
.isEqualTo("GET /SamplingOverrides/trackException");
assertThat(testing.mockedIngestion.getCountForType("ExceptionData")).isZero();
}

@Environment(TOMCAT_8_JAVA_8)
static class Tomcat8Java8Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_8_OPENJ9)
static class Tomcat8Java8OpenJ9Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_11)
static class Tomcat8Java11Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_11_OPENJ9)
static class Tomcat8Java11OpenJ9Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_17)
static class Tomcat8Java17Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_19)
static class Tomcat8Java19Test extends ExceptionSamplingOverridesTest {}

@Environment(TOMCAT_8_JAVA_20)
static class Tomcat8Java20Test extends ExceptionSamplingOverridesTest {}

@Environment(WILDFLY_13_JAVA_8)
static class Wildfly13Java8Test extends ExceptionSamplingOverridesTest {}

@Environment(WILDFLY_13_JAVA_8_OPENJ9)
static class Wildfly13Java8OpenJ9Test extends ExceptionSamplingOverridesTest {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
{
"role": {
"name": "testrolename",
"instance": "testroleinstance"
},
"sampling": {
"percentage": 100
},
"preview": {
"sampling": {
"overrides": [
{
"telemetryType": "exception",
"attributes": [
{
"key": "exception.message",
"value": "this is an expected exception",
"matchType": "strict"
},
{
"key": "exception.type",
"value": "java.lang.RuntimeException",
"matchType": "strict"
}
],
"percentage": 0
}
]
}
}
}

0 comments on commit 93f6ccb

Please sign in to comment.