Apply sampling override to Exceptions

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java

@@ -4,26 +4,24 @@
 package com.microsoft.applicationinsights.agent.internal.exporter;
 
 import static com.azure.monitor.opentelemetry.exporter.implementation.utils.AzureMonitorMsgId.EXPORTER_MAPPING_ERROR;
+import static com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils.shouldSample;
 
 import com.azure.monitor.opentelemetry.exporter.implementation.LogDataMapper;
 import com.azure.monitor.opentelemetry.exporter.implementation.logging.OperationLogger;
 import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem;
 import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulse;
 import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride;
-import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler;
 import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
 import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
 import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient;
 import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers;
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import io.opentelemetry.api.trace.SpanContext;
 import io.opentelemetry.sdk.common.CompletableResultCode;
 import io.opentelemetry.sdk.logs.data.LogRecordData;
 import io.opentelemetry.sdk.logs.export.LogRecordExporter;
 import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
 import java.util.Collection;
 import java.util.List;
-import java.util.concurrent.ThreadLocalRandom;
 import java.util.function.Consumer;
 import javax.annotation.Nullable;
 import org.slf4j.Logger;
@@ -134,23 +132,4 @@ public CompletableResultCode flush() {
   public CompletableResultCode shutdown() {
     return CompletableResultCode.ofSuccess();
   }
-
-  @SuppressFBWarnings(
-      value = "SECPR", // Predictable pseudorandom number generator
-      justification = "Predictable random is ok for sampling decision")
-  private static boolean shouldSample(SpanContext spanContext, double percentage) {
-    if (percentage == 100) {
-      // optimization, no need to calculate score
-      return true;
-    }
-    if (percentage == 0) {
-      // optimization, no need to calculate score
-      return false;
-    }
-    if (spanContext.isValid()) {
-      return AiSampler.shouldRecordAndSample(spanContext.getTraceId(), percentage);
-    }
-    // this is a standalone log (not part of a trace), so randomly sample at the given percentage
-    return ThreadLocalRandom.current().nextDouble() < percentage / 100;
-  }
 }

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentSpanExporter.java

@@ -10,13 +10,17 @@
 import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem;
 import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulse;
 import com.azure.monitor.opentelemetry.exporter.implementation.utils.Strings;
+import com.microsoft.applicationinsights.agent.internal.configuration.Configuration;
+import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
 import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
 import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient;
 import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers;
 import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.trace.data.EventData;
 import io.opentelemetry.sdk.trace.data.SpanData;
 import io.opentelemetry.sdk.trace.export.SpanExporter;
 import java.util.Collection;
+import java.util.List;
 import java.util.function.Consumer;
 import javax.annotation.Nullable;
 import org.slf4j.Logger;
@@ -31,11 +35,13 @@ public final class AgentSpanExporter implements SpanExporter {
 
   private final SpanDataMapper mapper;
   private final Consumer<TelemetryItem> telemetryItemConsumer;
+  private final SamplingOverrides exceptionSamplingOverrides;
 
   public AgentSpanExporter(
       SpanDataMapper mapper,
       @Nullable QuickPulse quickPulse,
-      BatchItemProcessor batchItemProcessor) {
+      BatchItemProcessor batchItemProcessor,
+      List<Configuration.SamplingOverride> samplingOverrides) {
     this.mapper = mapper;
     telemetryItemConsumer =
         telemetryItem -> {
@@ -47,6 +53,7 @@ public AgentSpanExporter(
               .forEach(consumer -> consumer.accept(telemetryItem));
           batchItemProcessor.trackAsync(telemetryItem);
         };
+    exceptionSamplingOverrides = new SamplingOverrides(samplingOverrides);
   }
 
   @Override
@@ -59,7 +66,7 @@ public CompletableResultCode export(Collection<SpanData> spans) {
     for (SpanData span : spans) {
       logger.debug("exporting span: {}", span);
       try {
-        mapper.map(span, telemetryItemConsumer);
+        mapper.map(span, telemetryItemConsumer, this::shouldSuppress);
         exportingSpanLogger.recordSuccess();
       } catch (Throwable t) {
         exportingSpanLogger.recordFailure(t.getMessage(), t, EXPORTER_MAPPING_ERROR);
@@ -78,4 +85,11 @@ public CompletableResultCode flush() {
   public CompletableResultCode shutdown() {
     return CompletableResultCode.ofSuccess();
   }
+
+  boolean shouldSuppress(SpanData span, EventData event) {
+    Double samplingPercentage =
+        exceptionSamplingOverrides.getOverridePercentage(event.getAttributes());
+    return samplingPercentage != null
+        && !ExporterUtils.shouldSample(span.getSpanContext(), samplingPercentage);
+  }
 }

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/ExporterUtils.java

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.microsoft.applicationinsights.agent.internal.exporter;
+
+import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import io.opentelemetry.api.trace.SpanContext;
+import java.util.concurrent.ThreadLocalRandom;
+
+final class ExporterUtils {
+
+  @SuppressFBWarnings(
+      value = "SECPR", // Predictable pseudorandom number generator
+      justification = "Predictable random is ok for sampling decision")
+  static boolean shouldSample(SpanContext spanContext, double percentage) {
+    if (percentage == 100) {
+      // optimization, no need to calculate score
+      return true;
+    }
+    if (percentage == 0) {
+      // optimization, no need to calculate score
+      return false;
+    }
+    if (spanContext.isValid()) {
+      return AiSampler.shouldRecordAndSample(spanContext.getTraceId(), percentage);
+    }
+    // this is a standalone log (not part of a trace), so randomly sample at the given percentage
+    return ThreadLocalRandom.current().nextDouble() < percentage / 100;
+  }
+
+  private ExporterUtils() {}
+}

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java

@@ -334,9 +334,16 @@ private static SdkTracerProviderBuilder configureTracing(
 
     String tracesExporter = otelConfig.getString("otel.traces.exporter");
     if ("none".equals(tracesExporter)) { // "none" is the default set in AiConfigCustomizer
+      List<Configuration.SamplingOverride> exceptionSamplingOverrides =
+          configuration.preview.sampling.overrides.stream()
+              .filter(override -> override.telemetryType == SamplingTelemetryType.EXCEPTION)
+              .collect(Collectors.toList());
       SpanExporter spanExporter =
           createSpanExporter(
-              telemetryClient, quickPulse, configuration.preview.captureHttpServer4xxAsError);
+              telemetryClient,
+              quickPulse,
+              configuration.preview.captureHttpServer4xxAsError,
+              exceptionSamplingOverrides);
 
       spanExporter = wrapSpanExporter(spanExporter, configuration);
 
@@ -355,7 +362,8 @@ private static SdkTracerProviderBuilder configureTracing(
   private static SpanExporter createSpanExporter(
       TelemetryClient telemetryClient,
       @Nullable QuickPulse quickPulse,
-      boolean captureHttpServer4xxAsError) {
+      boolean captureHttpServer4xxAsError,
+      List<Configuration.SamplingOverride> exceptionSamplingOverrides) {
 
     SpanDataMapper mapper =
         new SpanDataMapper(
@@ -381,7 +389,7 @@ private static SpanExporter createSpanExporter(
     BatchItemProcessor batchItemProcessor = telemetryClient.getGeneralBatchItemProcessor();
 
     return new StatsbeatSpanExporter(
-        new AgentSpanExporter(mapper, quickPulse, batchItemProcessor),
+        new AgentSpanExporter(mapper, quickPulse, batchItemProcessor, exceptionSamplingOverrides),
         telemetryClient.getStatsbeatModule());
   }
 

agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/AzureMonitorTraceExporter.java

@@ -48,7 +48,7 @@ public CompletableResultCode export(Collection<SpanData> spans) {
     for (SpanData span : spans) {
       LOGGER.verbose("exporting span: {}", span);
       try {
-        mapper.map(span, telemetryItems::add);
+        mapper.map(span, telemetryItems::add, (s, event) -> false);
         OPERATION_LOGGER.recordSuccess();
       } catch (Throwable t) {
         OPERATION_LOGGER.recordFailure(t.getMessage(), t, EXPORTER_MAPPING_ERROR);

agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/SpanDataMapper.java

@@ -120,15 +120,19 @@ public TelemetryItem map(SpanData span) {
     return map(span, itemCount);
   }
 
-  public void map(SpanData span, Consumer<TelemetryItem> consumer) {
+  public void map(
+      SpanData span,
+      Consumer<TelemetryItem> consumer,
+      BiPredicate<SpanData, EventData> shouldSuppress) {
     long itemCount = getItemCount(span);
     TelemetryItem telemetryItem = map(span, itemCount);
     consumer.accept(telemetryItem);
     exportEvents(
         span,
         telemetryItem.getTags().get(ContextTagKeys.AI_OPERATION_NAME.toString()),
         itemCount,
-        consumer);
+        consumer,
+        shouldSuppress);
   }
 
   public TelemetryItem map(SpanData span, long itemCount) {
@@ -689,7 +693,8 @@ private void exportEvents(
       SpanData span,
       @Nullable String operationName,
       long itemCount,
-      Consumer<TelemetryItem> consumer) {
+      Consumer<TelemetryItem> consumer,
+      BiPredicate<SpanData, EventData> shouldSuppress) {
     for (EventData event : span.getEvents()) {
       String instrumentationScopeName = span.getInstrumentationScopeInfo().getName();
       if (eventSuppressor.test(event, instrumentationScopeName)) {
@@ -704,10 +709,10 @@ private void exportEvents(
         if (!parentSpanContext.isValid() || parentSpanContext.isRemote()) {
           // TODO (trask) map OpenTelemetry exception to Application Insights exception better
           String stacktrace = event.getAttributes().get(SemanticAttributes.EXCEPTION_STACKTRACE);
-          if (stacktrace != null) {
-            consumer.accept(
-                createExceptionTelemetryItem(stacktrace, span, operationName, itemCount));
+          if (stacktrace != null && shouldSuppress.test(span, event)) {
+            continue;
           }
+          consumer.accept(createExceptionTelemetryItem(stacktrace, span, operationName, itemCount));
         }
         return;
       }

smoke-tests/apps/SamplingOverrides/src/main/java/com/microsoft/applicationinsights/smoketestapp/ExceptionSamplingOverridesServlet.java

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.microsoft.applicationinsights.smoketestapp;
+
+import javax.servlet.annotation.WebServlet;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+@WebServlet("/trackException")
+public class ExceptionSamplingOverridesServlet extends HttpServlet {
+
+  protected void doGet(HttpServletRequest request, HttpServletResponse response) {
+    throw new RuntimeException("this is an expected exception");
+  }
+}

smoke-tests/apps/SamplingOverrides/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/ExceptionSamplingOverridesTest.java

@@ -0,0 +1,63 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.microsoft.applicationinsights.smoketest;
+
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11_OPENJ9;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_17;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_19;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_20;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8_OPENJ9;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.microsoft.applicationinsights.smoketest.schemav2.Envelope;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@UseAgent("applicationinsights3.json")
+abstract class ExceptionSamplingOverridesTest {
+
+  @RegisterExtension static final SmokeTestExtension testing = SmokeTestExtension.create();
+
+  @Test
+  @TargetUri(value = "/trackException")
+  void testExceptionSamplingOverrides() throws Exception {
+    List<Envelope> rdList = testing.mockedIngestion.waitForItems("RequestData", 1);
+    Envelope rdEnvelope = rdList.get(0);
+    assertThat(rdEnvelope.getTags().get("ai.operation.name"))
+        .isEqualTo("GET /SamplingOverrides/trackException");
+    assertThat(testing.mockedIngestion.getCountForType("ExceptionData")).isZero();
+  }
+
+  @Environment(TOMCAT_8_JAVA_8)
+  static class Tomcat8Java8Test extends ExceptionSamplingOverridesTest {}
+
+  @Environment(TOMCAT_8_JAVA_8_OPENJ9)
+  static class Tomcat8Java8OpenJ9Test extends ExceptionSamplingOverridesTest {}
+
+  @Environment(TOMCAT_8_JAVA_11)
+  static class Tomcat8Java11Test extends ExceptionSamplingOverridesTest {}
+
+  @Environment(TOMCAT_8_JAVA_11_OPENJ9)
+  static class Tomcat8Java11OpenJ9Test extends ExceptionSamplingOverridesTest {}
+
+  @Environment(TOMCAT_8_JAVA_17)
+  static class Tomcat8Java17Test extends ExceptionSamplingOverridesTest {}
+
+  @Environment(TOMCAT_8_JAVA_19)
+  static class Tomcat8Java19Test extends ExceptionSamplingOverridesTest {}
+
+  @Environment(TOMCAT_8_JAVA_20)
+  static class Tomcat8Java20Test extends ExceptionSamplingOverridesTest {}
+
+  @Environment(WILDFLY_13_JAVA_8)
+  static class Wildfly13Java8Test extends ExceptionSamplingOverridesTest {}
+
+  @Environment(WILDFLY_13_JAVA_8_OPENJ9)
+  static class Wildfly13Java8OpenJ9Test extends ExceptionSamplingOverridesTest {}
+}

smoke-tests/apps/SamplingOverrides/src/smokeTest/resources/applicationinsights3.json

@@ -0,0 +1,31 @@
+{
+  "role": {
+    "name": "testrolename",
+    "instance": "testroleinstance"
+  },
+  "sampling": {
+    "percentage": 100
+  },
+  "preview": {
+    "sampling": {
+      "overrides": [
+        {
+          "telemetryType": "exception",
+          "attributes": [
+            {
+              "key": "exception.message",
+              "value": "this is an expected exception",
+              "matchType": "strict"
+            },
+            {
+              "key": "exception.type",
+              "value": "java.lang.RuntimeException",
+              "matchType": "strict"
+            }
+          ],
+          "percentage": 0
+        }
+      ]
+    }
+  }
+}