[pkg/stanza] Add send_quiet/drop_quiet options for on_error setti…

…ng (#32220)

**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->

When stanza operators fail to process a message, no matter if they send
the message or drop it an error message is logged. This can be quite
"noisy" when users want to set up a best effort operator which on error
should only send/drop the message without logging an error.
With introducing the `send_quiet` and `drop_quiet` options we provide
the option to set the level of the logged error. `send_quiet` and
`drop_quiet` log the error on debug level.

**Link to tracking Issue:**
#32145

**Testing:** <Describe what testing was performed and which tests were
added.> Added
[unit-tests](https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/32220/files#diff-6a03331b322fcfd255ead096ac5c7c9dd9267a2d6697ea629548f37f4049896aR70)

**Documentation:** <Describe the documentation added.> [Enhanced the
operator's
docs](https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/32220/files#diff-8f730eb2d61d9ca8c5cf863e9e7f3ddfe51984c46c6259dd7258902d6cc10090L2)

---------

Signed-off-by: ChrsMark <chrismarkou92@gmail.com>
Co-authored-by: Daniel Jaglowski <jaglows3@gmail.com>

.chloggen/mute_op_errors.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: filelogreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add `send_quiet` and `drop_quiet` options for `on_error` setting of operators
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [32145]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

pkg/stanza/docs/types/on_error.md

@@ -1,10 +1,21 @@
 # `on_error` parameter
-The `on_error` parameter determines the error handling strategy an operator should use when it fails to process an entry. There are 2 supported values: `drop` and `send`.
+The `on_error` parameter determines the error handling strategy an operator should use when it fails to
+process an entry. There are 4 supported values: `drop`, `drop_quiet`, `send` and `send_quiet`.
 
 Regardless of the method selected, all processing errors will be logged by the operator.
 
 ### `drop`
-In this mode, if an operator fails to process an entry, it will drop the entry altogether. This will stop the entry from being sent further down the pipeline.
+In this mode, if an operator fails to process an entry, it will drop the entry altogether.
+This will stop the entry from being sent further down the pipeline.
+
+### `drop_quiet`
+Same as `drop` with only difference that the failure will be logged in debug level. Useful, when best effort
+operators are defined which might flood the logs with errors.
 
 ### `send`
-In this mode, if an operator fails to process an entry, it will still send the entry down the pipeline. This may result in downstream operators receiving entries in an undesired format.
+In this mode, if an operator fails to process an entry, it will still send the entry down the pipeline.
+This may result in downstream operators receiving entries in an undesired format.
+
+### `send_quiet`
+Same as `send` with only difference that the failure will be logged in debug level. Useful, when best effort
+operators are defined which might flood the logs with errors.

pkg/stanza/operator/helper/transformer.go

@@ -37,11 +37,11 @@ func (c TransformerConfig) Build(logger *zap.SugaredLogger) (TransformerOperator
 	}
 
 	switch c.OnError {
-	case SendOnError, DropOnError:
+	case SendOnError, SendOnErrorQuiet, DropOnError, DropOnErrorQuiet:
 	default:
 		return TransformerOperator{}, errors.NewError(
 			"operator config has an invalid `on_error` field.",
-			"ensure that the `on_error` field is set to either `send` or `drop`.",
+			"ensure that the `on_error` field is set to one of `send`, `send_quiet`, `drop`, `drop_quiet`.",
 			"on_error", c.OnError,
 		)
 	}
@@ -95,8 +95,12 @@ func (t *TransformerOperator) ProcessWith(ctx context.Context, entry *entry.Entr
 
 // HandleEntryError will handle an entry error using the on_error strategy.
 func (t *TransformerOperator) HandleEntryError(ctx context.Context, entry *entry.Entry, err error) error {
-	t.Errorw("Failed to process entry", zap.Any("error", err), zap.Any("action", t.OnError))
-	if t.OnError == SendOnError {
+	if t.OnError == SendOnErrorQuiet || t.OnError == DropOnErrorQuiet {
+		t.Debugw("Failed to process entry", zap.Any("error", err), zap.Any("action", t.OnError))
+	} else {
+		t.Errorw("Failed to process entry", zap.Any("error", err), zap.Any("action", t.OnError))
+	}
+	if t.OnError == SendOnError || t.OnError == SendOnErrorQuiet {
 		t.Write(ctx, entry)
 	}
 	return err
@@ -124,5 +128,11 @@ type TransformFunction = func(*entry.Entry) error
 // SendOnError specifies an on_error mode for sending entries after an error.
 const SendOnError = "send"
 
+// SendOnErrorQuiet specifies an on_error mode for sending entries after an error but without logging on error level
+const SendOnErrorQuiet = "send_quiet"
+
 // DropOnError specifies an on_error mode for dropping entries after an error.
 const DropOnError = "drop"
+
+// DropOnError specifies an on_error mode for dropping entries after an error but without logging on error level
+const DropOnErrorQuiet = "drop_quiet"

pkg/stanza/operator/helper/transformer_test.go

@@ -10,6 +10,9 @@ import (
 
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"go.uber.org/zap/zaptest/observer"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator"
@@ -63,13 +66,17 @@ func TestTransformerDropOnError(t *testing.T) {
 	output := &testutil.Operator{}
 	output.On("ID").Return("test-output")
 	output.On("Process", mock.Anything, mock.Anything).Return(nil)
+
+	obs, logs := observer.New(zap.WarnLevel)
+	logger := zap.New(obs)
+
 	transformer := TransformerOperator{
 		OnError: DropOnError,
 		WriterOperator: WriterOperator{
 			BasicOperator: BasicOperator{
 				OperatorID:    "test-id",
 				OperatorType:  "test-type",
-				SugaredLogger: testutil.Logger(t),
+				SugaredLogger: logger.Sugar(),
 			},
 			OutputOperators: []operator.Operator{output},
 			OutputIDs:       []string{"test-output"},
@@ -84,19 +91,80 @@ func TestTransformerDropOnError(t *testing.T) {
 	err := transformer.ProcessWith(ctx, testEntry, transform)
 	require.Error(t, err)
 	output.AssertNotCalled(t, "Process", mock.Anything, mock.Anything)
+
+	// Test output logs
+	expectedLogs := []observer.LoggedEntry{
+		{
+			Entry: zapcore.Entry{Level: zap.ErrorLevel, Message: "Failed to process entry"},
+			Context: []zapcore.Field{
+				{Key: "error", Type: 26, Interface: fmt.Errorf("Failure")},
+				zap.Any("action", "drop"),
+			},
+		},
+	}
+	require.Equal(t, 1, logs.Len())
+	require.Equalf(t, expectedLogs, logs.AllUntimed(), "expected logs do not match")
+}
+
+func TestTransformerDropOnErrorQuiet(t *testing.T) {
+	output := &testutil.Operator{}
+	output.On("ID").Return("test-output")
+	output.On("Process", mock.Anything, mock.Anything).Return(nil)
+
+	obs, logs := observer.New(zap.DebugLevel)
+	logger := zap.New(obs)
+
+	transformer := TransformerOperator{
+		OnError: DropOnErrorQuiet,
+		WriterOperator: WriterOperator{
+			BasicOperator: BasicOperator{
+				OperatorID:    "test-id",
+				OperatorType:  "test-type",
+				SugaredLogger: logger.Sugar(),
+			},
+			OutputOperators: []operator.Operator{output},
+			OutputIDs:       []string{"test-output"},
+		},
+	}
+	ctx := context.Background()
+	testEntry := entry.New()
+	transform := func(_ *entry.Entry) error {
+		return fmt.Errorf("Failure")
+	}
+
+	err := transformer.ProcessWith(ctx, testEntry, transform)
+	require.Error(t, err)
+	output.AssertNotCalled(t, "Process", mock.Anything, mock.Anything)
+
+	// Test output logs
+	expectedLogs := []observer.LoggedEntry{
+		{
+			Entry: zapcore.Entry{Level: zap.DebugLevel, Message: "Failed to process entry"},
+			Context: []zapcore.Field{
+				{Key: "error", Type: 26, Interface: fmt.Errorf("Failure")},
+				zap.Any("action", "drop_quiet"),
+			},
+		},
+	}
+	require.Equal(t, 1, logs.Len())
+	require.Equalf(t, expectedLogs, logs.AllUntimed(), "expected logs do not match")
 }
 
 func TestTransformerSendOnError(t *testing.T) {
 	output := &testutil.Operator{}
 	output.On("ID").Return("test-output")
 	output.On("Process", mock.Anything, mock.Anything).Return(nil)
+
+	obs, logs := observer.New(zap.WarnLevel)
+	logger := zap.New(obs)
+
 	transformer := TransformerOperator{
 		OnError: SendOnError,
 		WriterOperator: WriterOperator{
 			BasicOperator: BasicOperator{
 				OperatorID:    "test-id",
 				OperatorType:  "test-type",
-				SugaredLogger: testutil.Logger(t),
+				SugaredLogger: logger.Sugar(),
 			},
 			OutputOperators: []operator.Operator{output},
 			OutputIDs:       []string{"test-output"},
@@ -111,6 +179,63 @@ func TestTransformerSendOnError(t *testing.T) {
 	err := transformer.ProcessWith(ctx, testEntry, transform)
 	require.Error(t, err)
 	output.AssertCalled(t, "Process", mock.Anything, mock.Anything)
+
+	// Test output logs
+	expectedLogs := []observer.LoggedEntry{
+		{
+			Entry: zapcore.Entry{Level: zap.ErrorLevel, Message: "Failed to process entry"},
+			Context: []zapcore.Field{
+				{Key: "error", Type: 26, Interface: fmt.Errorf("Failure")},
+				zap.Any("action", "send"),
+			},
+		},
+	}
+	require.Equal(t, 1, logs.Len())
+	require.Equalf(t, expectedLogs, logs.AllUntimed(), "expected logs do not match")
+}
+
+func TestTransformerSendOnErrorQuiet(t *testing.T) {
+	output := &testutil.Operator{}
+	output.On("ID").Return("test-output")
+	output.On("Process", mock.Anything, mock.Anything).Return(nil)
+
+	obs, logs := observer.New(zap.DebugLevel)
+	logger := zap.New(obs)
+
+	transformer := TransformerOperator{
+		OnError: SendOnErrorQuiet,
+		WriterOperator: WriterOperator{
+			BasicOperator: BasicOperator{
+				OperatorID:    "test-id",
+				OperatorType:  "test-type",
+				SugaredLogger: logger.Sugar(),
+			},
+			OutputOperators: []operator.Operator{output},
+			OutputIDs:       []string{"test-output"},
+		},
+	}
+	ctx := context.Background()
+	testEntry := entry.New()
+	transform := func(_ *entry.Entry) error {
+		return fmt.Errorf("Failure")
+	}
+
+	err := transformer.ProcessWith(ctx, testEntry, transform)
+	require.Error(t, err)
+	output.AssertCalled(t, "Process", mock.Anything, mock.Anything)
+
+	// Test output logs
+	expectedLogs := []observer.LoggedEntry{
+		{
+			Entry: zapcore.Entry{Level: zap.DebugLevel, Message: "Failed to process entry"},
+			Context: []zapcore.Field{
+				{Key: "error", Type: 26, Interface: fmt.Errorf("Failure")},
+				zap.Any("action", "send_quiet"),
+			},
+		},
+	}
+	require.Equal(t, 1, logs.Len())
+	require.Equalf(t, expectedLogs, logs.AllUntimed(), "expected logs do not match")
 }
 
 func TestTransformerProcessWithValid(t *testing.T) {