trace: add buffer limit

trace/export.go

@@ -73,4 +73,11 @@ type SpanData struct {
 	Status
 	Links           []Link
 	HasRemoteParent bool
+
+	// The count of all span.bufferLimit overflows.
+	// See trace.WithBufferLimit for more info.
+	// TODO: This is not currently exported anywhere (e.g. tracez page).
+	DroppedAnnotations   int
+	DroppedMessageEvents int
+	DroppedLinks         int
 }

trace/trace.go

@@ -46,6 +46,7 @@ type Span struct {
 	endOnce sync.Once
 
 	executionTracerTaskEnd func() // ends the execution tracer span
+	bufferLimit            int    // limits all variable span data (slices)
 }
 
 // IsRecordingEvents returns true if events are being recorded for this span.
@@ -110,6 +111,9 @@ const (
 	SpanKindClient
 )
 
+// DefaultBufferLimit is the default value for trace.StartOptions.BufferLimit.
+const DefaultBufferLimit = 1000
+
 // StartOptions contains options concerning how a span is started.
 type StartOptions struct {
 	// Sampler to consult for this Span. If provided, it is always consulted.
@@ -125,6 +129,11 @@ type StartOptions struct {
 	// SpanKind represents the kind of a span. If none is set,
 	// SpanKindUnspecified is used.
 	SpanKind int
+
+	// BufferLimit makes new spans with a custom buffer limit.
+	// This limits variable span data: Message Events, Links, and Annotations.
+	// The default is 1000 (trace.DefaultBufferLimit) and the minimum is 1.
+	BufferLimit int
 }
 
 // StartOption apply changes to StartOptions.
@@ -145,13 +154,25 @@ func WithSampler(sampler Sampler) StartOption {
 	}
 }
 
+// WithBufferLimit makes new spans with a custom buffer limit.
+// This limits variable trace span data: Message Events, Links, and Annotations.
+// The default is 1000 (trace.DefaultBufferLimit) and the minimum is 1.
+func WithBufferLimit(limit int) StartOption {
+	return func(o *StartOptions) {
+		o.BufferLimit = limit
+		if o.BufferLimit <= 0 {
+			o.BufferLimit = 1
+		}
+	}
+}
+
 // StartSpan starts a new child span of the current span in the context. If
 // there is no span in the context, creates a new trace and span.
 //
 // Returned context contains the newly created span. You can use it to
 // propagate the returned span in process.
 func StartSpan(ctx context.Context, name string, o ...StartOption) (context.Context, *Span) {
-	var opts StartOptions
+	opts := StartOptions{BufferLimit: DefaultBufferLimit}
 	var parent SpanContext
 	if p := FromContext(ctx); p != nil {
 		parent = p.spanContext
@@ -174,7 +195,7 @@ func StartSpan(ctx context.Context, name string, o ...StartOption) (context.Cont
 // Returned context contains the newly created span. You can use it to
 // propagate the returned span in process.
 func StartSpanWithRemoteParent(ctx context.Context, name string, parent SpanContext, o ...StartOption) (context.Context, *Span) {
-	var opts StartOptions
+	opts := StartOptions{BufferLimit: DefaultBufferLimit}
 	for _, op := range o {
 		op(&opts)
 	}
@@ -185,7 +206,7 @@ func StartSpanWithRemoteParent(ctx context.Context, name string, parent SpanCont
 }
 
 func startSpanInternal(name string, hasParent bool, parent SpanContext, remoteParent bool, o StartOptions) *Span {
-	span := &Span{}
+	span := &Span{bufferLimit: o.BufferLimit}
 	span.spanContext = parent
 
 	cfg := config.Load().(*Config)
@@ -340,6 +361,10 @@ func (s *Span) lazyPrintfInternal(attributes []Attribute, format string, a ...in
 		m = make(map[string]interface{})
 		copyAttributes(m, attributes)
 	}
+	if l := len(s.data.Annotations); l > 0 && l >= s.bufferLimit {
+		s.data.Annotations = s.data.Annotations[1:]
+		s.data.DroppedAnnotations++
+	}
 	s.data.Annotations = append(s.data.Annotations, Annotation{
 		Time:       now,
 		Message:    msg,
@@ -356,6 +381,10 @@ func (s *Span) printStringInternal(attributes []Attribute, str string) {
 		a = make(map[string]interface{})
 		copyAttributes(a, attributes)
 	}
+	if l := len(s.data.Annotations); l > 0 && l >= s.bufferLimit {
+		s.data.Annotations = s.data.Annotations[1:]
+		s.data.DroppedAnnotations++
+	}
 	s.data.Annotations = append(s.data.Annotations, Annotation{
 		Time:       now,
 		Message:    str,
@@ -393,6 +422,10 @@ func (s *Span) AddMessageSendEvent(messageID, uncompressedByteSize, compressedBy
 	}
 	now := time.Now()
 	s.mu.Lock()
+	if l := len(s.data.MessageEvents); l > 0 && l >= s.bufferLimit {
+		s.data.MessageEvents = s.data.MessageEvents[1:]
+		s.data.DroppedMessageEvents++
+	}
 	s.data.MessageEvents = append(s.data.MessageEvents, MessageEvent{
 		Time:                 now,
 		EventType:            MessageEventTypeSent,
@@ -415,6 +448,10 @@ func (s *Span) AddMessageReceiveEvent(messageID, uncompressedByteSize, compresse
 	}
 	now := time.Now()
 	s.mu.Lock()
+	if l := len(s.data.MessageEvents); l > 0 && l >= s.bufferLimit {
+		s.data.MessageEvents = s.data.MessageEvents[1:]
+		s.data.DroppedMessageEvents++
+	}
 	s.data.MessageEvents = append(s.data.MessageEvents, MessageEvent{
 		Time:                 now,
 		EventType:            MessageEventTypeRecv,
@@ -431,6 +468,10 @@ func (s *Span) AddLink(l Link) {
 		return
 	}
 	s.mu.Lock()
+	if l := len(s.data.Links); l > 0 && l >= s.bufferLimit {
+		s.data.Links = s.data.Links[1:]
+		s.data.DroppedLinks++
+	}
 	s.data.Links = append(s.data.Links, l)
 	s.mu.Unlock()
 }

trace/trace_test.go

@@ -224,6 +224,10 @@ func TestStartSpanWithRemoteParent(t *testing.T) {
 
 // startSpan returns a context with a new Span that is recording events and will be exported.
 func startSpan(o StartOptions) *Span {
+	bufferLimit := DefaultBufferLimit
+	if o.BufferLimit != 0 {
+		bufferLimit = o.BufferLimit
+	}
 	_, span := StartSpanWithRemoteParent(context.Background(), "span0",
 		SpanContext{
 			TraceID:      tid,
@@ -232,6 +236,7 @@ func startSpan(o StartOptions) *Span {
 		},
 		WithSampler(o.Sampler),
 		WithSpanKind(o.SpanKind),
+		WithBufferLimit(bufferLimit),
 	)
 	return span
 }
@@ -379,7 +384,9 @@ func TestSetSpanAttributes(t *testing.T) {
 }
 
 func TestAnnotations(t *testing.T) {
-	span := startSpan(StartOptions{})
+	span := startSpan(StartOptions{BufferLimit: 2})
+	span.Annotatef([]Attribute{StringAttribute("dropped1", "dropped1")}, "%f", 123.123)
+	span.Annotate([]Attribute{StringAttribute("dropped2", "dropped2")}, "dropped2")
 	span.Annotatef([]Attribute{StringAttribute("key1", "value1")}, "%f", 1.5)
 	span.Annotate([]Attribute{StringAttribute("key2", "value2")}, "Annotate")
 	got, err := endSpan(span)
@@ -405,15 +412,19 @@ func TestAnnotations(t *testing.T) {
 			{Message: "1.500000", Attributes: map[string]interface{}{"key1": "value1"}},
 			{Message: "Annotate", Attributes: map[string]interface{}{"key2": "value2"}},
 		},
-		HasRemoteParent: true,
+		HasRemoteParent:    true,
+		DroppedAnnotations: 2,
 	}
 	if !reflect.DeepEqual(got, want) {
 		t.Errorf("exporting span: got %#v want %#v", got, want)
 	}
 }
 
 func TestMessageEvents(t *testing.T) {
-	span := startSpan(StartOptions{})
+	span := startSpan(StartOptions{BufferLimit: 2})
+	const dropped1, dropped2 = 123, 456
+	span.AddMessageReceiveEvent(dropped1, dropped1, dropped1)
+	span.AddMessageSendEvent(dropped2, dropped2, dropped2)
 	span.AddMessageReceiveEvent(3, 400, 300)
 	span.AddMessageSendEvent(1, 200, 100)
 	got, err := endSpan(span)
@@ -439,7 +450,8 @@ func TestMessageEvents(t *testing.T) {
 			{EventType: 2, MessageID: 0x3, UncompressedByteSize: 0x190, CompressedByteSize: 0x12c},
 			{EventType: 1, MessageID: 0x1, UncompressedByteSize: 0xc8, CompressedByteSize: 0x64},
 		},
-		HasRemoteParent: true,
+		HasRemoteParent:      true,
+		DroppedMessageEvents: 2,
 	}
 	if !reflect.DeepEqual(got, want) {
 		t.Errorf("exporting span: got %#v want %#v", got, want)
@@ -522,13 +534,31 @@ func TestSetSpanStatus(t *testing.T) {
 }
 
 func TestAddLink(t *testing.T) {
-	span := startSpan(StartOptions{})
+	span := startSpan(StartOptions{BufferLimit: 2})
+	span.AddLink(Link{
+		TraceID:    tid,
+		SpanID:     sid,
+		Type:       LinkTypeParent,
+		Attributes: map[string]interface{}{"dropped1": "dropped1"},
+	})
+	span.AddLink(Link{
+		TraceID:    tid,
+		SpanID:     sid,
+		Type:       LinkTypeParent,
+		Attributes: map[string]interface{}{"dropped2": "dropped2"},
+	})
 	span.AddLink(Link{
 		TraceID:    tid,
 		SpanID:     sid,
 		Type:       LinkTypeParent,
 		Attributes: map[string]interface{}{"key5": "value5"},
 	})
+	span.AddLink(Link{
+		TraceID:    tid,
+		SpanID:     sid,
+		Type:       LinkTypeParent,
+		Attributes: map[string]interface{}{"key6": "value6"},
+	})
 	got, err := endSpan(span)
 	if err != nil {
 		t.Fatal(err)
@@ -542,13 +572,22 @@ func TestAddLink(t *testing.T) {
 		},
 		ParentSpanID: sid,
 		Name:         "span0",
-		Links: []Link{{
-			TraceID:    tid,
-			SpanID:     sid,
-			Type:       2,
-			Attributes: map[string]interface{}{"key5": "value5"},
-		}},
+		Links: []Link{
+			{
+				TraceID:    tid,
+				SpanID:     sid,
+				Type:       2,
+				Attributes: map[string]interface{}{"key5": "value5"},
+			},
+			{
+				TraceID:    tid,
+				SpanID:     sid,
+				Type:       2,
+				Attributes: map[string]interface{}{"key6": "value6"},
+			},
+		},
 		HasRemoteParent: true,
+		DroppedLinks:    2,
 	}
 	if !reflect.DeepEqual(got, want) {
 		t.Errorf("exporting span: got %#v want %#v", got, want)