Skip to content

Commit

Permalink
core: Migrate to new OpenCensus method & status tags (#5996)
Browse files Browse the repository at this point in the history 
Fixes #5593 and supersedes #5601

Now that census-instrumentation/opencensus-java#1854 has been merged & released as 0.21.0. We can start using the method & status tags.

Background:
Opencensus introduced new tags for status and method (census-instrumentation/opencensus-java#1115). The old views that used those tags were deprecated and new views were created that used the new tags. However grpc-java wasn't updated to use the new tags due to concern of breaking existing metrics. This resulted in the old views being deprecated while the new views were broken (goomics #50).

census-instrumentation/opencensus-java#1854 added a compatibility layer to opencensus that would remap new tags to old tags for old views. This should unblock grpc to switching to the new tags while allowing old views to still be populated. That commit was released as part of opencensus 0.21, which grpc currently uses
  • Loading branch information
@igorbernstein2 @zhangkun83
igorbernstein2 authored and zhangkun83 committed Aug 1, 2019
1 parent b8933fa commit 271cbff
Show file tree
Hide file tree
Showing 5 changed files with 192 additions and 77 deletions.
3 changes: 2 additions & 1 deletion core/build.gradle
View file
Expand Up @@ -26,7 +26,8 @@ dependencies {
project(':grpc-api').sourceSets.test.output,
project(':grpc-testing'),
project(':grpc-grpclb'),
libraries.guava_testlib
libraries.guava_testlib,
libraries.opencensus_impl

signature "org.codehaus.mojo.signature:java17:1.0@signature"
signature "net.sf.androidscents.signature:android-api-level-14:4.0_r4@signature"
Expand Down
8 changes: 4 additions & 4 deletions core/src/main/java/io/grpc/internal/CensusStatsModule.java
View file
Expand Up @@ -356,7 +356,7 @@ static final class ClientCallTracer extends ClientStreamTracer.Factory {
this.parentCtx = checkNotNull(parentCtx);
TagValue methodTag = TagValue.create(fullMethodName);
this.startCtx = module.tagger.toBuilder(parentCtx)
.putLocal(DeprecatedCensusConstants.RPC_METHOD, methodTag)
.putLocal(RpcMeasureConstants.GRPC_CLIENT_METHOD, methodTag)
.build();
this.stopwatch = module.stopwatchSupplier.get().start();
if (module.recordStartedRpcs) {
Expand Down Expand Up @@ -442,7 +442,7 @@ void callEnded(Status status) {
module
.tagger
.toBuilder(startCtx)
.putLocal(DeprecatedCensusConstants.RPC_STATUS, statusTag)
.putLocal(RpcMeasureConstants.GRPC_CLIENT_STATUS, statusTag)
.build());
}
}
Expand Down Expand Up @@ -647,7 +647,7 @@ public void streamClosed(Status status) {
module
.tagger
.toBuilder(parentCtx)
.putLocal(DeprecatedCensusConstants.RPC_STATUS, statusTag)
.putLocal(RpcMeasureConstants.GRPC_SERVER_STATUS, statusTag)
.build());
}

Expand All @@ -672,7 +672,7 @@ public ServerStreamTracer newServerStreamTracer(String fullMethodName, Metadata
parentCtx =
tagger
.toBuilder(parentCtx)
.putLocal(DeprecatedCensusConstants.RPC_METHOD, methodTag)
.putLocal(RpcMeasureConstants.GRPC_SERVER_METHOD, methodTag)
.build();
return new ServerTracer(CensusStatsModule.this, parentCtx);
}
Expand Down
5 changes: 0 additions & 5 deletions core/src/main/java/io/grpc/internal/DeprecatedCensusConstants.java
View file
Expand Up @@ -20,16 +20,11 @@
import io.opencensus.contrib.grpc.metrics.RpcMeasureConstants;
import io.opencensus.stats.Measure.MeasureDouble;
import io.opencensus.stats.Measure.MeasureLong;
import io.opencensus.tags.TagKey;

/** Holder class for the deprecated OpenCensus constants. */
@SuppressWarnings("deprecation")
@VisibleForTesting
public final class DeprecatedCensusConstants {

public static final TagKey RPC_STATUS = RpcMeasureConstants.RPC_STATUS;
public static final TagKey RPC_METHOD = RpcMeasureConstants.RPC_METHOD;

public static final MeasureLong RPC_CLIENT_ERROR_COUNT =
RpcMeasureConstants.RPC_CLIENT_ERROR_COUNT;
public static final MeasureDouble RPC_CLIENT_REQUEST_BYTES =
Expand Down
186 changes: 146 additions & 40 deletions core/src/test/java/io/grpc/internal/CensusModulesTest.java
View file
Expand Up @@ -17,6 +17,7 @@
package io.grpc.internal;

import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
Expand All @@ -40,6 +41,7 @@
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;

import com.google.common.collect.ImmutableList;
import io.grpc.Attributes;
import io.grpc.CallOptions;
import io.grpc.Channel;
Expand All @@ -61,8 +63,19 @@
import io.grpc.internal.testing.StatsTestUtils.FakeTagger;
import io.grpc.internal.testing.StatsTestUtils.MockableSpan;
import io.grpc.testing.GrpcServerRule;
import io.opencensus.common.Function;
import io.opencensus.common.Functions;
import io.opencensus.contrib.grpc.metrics.RpcMeasureConstants;
import io.opencensus.contrib.grpc.metrics.RpcViewConstants;
import io.opencensus.impl.stats.StatsComponentImpl;
import io.opencensus.stats.AggregationData;
import io.opencensus.stats.AggregationData.CountData;
import io.opencensus.stats.AggregationData.LastValueDataDouble;
import io.opencensus.stats.AggregationData.LastValueDataLong;
import io.opencensus.stats.AggregationData.SumDataDouble;
import io.opencensus.stats.Measure;
import io.opencensus.stats.StatsComponent;
import io.opencensus.stats.View;
import io.opencensus.tags.TagContext;
import io.opencensus.tags.TagValue;
import io.opencensus.trace.BlankSpan;
Expand Down Expand Up @@ -288,8 +301,8 @@ public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(

StatsTestUtils.MetricsRecord record = statsRecorder.pollRecord();
assertNotNull(record);
TagValue methodTagOld = record.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), methodTagOld.asString());
TagValue methodTag = record.tags.get(RpcMeasureConstants.GRPC_CLIENT_METHOD);
assertEquals(method.getFullMethodName(), methodTag.asString());
if (nonDefaultContext) {
TagValue extraTag = record.tags.get(StatsTestUtils.EXTRA_TAG);
assertEquals("extra value", extraTag.asString());
Expand Down Expand Up @@ -322,10 +335,10 @@ public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
// The intercepting listener calls callEnded() on ClientCallTracer, which records to Census.
record = statsRecorder.pollRecord();
assertNotNull(record);
methodTagOld = record.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), methodTagOld.asString());
TagValue statusTagOld = record.tags.get(DeprecatedCensusConstants.RPC_STATUS);
assertEquals(Status.Code.PERMISSION_DENIED.toString(), statusTagOld.asString());
methodTag = record.tags.get(RpcMeasureConstants.GRPC_CLIENT_METHOD);
assertEquals(method.getFullMethodName(), methodTag.asString());
TagValue statusTag = record.tags.get(RpcMeasureConstants.GRPC_CLIENT_STATUS);
assertEquals(Status.Code.PERMISSION_DENIED.toString(), statusTag.asString());
if (nonDefaultContext) {
TagValue extraTag = record.tags.get(StatsTestUtils.EXTRA_TAG);
assertEquals("extra value", extraTag.asString());
Expand Down Expand Up @@ -384,8 +397,8 @@ private void subtestClientBasicStatsDefaultContext(
assertNotNull(record);
assertNoServerContent(record);
assertEquals(1, record.tags.size());
TagValue methodTagOld = record.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), methodTagOld.asString());
TagValue methodTag = record.tags.get(RpcMeasureConstants.GRPC_CLIENT_METHOD);
assertEquals(method.getFullMethodName(), methodTag.asString());
assertEquals(
1, record.getMetricAsLongOrFail(DeprecatedCensusConstants.RPC_CLIENT_STARTED_COUNT));
} else {
Expand Down Expand Up @@ -446,10 +459,10 @@ private void subtestClientBasicStatsDefaultContext(
StatsTestUtils.MetricsRecord record = statsRecorder.pollRecord();
assertNotNull(record);
assertNoServerContent(record);
TagValue methodTagOld = record.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), methodTagOld.asString());
TagValue statusTagOld = record.tags.get(DeprecatedCensusConstants.RPC_STATUS);
assertEquals(Status.Code.OK.toString(), statusTagOld.asString());
TagValue methodTag = record.tags.get(RpcMeasureConstants.GRPC_CLIENT_METHOD);
assertEquals(method.getFullMethodName(), methodTag.asString());
TagValue statusTag = record.tags.get(RpcMeasureConstants.GRPC_CLIENT_STATUS);
assertEquals(Status.Code.OK.toString(), statusTag.asString());
assertEquals(
1, record.getMetricAsLongOrFail(DeprecatedCensusConstants.RPC_CLIENT_FINISHED_COUNT));
assertNull(record.getMetric(DeprecatedCensusConstants.RPC_CLIENT_ERROR_COUNT));
Expand Down Expand Up @@ -488,11 +501,16 @@ private void assertRealTimeMetric(
assertNotNull(record);
if (clientSide) {
assertNoServerContent(record);

TagValue methodTag = record.tags.get(RpcMeasureConstants.GRPC_CLIENT_METHOD);
assertEquals(method.getFullMethodName(), methodTag.asString());
} else {
assertNoClientContent(record);

TagValue methodTag = record.tags.get(RpcMeasureConstants.GRPC_SERVER_METHOD);
assertEquals(method.getFullMethodName(), methodTag.asString());
}
TagValue methodTagOld = record.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), methodTagOld.asString());

assertEquals(expectedValue, record.getMetricAsLongOrFail(measure));
}

Expand Down Expand Up @@ -564,8 +582,8 @@ public void clientStreamNeverCreatedStillRecordStats() {
assertNotNull(record);
assertNoServerContent(record);
assertEquals(1, record.tags.size());
TagValue methodTagOld = record.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), methodTagOld.asString());
TagValue methodTag = record.tags.get(RpcMeasureConstants.GRPC_CLIENT_METHOD);
assertEquals(method.getFullMethodName(), methodTag.asString());
assertEquals(
1,
record.getMetricAsLongOrFail(DeprecatedCensusConstants.RPC_CLIENT_STARTED_COUNT));
Expand All @@ -574,10 +592,10 @@ public void clientStreamNeverCreatedStillRecordStats() {
record = statsRecorder.pollRecord();
assertNotNull(record);
assertNoServerContent(record);
methodTagOld = record.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), methodTagOld.asString());
TagValue statusTagOld = record.tags.get(DeprecatedCensusConstants.RPC_STATUS);
assertEquals(Status.Code.DEADLINE_EXCEEDED.toString(), statusTagOld.asString());
methodTag = record.tags.get(RpcMeasureConstants.GRPC_CLIENT_METHOD);
assertEquals(method.getFullMethodName(), methodTag.asString());
TagValue statusTag = record.tags.get(RpcMeasureConstants.GRPC_CLIENT_STATUS);
assertEquals(Status.Code.DEADLINE_EXCEEDED.toString(), statusTag.asString());
assertEquals(
1,
record.getMetricAsLongOrFail(DeprecatedCensusConstants.RPC_CLIENT_FINISHED_COUNT));
Expand Down Expand Up @@ -668,8 +686,8 @@ private void subtestStatsHeadersPropagateTags(boolean propagate, boolean recordS
assertNotNull(clientRecord);
assertNoServerContent(clientRecord);
assertEquals(2, clientRecord.tags.size());
TagValue clientMethodTagOld = clientRecord.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), clientMethodTagOld.asString());
TagValue clientMethodTag = clientRecord.tags.get(RpcMeasureConstants.GRPC_CLIENT_METHOD);
assertEquals(method.getFullMethodName(), clientMethodTag.asString());
TagValue clientPropagatedTag = clientRecord.tags.get(StatsTestUtils.EXTRA_TAG);
assertEquals("extra-tag-value-897", clientPropagatedTag.asString());
}
Expand All @@ -690,7 +708,7 @@ private void subtestStatsHeadersPropagateTags(boolean propagate, boolean recordS
assertEquals(
tagger.toBuilder(clientCtx)
.putLocal(
DeprecatedCensusConstants.RPC_METHOD,
RpcMeasureConstants.GRPC_SERVER_METHOD,
TagValue.create(method.getFullMethodName()))
.build(),
io.opencensus.tags.unsafe.ContextUtils.getValue(serverContext));
Expand All @@ -704,19 +722,19 @@ private void subtestStatsHeadersPropagateTags(boolean propagate, boolean recordS
assertNotNull(serverRecord);
assertNoClientContent(serverRecord);
assertEquals(2, serverRecord.tags.size());
TagValue serverMethodTagOld = serverRecord.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), serverMethodTagOld.asString());
TagValue serverMethodTag = serverRecord.tags.get(RpcMeasureConstants.GRPC_SERVER_METHOD);
assertEquals(method.getFullMethodName(), serverMethodTag.asString());
TagValue serverPropagatedTag = serverRecord.tags.get(StatsTestUtils.EXTRA_TAG);
assertEquals("extra-tag-value-897", serverPropagatedTag.asString());

// Server completion record
serverRecord = statsRecorder.pollRecord();
assertNotNull(serverRecord);
assertNoClientContent(serverRecord);
serverMethodTagOld = serverRecord.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), serverMethodTagOld.asString());
TagValue serverStatusTagOld = serverRecord.tags.get(DeprecatedCensusConstants.RPC_STATUS);
assertEquals(Status.Code.OK.toString(), serverStatusTagOld.asString());
serverMethodTag = serverRecord.tags.get(RpcMeasureConstants.GRPC_SERVER_METHOD);
assertEquals(method.getFullMethodName(), serverMethodTag.asString());
TagValue serverStatusTag = serverRecord.tags.get(RpcMeasureConstants.GRPC_SERVER_STATUS);
assertEquals(Status.Code.OK.toString(), serverStatusTag.asString());
assertNull(serverRecord.getMetric(DeprecatedCensusConstants.RPC_SERVER_ERROR_COUNT));
serverPropagatedTag = serverRecord.tags.get(StatsTestUtils.EXTRA_TAG);
assertEquals("extra-tag-value-897", serverPropagatedTag.asString());
Expand All @@ -731,10 +749,10 @@ private void subtestStatsHeadersPropagateTags(boolean propagate, boolean recordS
StatsTestUtils.MetricsRecord clientRecord = statsRecorder.pollRecord();
assertNotNull(clientRecord);
assertNoServerContent(clientRecord);
TagValue clientMethodTagOld = clientRecord.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), clientMethodTagOld.asString());
TagValue clientStatusTagOld = clientRecord.tags.get(DeprecatedCensusConstants.RPC_STATUS);
assertEquals(Status.Code.OK.toString(), clientStatusTagOld.asString());
TagValue clientMethodTag = clientRecord.tags.get(RpcMeasureConstants.GRPC_CLIENT_METHOD);
assertEquals(method.getFullMethodName(), clientMethodTag.asString());
TagValue clientStatusTag = clientRecord.tags.get(RpcMeasureConstants.GRPC_CLIENT_STATUS);
assertEquals(Status.Code.OK.toString(), clientStatusTag.asString());
assertNull(clientRecord.getMetric(DeprecatedCensusConstants.RPC_CLIENT_ERROR_COUNT));
TagValue clientPropagatedTag = clientRecord.tags.get(StatsTestUtils.EXTRA_TAG);
assertEquals("extra-tag-value-897", clientPropagatedTag.asString());
Expand Down Expand Up @@ -911,8 +929,8 @@ private void subtestServerBasicStatsNoHeaders(
assertNotNull(record);
assertNoClientContent(record);
assertEquals(1, record.tags.size());
TagValue methodTagOld = record.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), methodTagOld.asString());
TagValue methodTag = record.tags.get(RpcMeasureConstants.GRPC_SERVER_METHOD);
assertEquals(method.getFullMethodName(), methodTag.asString());
assertEquals(
1,
record.getMetricAsLongOrFail(DeprecatedCensusConstants.RPC_SERVER_STARTED_COUNT));
Expand All @@ -926,7 +944,7 @@ private void subtestServerBasicStatsNoHeaders(
tagger
.emptyBuilder()
.putLocal(
DeprecatedCensusConstants.RPC_METHOD,
RpcMeasureConstants.GRPC_SERVER_METHOD,
TagValue.create(method.getFullMethodName()))
.build(),
statsCtx);
Expand Down Expand Up @@ -980,10 +998,10 @@ private void subtestServerBasicStatsNoHeaders(
StatsTestUtils.MetricsRecord record = statsRecorder.pollRecord();
assertNotNull(record);
assertNoClientContent(record);
TagValue methodTagOld = record.tags.get(DeprecatedCensusConstants.RPC_METHOD);
assertEquals(method.getFullMethodName(), methodTagOld.asString());
TagValue statusTagOld = record.tags.get(DeprecatedCensusConstants.RPC_STATUS);
assertEquals(Status.Code.CANCELLED.toString(), statusTagOld.asString());
TagValue methodTag = record.tags.get(RpcMeasureConstants.GRPC_SERVER_METHOD);
assertEquals(method.getFullMethodName(), methodTag.asString());
TagValue statusTag = record.tags.get(RpcMeasureConstants.GRPC_SERVER_STATUS);
assertEquals(Status.Code.CANCELLED.toString(), statusTag.asString());
assertEquals(
1, record.getMetricAsLongOrFail(DeprecatedCensusConstants.RPC_SERVER_FINISHED_COUNT));
assertEquals(
Expand Down Expand Up @@ -1148,4 +1166,92 @@ private static void assertNoClientContent(StatsTestUtils.MetricsRecord record) {
assertNull(record.getMetric(DeprecatedCensusConstants.RPC_CLIENT_UNCOMPRESSED_REQUEST_BYTES));
assertNull(record.getMetric(DeprecatedCensusConstants.RPC_CLIENT_UNCOMPRESSED_RESPONSE_BYTES));
}

@Deprecated
@Test
public void newTagsPopulateOldViews() throws InterruptedException {
StatsComponent localStats = new StatsComponentImpl();

// Test views that contain both of the remap tags: method & status.
localStats.getViewManager().registerView(RpcViewConstants.RPC_CLIENT_ERROR_COUNT_VIEW);
localStats.getViewManager().registerView(RpcViewConstants.GRPC_CLIENT_COMPLETED_RPC_VIEW);

CensusStatsModule localCensusStats = new CensusStatsModule(
tagger, tagCtxSerializer, localStats.getStatsRecorder(), fakeClock.getStopwatchSupplier(),
false, false, true, false /* real-time */);

CensusStatsModule.ClientCallTracer callTracer =
localCensusStats.newClientCallTracer(
tagger.empty(), method.getFullMethodName());

callTracer.newClientStreamTracer(STREAM_INFO, new Metadata());
fakeClock.forwardTime(30, MILLISECONDS);
callTracer.callEnded(Status.PERMISSION_DENIED.withDescription("No you don't"));

// Give OpenCensus a chance to update the views asynchronously.
Thread.sleep(100);

assertWithMessage("Legacy error count view had unexpected count")
.that(
getAggregationValueAsLong(
localStats,
RpcViewConstants.RPC_CLIENT_ERROR_COUNT_VIEW,
ImmutableList.of(
TagValue.create("PERMISSION_DENIED"),
TagValue.create(method.getFullMethodName()))))
.isEqualTo(1);

assertWithMessage("New error count view had unexpected count")
.that(
getAggregationValueAsLong(
localStats,
RpcViewConstants.GRPC_CLIENT_COMPLETED_RPC_VIEW,
ImmutableList.of(
TagValue.create(method.getFullMethodName()),
TagValue.create("PERMISSION_DENIED"))))
.isEqualTo(1);
}

@Deprecated
private long getAggregationValueAsLong(StatsComponent localStats, View view,
List<TagValue> dimension) {
AggregationData aggregationData = localStats.getViewManager()
.getView(view.getName())
.getAggregationMap()
.get(dimension);

return aggregationData.match(
new Function<SumDataDouble, Long>() {
@Override
public Long apply(SumDataDouble arg) {
return (long) arg.getSum();
}
},
Functions.<Long>throwAssertionError(),
new Function<CountData, Long>() {
@Override
public Long apply(CountData arg) {
return arg.getCount();
}
},
Functions.<Long>throwAssertionError(),
new Function<LastValueDataDouble, Long>() {
@Override
public Long apply(LastValueDataDouble arg) {
return (long) arg.getLastValue();
}
},
new Function<LastValueDataLong, Long>() {
@Override
public Long apply(LastValueDataLong arg) {
return arg.getLastValue();
}
},
new Function<AggregationData, Long>() {
@Override
public Long apply(AggregationData arg) {
return ((io.opencensus.stats.AggregationData.MeanData) arg).getCount();
}
});
}
}

0 comments on commit 271cbff

Please sign in to comment.