Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

xds: Fix LBs blindly propagating XdsClient errors #9012

Merged
merged 1 commit into from Mar 23, 2022
Merged

xds: Fix LBs blindly propagating XdsClient errors #9012

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 7 additions & 2 deletions xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java
View file
Open in desktop
Expand Up @@ -246,7 +246,12 @@ void shutdown() {
}

@Override
public void onError(final Status error) {
public void onError(Status error) {
Status status = Status.UNAVAILABLE
.withDescription(
String.format("Unable to load CDS %s. xDS server returned: %s: %s",
name, error.getCode(), error.getDescription()))
.withCause(error.getCause());
syncContext.execute(new Runnable() {
@Override
public void run() {
Expand All @@ -255,7 +260,7 @@ public void run() {
}
// All watchers should receive the same error, so we only propagate it once.
if (ClusterState.this == root) {
handleClusterDiscoveryError(error);
handleClusterDiscoveryError(status);
}
}
});
Expand Down
6 changes: 5 additions & 1 deletion xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java
View file
Open in desktop
Expand Up @@ -461,7 +461,11 @@ public void run() {
if (shutdown) {
return;
}
status = error;
String resourceName = edsServiceName != null ? edsServiceName : name;
status = Status.UNAVAILABLE
.withDescription(String.format("Unable to load EDS %s. xDS server returned: %s: %s",
resourceName, error.getCode(), error.getDescription()))
.withCause(error.getCause());
logger.log(XdsLogLevel.WARNING, "Received EDS error: {0}", error);
handleEndpointResolutionError();
}
Expand Down
8 changes: 6 additions & 2 deletions xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java
View file
Open in desktop
Expand Up @@ -459,7 +459,10 @@ public void aggregateCluster_discoveryErrorBeforeChildLbCreated_returnErrorPicke
xdsClient.deliverError(error);
verify(helper).updateBalancingState(
eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture());
assertPicker(pickerCaptor.getValue(), error, null);
Status expectedError = Status.UNAVAILABLE.withDescription(
"Unable to load CDS cluster-foo.googleapis.com. xDS server returned: "
+ "RESOURCE_EXHAUSTED: OOM");
assertPicker(pickerCaptor.getValue(), expectedError, null);
assertThat(childBalancers).isEmpty();
}

Expand All @@ -481,7 +484,8 @@ public void aggregateCluster_discoveryErrorAfterChildLbCreated_propagateToChildL

Status error = Status.RESOURCE_EXHAUSTED.withDescription("OOM");
xdsClient.deliverError(error);
assertThat(childLb.upstreamError).isEqualTo(error);
assertThat(childLb.upstreamError.getCode()).isEqualTo(Status.Code.UNAVAILABLE);
assertThat(childLb.upstreamError.getDescription()).contains("RESOURCE_EXHAUSTED: OOM");
assertThat(childLb.shutdown).isFalse(); // child LB may choose to keep working
}

Expand Down
18 changes: 18 additions & 0 deletions xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java
View file
Open in desktop
Expand Up @@ -850,6 +850,24 @@ public void resolutionErrorBeforeChildLbCreated_returnErrorPickerIfAllClustersEn
null);
}

@Test
public void resolutionErrorBeforeChildLbCreated_edsOnly_returnErrorPicker() {
ClusterResolverConfig config = new ClusterResolverConfig(
Arrays.asList(edsDiscoveryMechanism1), roundRobin);
deliverLbConfig(config);
assertThat(xdsClient.watchers.keySet()).containsExactly(EDS_SERVICE_NAME1);
assertThat(childBalancers).isEmpty();
reset(helper);
xdsClient.deliverError(Status.RESOURCE_EXHAUSTED.withDescription("OOM"));
assertThat(childBalancers).isEmpty();
verify(helper).updateBalancingState(
eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture());
PickResult result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class));
Status actualStatus = result.getStatus();
assertThat(actualStatus.getCode()).isEqualTo(Status.Code.UNAVAILABLE);
assertThat(actualStatus.getDescription()).contains("RESOURCE_EXHAUSTED: OOM");
}

@Test
public void handleNameResolutionErrorFromUpstream_beforeChildLbCreated_returnErrorPicker() {
ClusterResolverConfig config = new ClusterResolverConfig(
Expand Down