grpc-go
grpc-go copied to clipboard
Published
20 hours ago •
grpc
grpc
Data race in TestOutlierDetectionConfigPropagationToChildPolicy
WARNING: DATA RACE
Write at 0x00c0000b11a0 by goroutine 2937:
runtime.mapassign_faststr()
/opt/hostedtoolcache/go/1.20.5/x64/src/runtime/map_faststr.go:203 +0x0
google.golang.org/grpc/balancer.Register()
/home/runner/work/grpc-go/grpc-go/balancer/balancer.go:54 +0x77
google.golang.org/grpc/xds/internal/balancer/clusterresolver/e2e_test_test.s.TestOutlierDetectionConfigPropagationToChildPolicy.func1()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/e2e_test/balancer_test.go:440 +0x47
runtime.deferreturn()
/opt/hostedtoolcache/go/1.20.5/x64/src/runtime/panic.go:476 +0x32
google.golang.org/grpc/xds/internal/balancer/clusterresolver/e2e_test_test.(*s).TestOutlierDetectionConfigPropagationToChildPolicy()
<autogenerated>:1 +0x44
runtime.call16()
/opt/hostedtoolcache/go/1.20.5/x64/src/runtime/asm_amd64.s:728 +0x48
reflect.callMethod()
<autogenerated>:1 +0x64
reflect.methodValueCall()
/opt/hostedtoolcache/go/1.20.5/x64/src/reflect/asm_amd64.s:76 +0x79
testing.tRunner()
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1576 +0x[216](https://github.com/grpc/grpc-go/actions/runs/5425936248/jobs/9867334312?pr=6425#step:8:217)
testing.(*T).Run.func1()
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1629 +0x47
Previous read at 0x00c0000b11a0 by goroutine 2964:
runtime.mapaccess2_faststr()
/opt/hostedtoolcache/go/1.20.5/x64/src/runtime/map_faststr.go:108 +0x0
google.golang.org/grpc/balancer.Get()
/home/runner/work/grpc-go/grpc-go/balancer/balancer.go:73 +0xa7
google.golang.org/grpc/xds/internal/balancer/wrrlocality.bb.Build()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/wrrlocality/balancer.go:63 +0x46
google.golang.org/grpc/xds/internal/balancer/wrrlocality.(*bb).Build()
<autogenerated>:1 +0xa7
google.golang.org/grpc/xds/internal/balancer/clusterimpl.(*clusterImplBalancer).UpdateClientConnState()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterimpl/clusterimpl.go:259 +0x6b8
google.golang.org/grpc/internal/balancer/gracefulswitch.(*Balancer).UpdateClientConnState()
/home/runner/work/grpc-go/grpc-go/internal/balancer/gracefulswitch/gracefulswitch.go:164 +0x10d
google.golang.org/grpc/xds/internal/balancer/outlierdetection.(*outlierDetectionBalancer).UpdateClientConnState()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/outlierdetection/balancer.go:326 +0xe89
google.golang.org/grpc/internal/balancer/gracefulswitch.(*Balancer).UpdateClientConnState()
/home/runner/work/grpc-go/grpc-go/internal/balancer/gracefulswitch/gracefulswitch.go:164 +0x10d
google.golang.org/grpc/internal/balancergroup.(*subBalancerWrapper).updateClientConnState()
/home/runner/work/grpc-go/grpc-go/internal/balancergroup/balancergroup.go:144 +0x2da
google.golang.org/grpc/internal/balancergroup.(*BalancerGroup).UpdateClientConnState()
/home/runner/work/grpc-go/grpc-go/internal/balancergroup/balancergroup.go:478 +0x136
google.golang.org/grpc/xds/internal/balancer/priority.(*childBalancer).sendUpdate()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer_child.go:113 +0x1b8
google.golang.org/grpc/xds/internal/balancer/priority.(*childBalancer).start()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer_child.go:106 +0x15e
google.golang.org/grpc/xds/internal/balancer/priority.(*priorityBalancer).switchToChild()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer_priority.go:165 +0x157
google.golang.org/grpc/xds/internal/balancer/priority.(*priorityBalancer).syncPriority()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer_priority.go:108 +0x62c
google.golang.org/grpc/xds/internal/balancer/priority.(*priorityBalancer).run()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer.go:268 +0x2f9
google.golang.org/grpc/xds/internal/balancer/priority.bb.Build.func1()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer.go:65 +0x39
Goroutine 2937 (running) created at:
testing.(*T).Run()
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1629 +0x805
google.golang.org/grpc/internal/grpctest.RunSubTests()
/home/runner/work/grpc-go/grpc-go/internal/grpctest/grpctest.go:101 +0x50e
google.golang.org/grpc/xds/internal/balancer/clusterresolver/e2e_test_test.Test()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/e2e_test/eds_impl_test.go:74 +0x3c
testing.tRunner()
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1576 +0x216
testing.(*T).Run.func1()
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1629 +0x47
Goroutine 2964 (running) created at:
google.golang.org/grpc/xds/internal/balancer/priority.bb.Build()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer.go:65 +0x925
google.golang.org/grpc/xds/internal/balancer/priority.(*bb).Build()
<autogenerated>:1 +0xa7
google.golang.org/grpc/xds/internal/balancer/clusterresolver/e2e_test_test.(*wrappedPriorityBuilder).Build()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/e2e_test/balancer_test.go:342 +0xc7
google.golang.org/grpc/xds/internal/balancer/clusterresolver.glob..func1()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/clusterresolver.go:53 +0xaf
google.golang.org/grpc/xds/internal/balancer/clusterresolver.(*clusterResolverBalancer).updateChildConfig()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/clusterresolver.go:241 +0x177
google.golang.org/grpc/xds/internal/balancer/clusterresolver.(*clusterResolverBalancer).handleResourceUpdate()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/clusterresolver.go:230 +0x109
google.golang.org/grpc/xds/internal/balancer/clusterresolver.(*clusterResolverBalancer).run()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/clusterresolver.go:331 +0x512
google.golang.org/grpc/xds/internal/balancer/clusterresolver.bb.Build.func1()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/clusterresolver.go:94 +0x39
==================
--- FAIL: Test (5.40s)
--- FAIL: Test/OutlierDetectionConfigPropagationToChildPolicy (0.02s)
tlogger.go:116: INFO server.go:132 [xds-e2e] Created new snapshot cache... (t=+59.5µs)
tlogger.go:116: INFO server.go:652 [core] [Server #573] Server created (t=+419.102µs)
tlogger.go:116: INFO server.go:158 [xds-e2e] Registered Aggregated Discovery Service (ADS)... (t=+479.303µs)
tlogger.go:116: INFO server.go:177 [xds-e2e] xDS management server serving at: 127.0.0.1:45339... (t=+526.903µs)
tlogger.go:116: INFO server.go:840 [core] [Server #573 ListenSocket #574] ListenSocket created (t=+1.42921ms)
tlogger.go:116: INFO server.go:652 [core] [Server #575] Server created (t=+2.137015ms)
stubserver.go:222: Started test service backend at "127.0.0.1:40833"
tlogger.go:116: INFO server.go:[220](https://github.com/grpc/grpc-go/actions/runs/5425936248/jobs/9867334312?pr=6425#step:8:221) [xds-e2e] Created new resource snapshot... (t=+2.281516ms)
tlogger.go:116: INFO server.go:226 [xds-e2e] Updated snapshot cache with resource snapshot... (t=+2.327116ms)
tlogger.go:116: INFO server.go:840 [core] [Server #575 ListenSocket #576] ListenSocket created (t=+3.686626ms)
tlogger.go:116: INFO client_new.go:80 [xds] [xds-client 0xc0005769b0] Created client to xDS management server: 127.0.0.1:45339-insecure-xds_v3 (t=+5.409738ms)
tlogger.go:116: INFO clientconn.go:316 [core] [Channel #577] Channel created (t=+5.652939ms)
tlogger.go:116: INFO clientconn.go:1770 [core] [Channel #577] original dial target is: "whatever:///test.service" (t=+5.70334ms)
tlogger.go:116: INFO clientconn.go:1777 [core] [Channel #577] parsed dial target is: {Scheme:whatever Authority: URL:{Scheme:whatever Opaque: User: Host: Path:/test.service RawPath: OmitHost:false ForceQuery:false RawQuery: Fragment: RawFragment:}} (t=+5.78894ms)
tlogger.go:116: INFO clientconn.go:1877 [core] [Channel #577] Channel authority set to "test.service" (t=+5.832841ms)
tlogger.go:116: INFO resolver_conn_wrapper.go:238 [core] [Channel #577] Resolver state updated: {
"Addresses": null,
"ServiceConfig": {
"Config": {
"Config": null,
"LB": null,
"Methods": {}
},
"Err": null
},
"Attributes": {}
} (service config updated) (t=+6.026142ms)
tlogger.go:116: INFO balancer_conn_wrappers.go:192 [core] [Channel #577] Channel switches to new LB policy "cds_experimental" (t=+6.124043ms)
tlogger.go:116: INFO cdsbalancer.go:99 [xds] [cds-lb 0xc0007c9860] Created (t=+6.188143ms)
tlogger.go:116: INFO cdsbalancer.go:110 [xds] [cds-lb 0xc0007c9860] xDS credentials in use: false (t=+6.[224](https://github.com/grpc/grpc-go/actions/runs/5425936248/jobs/9867334312?pr=6425#step:8:225)543ms)
tlogger.go:116: INFO cdsbalancer.go:514 [xds] [cds-lb 0xc0007c9860] Received balancer config update: {
"LoadBalancingConfig": null,
"Cluster": "cluster-my-service-client-side-xds"
} (t=+6.338744ms)
tlogger.go:116: INFO cluster_handler.go:190 [xds] [cds-lb 0xc0007c9860] CDS watch started on cluster-my-service-client-side-xds (t=+6.421145ms)
tlogger.go:116: INFO clientconn.go:316 [core] [Channel #578] Channel created (t=+6.572946ms)
tlogger.go:116: INFO clientconn.go:1770 [core] [Channel #578] original dial target is: "127.0.0.1:45339" (t=+6.615646ms)
tlogger.go:116: INFO clientconn.go:1775 [core] [Channel #578] dial target "127.0.0.1:45339" parse failed: parse "127.0.0.1:45339": first path segment in URL cannot contain colon (t=+6.666547ms)
tlogger.go:116: INFO clientconn.go:1791 [core] [Channel #578] fallback to scheme "passthrough" (t=+6.699047ms)
tlogger.go:116: INFO clientconn.go:1799 [core] [Channel #578] parsed dial target is: {Scheme:passthrough Authority: URL:{Scheme:passthrough Opaque: User: Host: Path:/127.0.0.1:45339 RawPath: OmitHost:false ForceQuery:false RawQuery: Fragment: RawFragment:}} (t=+6.758147ms)
tlogger.go:116: INFO clientconn.go:1877 [core] [Channel #578] Channel authority set to "127.0.0.1:45339" (t=+6.799348ms)
tlogger.go:116: INFO resolver_conn_wrapper.go:238 [core] [Channel #578] Resolver state updated: {
"Addresses": [
{
"Addr": "127.0.0.1:45339",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Type": 0,
"Metadata": null
}
],
"ServiceConfig": null,
"Attributes": null
} (resolver returned new addresses) (t=+7.038749ms)
tlogger.go:116: INFO balancer_conn_wrappers.go:192 [core] [Channel #578] Channel switches to new LB policy "pick_first" (t=+7.16305ms)
tlogger.go:116: INFO clientconn.go:897 [core] [Channel #578 SubChannel #579] Subchannel created (t=+7.260951ms)
tlogger.go:116: INFO clientconn.go:564 [core] [Channel #578] Channel Connectivity change to CONNECTING (t=+7.311151ms)
tlogger.go:116: INFO transport.go:238 [xds] [xds-client 0xc0005769b0] [127.0.0.1:45339] Created transport to server "127.0.0.1:45339" (t=+7.578853ms)
tlogger.go:116: INFO clientconn.go:1285 [core] [Channel #578 SubChannel #579] Subchannel Connectivity change to CONNECTING (t=+7.655654ms)
tlogger.go:116: INFO clientconn.go:1398 [core] [Channel #578 SubChannel #579] Subchannel picks a new address "127.0.0.1:45339" to connect (t=+7.737954ms)
tlogger.go:116: INFO clientconn.go:1285 [core] [Channel #578 SubChannel #579] Subchannel Connectivity change to READY (t=+8.814462ms)
tlogger.go:116: INFO clientconn.go:564 [core] [Channel #578] Channel Connectivity change to READY (t=+8.909362ms)
tlogger.go:116: INFO transport.go:347 [xds] [xds-client 0xc0005769b0] [127.0.0.1:45339] ADS stream created (t=+9.035763ms)
tlogger.go:116: INFO simple.go:359 [xds-e2e] nodeID "7a165661-0c37-4992-9315-770a7dc9bd19" requested type.googleapis.com/envoy.config.cluster.v3.Cluster[cluster-my-service-client-side-xds] and known map[]. Diff [cluster-my-service-client-side-xds] (t=+9.882069ms)
tlogger.go:116: INFO simple.go:428 [xds-e2e] respond type.googleapis.com/envoy.config.cluster.v3.Cluster[cluster-my-service-client-side-xds] version "" with version "1" (t=+9.94087ms)
tlogger.go:116: INFO cdsbalancer.go:297 [xds] [cds-lb 0xc0007c9860] Received Cluster resource contains content: [
{
"ClusterType": 0,
"ClusterName": "cluster-my-service-client-side-xds",
"EDSServiceName": "endpoints-my-service-client-side-xds",
"LRSServerConfig": 0,
"SecurityCfg": null,
"MaxRequests": null,
"DNSHostName": "",
"PrioritizedClusterNames": null,
"LBPolicy": [
{
"xds_wrr_locality_experimental": {
"childPolicy": [
{
"round_robin": {}
}
]
}
}
],
"OutlierDetection": {
"interval": "10s",
"baseEjectionTime": "30s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 10,
"successRateEjection": {
"stdevFactor": 2000,
"enforcementPercentage": 50,
"minimumHosts": 10,
"requestVolume": 50
}
},
"Raw": {
"type_url": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"value": "CiJjbHVzdGVyLW15LXNlcnZpY2UtY2xpZW50LXNpZGUteGRzGioKAhoAEiRlbmRwb2ludHMtbXktc2VydmljZS1jbGllbnQtc2lkZS14ZHOaASMSAggKGgIIHiICCAoyAggyOgIICkICCDJKAwjQD6oBAwisAhAD"
}
}
], security config: null (t=+10.856176ms)
tlogger.go:116: INFO clusterresolver.go:86 [xds] [xds-cluster-resolver-lb 0xc0002ef6c0] Created (t=+10.941277ms)
tlogger.go:116: INFO cdsbalancer.go:323 [xds] [cds-lb 0xc0007c9860] Created child policy 0xc0002ef6c0 of type cluster_resolver_experimental (t=+10.984877ms)
tlogger.go:116: INFO clusterresolver.go:196 [xds] [xds-cluster-resolver-lb 0xc0002ef6c0] Received new balancer config: {
"discoveryMechanisms": [
{
"cluster": "cluster-my-service-client-side-xds",
"edsServiceName": "endpoints-my-service-client-side-xds",
"outlierDetection": {
"interval": "10s",
"baseEjectionTime": "30s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 10,
"successRateEjection": {
"stdevFactor": 2000,
"enforcementPercentage": 50,
"minimumHosts": 10,
"requestVolume": 50
}
}
}
],
"xdsLbPolicy": [
{
"xds_wrr_locality_experimental": {
"childPolicy": [
{
"round_robin": {}
}
]
}
}
]
} (t=+11.588281ms)
tlogger.go:116: INFO simple.go:359 [xds-e2e] nodeID "7a165661-0c37-4992-9315-770a7dc9bd19" requested type.googleapis.com/envoy.config.cluster.v3.Cluster[cluster-my-service-client-side-xds] and known map[cluster-my-service-client-side-xds:{}]. Diff [] (t=+12.113285ms)
tlogger.go:116: INFO simple.go:380 [xds-e2e] open watch 1 for type.googleapis.com/envoy.config.cluster.v3.Cluster[cluster-my-service-client-side-xds] from nodeID "7a165661-0c37-4992-9315-770a7dc9bd19", version "1" (t=+12.176885ms)
tlogger.go:116: INFO simple.go:359 [xds-e2e] nodeID "7a165661-0c37-4992-9315-770a7dc9bd19" requested type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment[endpoints-my-service-client-side-xds] and known map[]. Diff [endpoints-my-service-client-side-xds] (t=+12.324487ms)
tlogger.go:116: INFO simple.go:428 [xds-e2e] respond type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment[endpoints-my-service-client-side-xds] version "" with version "1" (t=+12.405587ms)
tlogger.go:116: INFO balancer.go:66 [xds] [priority-lb 0xc000146cb0] Created (t=+13.208593ms)
tlogger.go:116: INFO clusterresolver.go:254 [xds] [xds-cluster-resolver-lb 0xc0002ef6c0] Built child policy config: {
"children": {
"priority-0-0": {
"config": [
{
"outlier_detection_experimental": {
"interval": "10s",
"baseEjectionTime": "30s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 10,
"successRateEjection": {
"stdevFactor": 2000,
"enforcementPercentage": 50,
"minimumHosts": 10,
"requestVolume": 50
},
"childPolicy": [
{
"xds_cluster_impl_experimental": {
"cluster": "cluster-my-service-client-side-xds",
"edsServiceName": "endpoints-my-service-client-side-xds",
"childPolicy": [
{
"xds_wrr_locality_experimental": {
"childPolicy": [
{
"round_robin": {}
}
]
}
}
]
}
}
]
}
}
],
"ignoreReresolutionRequests": true
}
},
"priorities": [
"priority-0-0"
]
} (t=+14.477402ms)
tlogger.go:116: INFO clientconn.go:564 [core] [Channel #577] Channel Connectivity change to CONNECTING (t=+15.018606ms)
tlogger.go:116: INFO balancergroup.go:311 [xds] [priority-lb 0xc000146cb0] Adding child policy of type "outlier_detection_experimental" for locality "priority-0-0" (t=+15.070506ms)
tlogger.go:116: INFO balancergroup.go:100 [xds] [priority-lb 0xc000146cb0] Creating child policy of type "outlier_detection_experimental" for locality "priority-0-0" (t=+15.131906ms)
tlogger.go:116: WARNING authority.go:416 [xds] [xds-client 0xc0005769b0] [127.0.0.1:45339] Watchers not notified since ADS stream failed after having received at least one response: rpc error: code = Canceled desc = context canceled (t=+15.193607ms)
tlogger.go:116: WARNING transport.go:497 [xds] [xds-client 0xc0005769b0] [127.0.0.1:45339] ADS stream closed: rpc error: code = Canceled desc = context canceled (t=+15.248707ms)
tlogger.go:116: INFO clientconn.go:564 [core] [Channel #578] Channel Connectivity change to SHUTDOWN (t=+15.337208ms)
tlogger.go:116: INFO balancer_conn_wrappers.go:203 [core] [Channel #578] ccBalancerWrapper: closing (t=+15.384008ms)
tlogger.go:116: INFO resolver_conn_wrapper.go:126 [core] [Channel #578] Closing the name resolver (t=+15.484609ms)
tlogger.go:116: INFO clientconn.go:1285 [core] [Channel #578 SubChannel #579] Subchannel Connectivity change to SHUTDOWN (t=+15.551609ms)
tlogger.go:116: INFO clientconn.go:1638 [core] [Channel #578 SubChannel #579] Subchannel deleted (t=+15.60701ms)
tlogger.go:116: INFO clientconn.go:316 [core] [Channel #578] Channel deleted (t=+15.64371ms)
tlogger.go:116: INFO clientimpl.go:88 [xds] [xds-client 0xc0005769b0] Shutdown (t=+15.69871ms)
tlogger.go:116: INFO clientconn.go:564 [core] [Channel #577] Channel Connectivity change to SHUTDOWN (t=+15.743811ms)
tlogger.go:116: INFO balancer_conn_wrappers.go:203 [core] [Channel #577] ccBalancerWrapper: closing (t=+15.784511ms)
tlogger.go:116: INFO resolver_conn_wrapper.go:126 [core] [Channel #577] Closing the name resolver (t=+15.863111ms)
tlogger.go:116: INFO cluster_handler.go:193 [xds] [cds-lb 0xc0007c9860] CDS watch canceled on cluster-my-service-client-side-xds (t=+15.918312ms)
tlogger.go:116: INFO balancer.go:77 [xds] [outlier-detection-lb 0xc000000c00] Created (t=+15.991712ms)
tlogger.go:116: INFO clientconn.go:316 [core] [Channel #577] Channel deleted (t=+16.060013ms)
tlogger.go:116: INFO server.go:791 [core] [Server #575 ListenSocket #576] ListenSocket deleted (t=+16.243714ms)
tlogger.go:116: INFO clusterimpl.go:73 [xds] [xds-cluster-impl-lb 0xc00052c6c0] Created (t=+16.342415ms)
tlogger.go:116: INFO server.go:791 [core] [Server #573 ListenSocket #574] ListenSocket deleted (t=+16.437916ms)
tlogger.go:116: INFO clusterimpl.go:224 [xds] [xds-cluster-impl-lb 0xc00052c6c0] Received update from resolver, balancer config: {
"cluster": "cluster-my-service-client-side-xds",
"edsServiceName": "endpoints-my-service-client-side-xds",
"childPolicy": [
{
"xds_wrr_locality_experimental": {
"childPolicy": [
{
"round_robin": {}
}
]
}
}
]
} (t=+16.533116ms)
tlogger.go:116: INFO weightedtarget.go:59 [xds] [weighted-target-lb 0xc0005faae0] Created (t=+19.445937ms)
tlogger.go:116: INFO balancer.go:88 [xds] [wrrlocality-lb 0xc0006e13e0] Created (t=+19.518137ms)
tlogger.go:116: INFO weightedtarget.go:84 [xds] [weighted-target-lb 0xc0005faae0] Received update from resolver, balancer config: {
"targets": {
"{\"region\":\"region-1\",\"zone\":\"zone-1\",\"subZone\":\"subzone-1\"}": {
"weight": 1,
"childPolicy": [
{
"round_robin": {}
}
]
}
}
} (t=+19.789639ms)
tlogger.go:116: INFO balancergroup.go:311 [xds] [weighted-target-lb 0xc0005faae0] Adding child policy of type "round_robin" for locality "{\"region\":\"region-1\",\"zone\":\"zone-1\",\"subZone\":\"subzone-1\"}" (t=+19.87314ms)
tlogger.go:116: INFO balancergroup.go:100 [xds] [weighted-target-lb 0xc0005faae0] Creating child policy of type "round_robin" for locality "{\"region\":\"region-1\",\"zone\":\"zone-1\",\"subZone\":\"subzone-1\"}" (t=+19.93124ms)
tlogger.go:116: WARNING balancer.go:110 [balancer] base.baseBalancer: failed to create new SubConn: *gracefulswitch.balancerWrapper at address 0xc000418c80 that called NewSubConn is deleted (t=+20.011441ms)
tlogger.go:116: INFO roundrobin.go:50 [roundrobin] roundrobinPicker: Build called with info: {map[]} (t=+20.083541ms)
tlogger.go:116: INFO balancergroup.go:524 [xds] [weighted-target-lb 0xc0005faae0] Balancer state update from locality {"region":"region-1","zone":"zone-1","subZone":"subzone-1"}, new state: {ConnectivityState:CONNECTING Picker:0xc0005f5880} (t=+20.138042ms)
tlogger.go:116: INFO aggregator.go:252 [xds] [weighted-target-lb 0xc0005faae0] Child pickers with config: map[{"region":"region-1","zone":"zone-1","subZone":"subzone-1"}:weight:1,picker:0xc0005f5880,state:CONNECTING,stateToAggregate:CONNECTING] (t=+20.198442ms)
tlogger.go:116: INFO balancergroup.go:524 [xds] [priority-lb 0xc000146cb0] Balancer state update from locality priority-0-0, new state: {ConnectivityState:CONNECTING Picker:0xc0003da618} (t=+20.303343ms)
tlogger.go:116: INFO clusterimpl.go:338 [xds] [xds-cluster-impl-lb 0xc00052c6c0] Shutdown (t=+20.433644ms)
tlogger.go:116: INFO balancergroup.go:394 [xds] [priority-lb 0xc000146cb0] Removing child policy for locality "priority-0-0" (t=+20.496044ms)
tlogger.go:116: INFO clusterresolver.go:344 [xds] [xds-cluster-resolver-lb 0xc0002ef6c0] Shutdown (t=+20.537644ms)
tlogger.go:116: INFO cdsbalancer.go:454 [xds] [cds-lb 0xc0007c9860] Shutdown (t=+20.629345ms)
testing.go:1446: race detected during execution of test
testing.go:1446: race detected during execution of test
FAIL
FAIL google.golang.org/grpc/xds/internal/balancer/clusterresolver/e2e_test 11.270s
ok google.golang.org/grpc/xds/internal/balancer/outlierdetection 0.409s
ok google.golang.org/grpc/xds/internal/balancer/outlierdetection/e2e_test 1.015s
ok google.golang.org/grpc/xds/internal/balancer/priority 5.[228](https://github.com/grpc/grpc-go/actions/runs/5425936248/jobs/9867334312?pr=6425#step:8:229)s
ok google.golang.org/grpc/xds/internal/balancer/ringhash 0.501s
ok google.golang.org/grpc/xds/internal/balancer/ringhash/e2e 2.086s
ok google.golang.org/grpc/xds/internal/balancer/wrrlocality 0.050s
ok google.golang.org/grpc/xds/internal/clusterspecifier/rls 0.059s
ok google.golang.org/grpc/xds/internal/httpfilter/fault 1.[268](https://github.com/grpc/grpc-go/actions/runs/5425936248/jobs/9867334312?pr=6425#step:8:269)s
ok google.golang.org/grpc/xds/internal/resolver 2.084s
ok google.golang.org/grpc/xds/internal/server 0.410s
ok google.golang.org/grpc/xds/internal/test/e2e 0.072s
ok google.golang.org/grpc/xds/internal/testutils 0.064s
ok google.golang.org/grpc/xds/internal/xdsclient 6.505s
ok google.golang.org/grpc/xds/internal/xdsclient/bootstrap 0.191s
ok google.golang.org/grpc/xds/internal/xdsclient/load 0.089s
ok google.golang.org/grpc/xds/internal/xdsclient/tests 34.144s
ok google.golang.org/grpc/xds/internal/xdsclient/transport 3.[407](https://github.com/grpc/grpc-go/actions/runs/5425936248/jobs/9867334312?pr=6425#step:8:408)s
ok google.golang.org/grpc/xds/internal/xdsclient/xdslbregistry 6.702s
ok google.golang.org/grpc/xds/internal/xdsclient/xdsresource 2.646s
ok google.golang.org/grpc/xds/internal/xdsclient/xdsresource/tests 0.092s
FAIL
https://github.com/grpc/grpc-go/actions/runs/5425936248/jobs/9867334312?pr=6425
https://github.com/grpc/grpc-go/actions/runs/5648427585/job/15300626084?pr=6471
https://github.com/grpc/grpc-go/actions/runs/5660143001/job/15335218210?pr=6477