diff --git a/xds/internal/balancer/outlierdetection/balancer.go b/xds/internal/balancer/outlierdetection/balancer.go index 062a8e5e48d2..3d29bdaa0e60 100644 --- a/xds/internal/balancer/outlierdetection/balancer.go +++ b/xds/internal/balancer/outlierdetection/balancer.go @@ -35,6 +35,7 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/internal/balancer/gracefulswitch" "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/envconfig" "google.golang.org/grpc/internal/grpclog" "google.golang.org/grpc/internal/grpcrand" @@ -62,13 +63,14 @@ type bb struct{} func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { b := &outlierDetectionBalancer{ - cc: cc, - closed: grpcsync.NewEvent(), - done: grpcsync.NewEvent(), - addrs: make(map[string]*addressInfo), - scWrappers: make(map[balancer.SubConn]*subConnWrapper), - scUpdateCh: buffer.NewUnbounded(), - pickerUpdateCh: buffer.NewUnbounded(), + cc: cc, + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + addrs: make(map[string]*addressInfo), + scWrappers: make(map[balancer.SubConn]*subConnWrapper), + scUpdateCh: buffer.NewUnbounded(), + pickerUpdateCh: buffer.NewUnbounded(), + channelzParentID: bOpts.ChannelzParentID, } b.logger = prefixLogger(b) b.logger.Infof("Created") @@ -159,10 +161,11 @@ type outlierDetectionBalancer struct { // to suppress redundant picker updates. recentPickerNoop bool - closed *grpcsync.Event - done *grpcsync.Event - cc balancer.ClientConn - logger *grpclog.PrefixLogger + closed *grpcsync.Event + done *grpcsync.Event + cc balancer.ClientConn + logger *grpclog.PrefixLogger + channelzParentID *channelz.Identifier // childMu guards calls into child (to uphold the balancer.Balancer API // guarantee of synchronous calls). @@ -813,7 +816,11 @@ func (b *outlierDetectionBalancer) successRateAlgorithm() { return } successRate := float64(bucket.numSuccesses) / float64(bucket.numSuccesses+bucket.numFailures) - if successRate < (mean - stddev*(float64(ejectionCfg.StdevFactor)/1000)) { + requiredSuccessRate := mean - stddev*(float64(ejectionCfg.StdevFactor)/1000) + if successRate < requiredSuccessRate { + channelz.Infof(logger, b.channelzParentID, "SuccessRate algorithm detected outlier: %s. "+ + "Parameters: successRate=%f, mean=%f, stddev=%f, requiredSuccessRate=%f", + addrInfo.string(), successRate, mean, stddev, requiredSuccessRate) if uint32(grpcrand.Int31n(100)) < ejectionCfg.EnforcementPercentage { b.ejectAddress(addrInfo) } @@ -840,6 +847,8 @@ func (b *outlierDetectionBalancer) failurePercentageAlgorithm() { } failurePercentage := (float64(bucket.numFailures) / float64(bucket.numSuccesses+bucket.numFailures)) * 100 if failurePercentage > float64(b.cfg.FailurePercentageEjection.Threshold) { + channelz.Infof(logger, b.channelzParentID, "FailurePercentage algorithm detected outlier: %s, failurePercentage=%f", + addrInfo.string(), failurePercentage) if uint32(grpcrand.Int31n(100)) < ejectionCfg.EnforcementPercentage { b.ejectAddress(addrInfo) } @@ -854,7 +863,9 @@ func (b *outlierDetectionBalancer) ejectAddress(addrInfo *addressInfo) { addrInfo.ejectionTimeMultiplier++ for _, sbw := range addrInfo.sws { sbw.eject() + channelz.Infof(logger, b.channelzParentID, "Subchannel ejected: %+v", sbw.string()) } + } // Caller must hold b.mu. @@ -863,6 +874,7 @@ func (b *outlierDetectionBalancer) unejectAddress(addrInfo *addressInfo) { addrInfo.latestEjectionTimestamp = time.Time{} for _, sbw := range addrInfo.sws { sbw.uneject() + channelz.Infof(logger, b.channelzParentID, "Subchannel unejected: %+v", sbw.string()) } } @@ -887,6 +899,15 @@ type addressInfo struct { sws []*subConnWrapper } +func (a *addressInfo) string() string { + res := "[" + for _, sw := range a.sws { + res += sw.string() + } + res += "]" + return res +} + func newAddressInfo() *addressInfo { return &addressInfo{ callCounter: newCallCounter(), diff --git a/xds/internal/balancer/outlierdetection/subconn_wrapper.go b/xds/internal/balancer/outlierdetection/subconn_wrapper.go index 8e25eb788b1d..be631387b2f3 100644 --- a/xds/internal/balancer/outlierdetection/subconn_wrapper.go +++ b/xds/internal/balancer/outlierdetection/subconn_wrapper.go @@ -18,6 +18,7 @@ package outlierdetection import ( + "fmt" "unsafe" "google.golang.org/grpc/balancer" @@ -66,3 +67,7 @@ func (scw *subConnWrapper) uneject() { isEjected: false, }) } + +func (scw *subConnWrapper) string() string { + return fmt.Sprintf("%+v", scw.addresses) +}