Skip to content

Commit

Permalink
Add logger to outlierDetection LB
Browse files Browse the repository at this point in the history
  • Loading branch information
s-matyukevich committed Mar 22, 2023
1 parent a02aae6 commit 1b8d781
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 12 deletions.
45 changes: 33 additions & 12 deletions xds/internal/balancer/outlierdetection/balancer.go
Expand Up @@ -35,6 +35,7 @@ import (
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/internal/balancer/gracefulswitch"
"google.golang.org/grpc/internal/buffer"
"google.golang.org/grpc/internal/channelz"
"google.golang.org/grpc/internal/envconfig"
"google.golang.org/grpc/internal/grpclog"
"google.golang.org/grpc/internal/grpcrand"
Expand Down Expand Up @@ -62,13 +63,14 @@ type bb struct{}

func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
b := &outlierDetectionBalancer{
cc: cc,
closed: grpcsync.NewEvent(),
done: grpcsync.NewEvent(),
addrs: make(map[string]*addressInfo),
scWrappers: make(map[balancer.SubConn]*subConnWrapper),
scUpdateCh: buffer.NewUnbounded(),
pickerUpdateCh: buffer.NewUnbounded(),
cc: cc,
closed: grpcsync.NewEvent(),
done: grpcsync.NewEvent(),
addrs: make(map[string]*addressInfo),
scWrappers: make(map[balancer.SubConn]*subConnWrapper),
scUpdateCh: buffer.NewUnbounded(),
pickerUpdateCh: buffer.NewUnbounded(),
channelzParentID: bOpts.ChannelzParentID,
}
b.logger = prefixLogger(b)
b.logger.Infof("Created")
Expand Down Expand Up @@ -159,10 +161,11 @@ type outlierDetectionBalancer struct {
// to suppress redundant picker updates.
recentPickerNoop bool

closed *grpcsync.Event
done *grpcsync.Event
cc balancer.ClientConn
logger *grpclog.PrefixLogger
closed *grpcsync.Event
done *grpcsync.Event
cc balancer.ClientConn
logger *grpclog.PrefixLogger
channelzParentID *channelz.Identifier

// childMu guards calls into child (to uphold the balancer.Balancer API
// guarantee of synchronous calls).
Expand Down Expand Up @@ -813,7 +816,11 @@ func (b *outlierDetectionBalancer) successRateAlgorithm() {
return
}
successRate := float64(bucket.numSuccesses) / float64(bucket.numSuccesses+bucket.numFailures)
if successRate < (mean - stddev*(float64(ejectionCfg.StdevFactor)/1000)) {
requiredSuccessRate := mean - stddev*(float64(ejectionCfg.StdevFactor)/1000)
if successRate < requiredSuccessRate {
channelz.Infof(logger, b.channelzParentID, "SuccessRate algorithm detected outlier: %s. "+
"Parameters: successRate=%f, mean=%f, stddev=%f, requiredSuccessRate=%f",
addrInfo.string(), successRate, mean, stddev, requiredSuccessRate)
if uint32(grpcrand.Int31n(100)) < ejectionCfg.EnforcementPercentage {
b.ejectAddress(addrInfo)
}
Expand All @@ -840,6 +847,8 @@ func (b *outlierDetectionBalancer) failurePercentageAlgorithm() {
}
failurePercentage := (float64(bucket.numFailures) / float64(bucket.numSuccesses+bucket.numFailures)) * 100
if failurePercentage > float64(b.cfg.FailurePercentageEjection.Threshold) {
channelz.Infof(logger, b.channelzParentID, "FailurePercentage algorithm detected outlier: %s, failurePercentage=%f",
addrInfo.string(), failurePercentage)
if uint32(grpcrand.Int31n(100)) < ejectionCfg.EnforcementPercentage {
b.ejectAddress(addrInfo)
}
Expand All @@ -854,7 +863,9 @@ func (b *outlierDetectionBalancer) ejectAddress(addrInfo *addressInfo) {
addrInfo.ejectionTimeMultiplier++
for _, sbw := range addrInfo.sws {
sbw.eject()
channelz.Infof(logger, b.channelzParentID, "Subchannel ejected: %+v", sbw.string())
}

}

// Caller must hold b.mu.
Expand All @@ -863,6 +874,7 @@ func (b *outlierDetectionBalancer) unejectAddress(addrInfo *addressInfo) {
addrInfo.latestEjectionTimestamp = time.Time{}
for _, sbw := range addrInfo.sws {
sbw.uneject()
channelz.Infof(logger, b.channelzParentID, "Subchannel unejected: %+v", sbw.string())
}
}

Expand All @@ -887,6 +899,15 @@ type addressInfo struct {
sws []*subConnWrapper
}

func (a *addressInfo) string() string {
res := "["
for _, sw := range a.sws {
res += sw.string()
}
res += "]"
return res
}

func newAddressInfo() *addressInfo {
return &addressInfo{
callCounter: newCallCounter(),
Expand Down
5 changes: 5 additions & 0 deletions xds/internal/balancer/outlierdetection/subconn_wrapper.go
Expand Up @@ -18,6 +18,7 @@
package outlierdetection

import (
"fmt"
"unsafe"

"google.golang.org/grpc/balancer"
Expand Down Expand Up @@ -66,3 +67,7 @@ func (scw *subConnWrapper) uneject() {
isEjected: false,
})
}

func (scw *subConnWrapper) string() string {
return fmt.Sprintf("%+v", scw.addresses)
}

0 comments on commit 1b8d781

Please sign in to comment.