New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
xds/internal/xdsclient: Add least request support in xDS #6517
Changes from 4 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -30,6 +30,7 @@ import ( | |
"github.com/golang/protobuf/proto" | ||
"google.golang.org/grpc" | ||
"google.golang.org/grpc/balancer" | ||
"google.golang.org/grpc/balancer/leastrequest" | ||
"google.golang.org/grpc/balancer/roundrobin" | ||
"google.golang.org/grpc/balancer/weightedroundrobin" | ||
"google.golang.org/grpc/internal/envconfig" | ||
|
@@ -41,6 +42,7 @@ import ( | |
v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" | ||
v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" | ||
v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3" | ||
v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3" | ||
v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3" | ||
v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3" | ||
v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3" | ||
|
@@ -53,13 +55,15 @@ func init() { | |
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.pick_first.v3.PickFirst", convertPickFirstProtoToServiceConfig) | ||
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin", convertRoundRobinProtoToServiceConfig) | ||
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.wrr_locality.v3.WrrLocality", convertWRRLocalityProtoToServiceConfig) | ||
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest", convertLeastRequestProtoToServiceConfig) | ||
xdslbregistry.Register("type.googleapis.com/udpa.type.v1.TypedStruct", convertV1TypedStructToServiceConfig) | ||
xdslbregistry.Register("type.googleapis.com/xds.type.v3.TypedStruct", convertV3TypedStructToServiceConfig) | ||
} | ||
|
||
const ( | ||
defaultRingHashMinSize = 1024 | ||
defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M | ||
defaultRingHashMinSize = 1024 | ||
defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M | ||
defaultLeastRequestChoiceCount = 2 | ||
) | ||
|
||
func convertRingHashProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { | ||
|
@@ -177,6 +181,29 @@ func convertWeightedRoundRobinProtoToServiceConfig(rawProto []byte, _ int) (json | |
return makeBalancerConfigJSON(weightedroundrobin.Name, lbCfgJSON), nil | ||
} | ||
|
||
func convertLeastRequestProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { | ||
easwars marked this conversation as resolved.
Show resolved
Hide resolved
|
||
if !envconfig.LeastRequestLB { | ||
return nil, nil | ||
} | ||
lrProto := &v3leastrequestpb.LeastRequest{} | ||
if err := proto.Unmarshal(rawProto, lrProto); err != nil { | ||
return nil, fmt.Errorf("failed to unmarshal resource: %v", err) | ||
} | ||
// "The configuration for the Least Request LB policy is the | ||
// least_request_lb_config field. The field is optional; if not present, | ||
// defaults will be assumed for all of its values." - A48 | ||
var choiceCount uint32 = defaultLeastRequestChoiceCount | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not asking you to change, but just FYI There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Switched. I feel like throughout the codebase and my years on the team, x := y vs. var x type = y has always been preferred for local vars declaration to a non zero value. Noted for future. |
||
if cc := lrProto.GetChoiceCount(); cc != nil { | ||
choiceCount = cc.GetValue() | ||
} | ||
lrCfg := &leastrequest.LBConfig{ChoiceCount: choiceCount} | ||
js, err := json.Marshal(lrCfg) | ||
if err != nil { | ||
return nil, fmt.Errorf("error marshaling JSON for type %T: %v", lrCfg, err) | ||
} | ||
return makeBalancerConfigJSON(leastrequest.Name, js), nil | ||
} | ||
|
||
func convertV1TypedStructToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { | ||
tsProto := &v1xdsudpatypepb.TypedStruct{} | ||
if err := proto.Unmarshal(rawProto, tsProto); err != nil { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -69,6 +69,9 @@ func wrrLocalityBalancerConfig(childPolicy *internalserviceconfig.BalancerConfig | |
} | ||
|
||
func (s) TestConvertToServiceConfigSuccess(t *testing.T) { | ||
defer func(old bool) { envconfig.LeastRequestLB = old }(envconfig.LeastRequestLB) | ||
envconfig.LeastRequestLB = true | ||
easwars marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
const customLBPolicyName = "myorg.MyCustomLeastRequestPolicy" | ||
stub.Register(customLBPolicyName, stub.BalancerFuncs{}) | ||
|
||
|
@@ -78,6 +81,7 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) { | |
wantConfig string // JSON config | ||
rhDisabled bool | ||
pfDisabled bool | ||
lrDisabled bool | ||
}{ | ||
{ | ||
name: "ring_hash", | ||
|
@@ -96,6 +100,21 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) { | |
}, | ||
wantConfig: `[{"ring_hash_experimental": { "minRingSize": 10, "maxRingSize": 100 }}]`, | ||
}, | ||
{ | ||
name: "least_request", | ||
policy: &v3clusterpb.LoadBalancingPolicy{ | ||
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ | ||
{ | ||
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ | ||
TypedConfig: testutils.MarshalAny(&v3leastrequestpb.LeastRequest{ | ||
ChoiceCount: wrapperspb.UInt32(3), | ||
}), | ||
}, | ||
}, | ||
}, | ||
}, | ||
wantConfig: `[{"least_request_experimental": { "choiceCount": 3 }}]`, | ||
}, | ||
{ | ||
name: "pick_first_shuffle", | ||
policy: &v3clusterpb.LoadBalancingPolicy{ | ||
|
@@ -183,7 +202,7 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) { | |
rhDisabled: true, | ||
}, | ||
{ | ||
name: "pick_first_disabled_pf_rr_use_first_supported", | ||
name: "pick_first_enabled_pf_rr_use_pick_first", | ||
policy: &v3clusterpb.LoadBalancingPolicy{ | ||
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ | ||
{ | ||
|
@@ -200,17 +219,16 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) { | |
}, | ||
}, | ||
}, | ||
wantConfig: `[{"round_robin": {}}]`, | ||
pfDisabled: true, | ||
wantConfig: `[{"pick_first": { "shuffleAddressList": true }}]`, | ||
}, | ||
{ | ||
name: "pick_first_enabled_pf_rr_use_pick_first", | ||
name: "least_request_disabled_pf_rr_use_first_supported", | ||
policy: &v3clusterpb.LoadBalancingPolicy{ | ||
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ | ||
{ | ||
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ | ||
TypedConfig: testutils.MarshalAny(&v3pickfirstpb.PickFirst{ | ||
ShuffleAddressList: true, | ||
TypedConfig: testutils.MarshalAny(&v3leastrequestpb.LeastRequest{ | ||
ChoiceCount: wrapperspb.UInt32(32), | ||
}), | ||
}, | ||
}, | ||
|
@@ -221,7 +239,8 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) { | |
}, | ||
}, | ||
}, | ||
wantConfig: `[{"pick_first": { "shuffleAddressList": true }}]`, | ||
wantConfig: `[{"round_robin": {}}]`, | ||
lrDisabled: true, | ||
}, | ||
{ | ||
name: "custom_lb_type_v3_struct", | ||
|
@@ -317,6 +336,10 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) { | |
defer func(old bool) { envconfig.XDSRingHash = old }(envconfig.XDSRingHash) | ||
envconfig.XDSRingHash = false | ||
} | ||
if test.lrDisabled { | ||
defer func(old bool) { envconfig.LeastRequestLB = old }(envconfig.LeastRequestLB) | ||
envconfig.LeastRequestLB = false | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Default value of Why not have a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done :). Good suggestion. |
||
if test.pfDisabled { | ||
defer func(old bool) { envconfig.PickFirstLBConfig = old }(envconfig.PickFirstLBConfig) | ||
envconfig.PickFirstLBConfig = false | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -76,6 +76,8 @@ const ( | |
defaultRingHashMinSize = 1024 | ||
defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M | ||
ringHashSizeUpperBound = 8 * 1024 * 1024 // 8M | ||
|
||
defaultLeastRequestChoiceCount = 2 | ||
easwars marked this conversation as resolved.
Show resolved
Hide resolved
|
||
) | ||
|
||
func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) { | ||
|
@@ -104,6 +106,26 @@ func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (Clu | |
|
||
rhLBCfg := []byte(fmt.Sprintf("{\"minRingSize\": %d, \"maxRingSize\": %d}", minSize, maxSize)) | ||
lbPolicy = []byte(fmt.Sprintf(`[{"ring_hash_experimental": %s}]`, rhLBCfg)) | ||
case v3clusterpb.Cluster_LEAST_REQUEST: | ||
if !envconfig.LeastRequestLB { | ||
return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster) | ||
} | ||
|
||
// "The configuration for the Least Request LB policy is the | ||
// least_request_lb_config field. The field is optional; if not present, | ||
// defaults will be assumed for all of its values." - A48 | ||
lr := cluster.GetLeastRequestLbConfig() | ||
var choiceCount uint32 = defaultLeastRequestChoiceCount | ||
if cc := lr.GetChoiceCount(); cc != nil { | ||
choiceCount = cc.GetValue() | ||
} | ||
// nack if it's < 2, and also add test for it | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please remove the And probably quote the relevant language from the gRFC if that makes sense. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Switched comment to: // "If choice_count < 2, the config will be rejected." - A48 |
||
if choiceCount < 2 { | ||
return ClusterUpdate{}, fmt.Errorf("Cluster_LeastRequestLbConfig.ChoiceCount must be >= 2, got: %v", choiceCount) | ||
} | ||
|
||
lrLBCfg := []byte(fmt.Sprintf("{\"choiceCount\": %d}", choiceCount)) | ||
lbPolicy = []byte(fmt.Sprintf(`[{"least_request_experimental": %s}]`, lrLBCfg)) | ||
default: | ||
return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster) | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Something is incomplete in this sentence:
unary RPCs which don't persist so never any actual RPC counts over iterations
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Changed to "The test performs a Unary RPC, and blocks until the RPC returns, and then makes the next RPC. Thus, over iterations, no RPC counts are present. This causes Least request's randomness of indexes to sample to converge onto a round robin distribution per locality. Thus, expect the same distribution as round robin above."