Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

xdsclient: use dataplane authority for virtualhost lookup #6997

Merged
merged 14 commits into from Feb 28, 2024
69 changes: 69 additions & 0 deletions test/xds/xds_client_federation_test.go
Expand Up @@ -140,6 +140,75 @@ func (s) TestClientSideFederation(t *testing.T) {
}
}

// TestClientSideFederationWithTDOM tests that federation is supported with new
// xdstp style names with TD Open Mesh (TDOM) Authority. TD Open Mesh Authority
// currently only support new style for LDS. So, this test only tests the new
// style for LDS resources and the old style for other resources.
func (s) TestClientSideFederationWithTDOM(t *testing.T) {
// Start a management server as TDOM authority.
const tdomAuth = "traffic-director-global.xds.googleapis.com"
tdomAuthServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{})
if err != nil {
t.Fatalf("Failed to spin up the xDS management server: %v", err)
}
t.Cleanup(tdomAuthServer.Stop)

// Create a bootstrap file in a temporary directory.
nodeID := uuid.New().String()
bootstrapContents, err := bootstrap.Contents(bootstrap.Options{
NodeID: nodeID,
ServerURI: tdomAuthServer.Address,
ClientDefaultListenerResourceNameTemplate: fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/%%s", tdomAuth),
Authorities: map[string]string{tdomAuth: tdomAuthServer.Address},
})
if err != nil {
t.Fatalf("Failed to create bootstrap file: %v", err)
}

resolverBuilder := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))
resolver, err := resolverBuilder(bootstrapContents)
if err != nil {
t.Fatalf("Failed to create xDS resolver for testing: %v", err)
}
server := stubserver.StartTestService(t, nil)
defer server.Stop()

const serviceName = "my-service-client-side-xds"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This string looks like it would have worked before your changes. Does this test fail without your changes to xds_resolver.go?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed. Now the serviceName has 2 special chars - , and /.

// LDS is new style name.
ldsName := fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/%s", tdomAuth, serviceName)
// All other resources are with old style name.
rdsName := "route-" + serviceName
cdsName := "cluster-" + serviceName
edsName := "endpoints-" + serviceName
arvindbr8 marked this conversation as resolved.
Show resolved Hide resolved

resourceUpdate := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)},
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, serviceName, cdsName)},
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
SkipValidation: true,
}

ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := tdomAuthServer.Update(ctx, resourceUpdate); err != nil {
t.Fatal(err)
}

// Create a ClientConn and make a successful RPC.
cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
if err != nil {
t.Fatalf("failed to dial local test server: %v", err)
}
defer cc.Close()

client := testgrpc.NewTestServiceClient(cc)
if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
t.Fatalf("rpc EmptyCall() failed: %v", err)
}
}

// TestFederation_UnknownAuthorityInDialTarget tests the case where a ClientConn
// is created with a dial target containing an authority which is not specified
// in the bootstrap configuration. The test verifies that RPCs on the ClientConn
Expand Down
14 changes: 12 additions & 2 deletions xds/internal/resolver/xds_resolver.go
Expand Up @@ -119,6 +119,7 @@
endpoint = target.URL.Opaque
}
endpoint = strings.TrimPrefix(endpoint, "/")
r.dataplaneAuthority = endpoint
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This isn't the "last path component". If you have xds:///foo/bar then the string here is "foo/bar".

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for bringing this up. Just by intuition I assumed that the dataplane authority (or the virtual host domain name to lookup for) should be the endpoint from the dial target.

I brought this up with the gRPC xDS group and Mark has a patchset to the A47 gRFC to clarify this.

Given that I think this is the expected behavior.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree they should be the same, but the docs you copied said it was different than you were using. (And the docs did say they were the same.)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

:) yes, comment needs updating.. I'll fix the comment once there is an LGTM on the doc update

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've updated excerpt used in the docstring. PTAL :) @ejona86

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see "any remaining '/' characters will be percent-encoded" implemented here.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did some investigation and it seems like we are percentage encoding everything in the URI path except for /s intentionally (see here). I have opened #7002 to track the investigation.

I've updated the PR to only change the virtualhost lookup logic and will track the % encoding question as a followup.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was able to track #7002 and close it. Basically xDS URI targets use the default way to determine channel authority which is to %encoding the path component.

And yes I verified that we %encoding /s after fc0aa46 was merged

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Currently, there is an open question about whether or not to use the %-encoded authority to match the virtual host domain.

per @markdroth -
C-core currently uses the %-encoded authority to match the virtual host domain.

@ejona86 , @dfawley -- Do you have a strong preference or shall we be guided by the existing implementation?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Per @ejona86 --
We will send the percent-encoded form on the wire, so it seems appropriate to match against the same here. That's what the server would match against.

In general, the fewer different representations of authority, the fewer chances for subtle bugs.

r.ldsResourceName = bootstrap.PopulateResourceTemplate(template, endpoint)
r.listenerWatcher = newListenerWatcher(r.ldsResourceName, r)
return r, nil
Expand Down Expand Up @@ -190,6 +191,15 @@
serializer *grpcsync.CallbackSerializer
serializerCancel context.CancelFunc

// Per [A47] the authority used for the data plane connections (which is
// also used to select the VirtualHost within the xDS RouteConfiguration)
// will continue to be the last path component of the xds URI used to create
// the gRPC channel (i.e., the part following the last / character, or the
// entire path if the path contains no / character).
//
// [A47]: https://github.com/grpc/proposal/blob/master/A47-xds-federation.md
dataplaneAuthority string

ldsResourceName string
listenerWatcher *listenerWatcher
listenerUpdateRecvd bool
Expand Down Expand Up @@ -413,9 +423,9 @@
}

func (r *xdsResolver) applyRouteConfigUpdate(update xdsresource.RouteConfigUpdate) {
matchVh := xdsresource.FindBestMatchingVirtualHost(r.ldsResourceName, update.VirtualHosts)
matchVh := xdsresource.FindBestMatchingVirtualHost(r.dataplaneAuthority, update.VirtualHosts)
if matchVh == nil {
r.onError(fmt.Errorf("no matching virtual host found for %q", r.ldsResourceName))
r.onError(fmt.Errorf("no matching virtual host found for %q", r.dataplaneAuthority))

Check warning on line 428 in xds/internal/resolver/xds_resolver.go

View check run for this annotation

Codecov / codecov/patch

xds/internal/resolver/xds_resolver.go#L428

Added line #L428 was not covered by tests
return
}
r.currentRouteConfig = update
Expand Down