Upgrade k8s.io/* to v0.27.5, sigs.k8s.io/controller-runtime to …

…`v0.15.2` (#8245)

* Update k8s.io/* to v0.27.5

* Update service.kubernetes.io/topology-aware-hints annotation

* update sigs.k8s.io/controller-runtime to v0.15.0

* Adapt deprecated pkg/config/v1alpha1 types

* Adapt "Refactor source/handler/predicate packages to remove dep injection"

Adapt `Refactor source/handler/predicate packages to remove dep injection`

* Use `WatchesRawSource` instead of `Watches`

* Adapt `handler.EnqueueRequestForOwner` signature

* Use `ByObject` instead of `SelectorsByObject`

* Remove cache.BuilderWithOptions and use Cache instead

* Remove `DefaultSelector` and use `DefaultFieldSelector` and `DefaultLabelSelector`

* Remove `cache.Options.Namespace` and use `cache.Options.Namespaces` slice

* Rename `cache.Options.Resync` to `SyncPeriod`

* Adapt `client/apiutil.NewDynamicRESTMapper` signature and add `*http.Client` as parameter

* Move cluster.NewClientFunc to client.NewClientFunc. Adapt NewClientFunc signature.

* Adapt handler.EventHandler and handler.Funcs and add context.Context as the first parameter for every method.

* Adapt webhook admission to accept `mgr.GetScheme()` as first argument.

* Adapt webhook/admission custom Validator and CustomValidator interfaces to return warnings

* Adapt webhook.Server interface.

Earlier `webhook.Server` was a struct. All the struct fields like `Port`, `CertDir` is being moved to `Options` struct.

* Adapt deprecated MultiNamespacedCacheBuilder

* Adapt removal of client.NewDelegatingClient constructor

* Make generate

* Drop InjectDecoder

* use Result.Message instead of Result.Reason

* Adapt fake client deletionTimestamp behavior

* Adapt fake client status handling

- Add `WithStatusSubresource` option and pass resources with status subresource

* Adapt Kublet's ContainerRuntimeEndpoint flag

* Add singular name for the rest of types

* Fix test

* Fix make check

* Adapt deprecated ClientDisableCacheFor and use client.CacheOptions instead

* Add Pod Nodename indexer

* Update env test

* Use `old` restmapper

- Latest restmapper implementation doesnot handle cases where an API Group / GroupVersion is not present in the system. In such cases Clients expect No{Kind,Resource}MatchErrors.
- Revert this commit once the upstream fix is available.

* Use `old` code-generator scripts

* Make clean and Make generate

* Address PR review feedback

* Update to latest dependencies

* Remove obsolete dashboards

These metrics are not registered by controller-runtime anymore due to high cardinality

* Update envtest version in tests

Use the same version as the envtest package

* Resolve TODOs for k8s 1.27

* Run `make generate`

* Adapt unit tests for dashboard change

* [PR feedback] Adapt deprecated manager options

- manager.Options.Port
- manager.Options.Host
- manager.Options.CertDir
- manager.Options.TLSOpts

Update options_test.go

* [PR feedback] Upgrade controller-runtime to v0.15.2

* Address PR review feedback

* Update docURLTemplate url to v1.27

* Make generate

* Address PR review feedback

* Remove uncached object from Config, It can be set using `clientoptions.Cache.DisableFor`

Co-Authored-By: Sonu Kumar Singh <sonu.kumar.singh02@sap.com>

* Set `clientOptions.Cache.Reader` to nil for creating uncached client

Co-Authored-By: Sonu Kumar Singh <sonu.kumar.singh02@sap.com>

* Address Review

* Address Review

---------

Co-authored-by: Shafeeque E S <shafeeque.e.s@sap.com>
Co-authored-by: Sonu Kumar Singh <sonu.kumar.singh02@sap.com>

charts/gardener/operator/templates/customresouredefintion.yaml

@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.3
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.11.4
   name: gardens.operator.gardener.cloud
 spec:
   group: operator.gardener.cloud

cmd/gardener-admission-controller/app/app.go

@@ -33,6 +33,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	controllerwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	"github.com/gardener/gardener/pkg/admissioncontroller/apis/config"
 	"github.com/gardener/gardener/pkg/admissioncontroller/webhook"
@@ -109,14 +110,16 @@ func run(ctx context.Context, log logr.Logger, cfg *config.AdmissionControllerCo
 		Scheme:                  kubernetes.GardenScheme,
 		GracefulShutdownTimeout: pointer.Duration(5 * time.Second),
 
-		Host:    cfg.Server.Webhooks.BindAddress,
-		Port:    cfg.Server.Webhooks.Port,
-		CertDir: cfg.Server.Webhooks.TLS.ServerCertDir,
-
 		HealthProbeBindAddress: net.JoinHostPort(cfg.Server.HealthProbes.BindAddress, strconv.Itoa(cfg.Server.HealthProbes.Port)),
 		MetricsBindAddress:     net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
 
 		LeaderElection: false,
+
+		WebhookServer: controllerwebhook.NewServer(controllerwebhook.Options{
+			Host:    cfg.Server.Webhooks.BindAddress,
+			Port:    cfg.Server.Webhooks.Port,
+			CertDir: cfg.Server.Webhooks.TLS.ServerCertDir,
+		}),
 	})
 	if err != nil {
 		return err

cmd/gardener-apiserver/.import-restrictions

@@ -6,4 +6,4 @@ rules:
   # make sure apiserver can build without these packages
   - github.com/gardener/gardener/charts
   - github.com/gardener/gardener/pkg/operation
-  - github.com/gardener/gardener/third_party
+  - github.com/gardener/gardener/third_party/apiserver

cmd/gardener-controller-manager/app/app.go

@@ -32,7 +32,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	controllerconfigv1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
+	controllerconfig "sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -137,7 +137,7 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ControllerManagerConf
 		LeaseDuration:                 &cfg.LeaderElection.LeaseDuration.Duration,
 		RenewDeadline:                 &cfg.LeaderElection.RenewDeadline.Duration,
 		RetryPeriod:                   &cfg.LeaderElection.RetryPeriod.Duration,
-		Controller: controllerconfigv1alpha1.ControllerConfigurationSpec{
+		Controller: controllerconfig.Controller{
 			RecoverPanic: pointer.Bool(true),
 		},
 	})

cmd/gardener-operator/app/app.go

@@ -35,10 +35,11 @@ import (
 	"k8s.io/utils/clock"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	controllerconfigv1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
+	controllerconfig "sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	controllerwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook"
 	"github.com/gardener/gardener/extensions/pkg/webhook/certificates"
@@ -124,9 +125,6 @@ func run(ctx context.Context, log logr.Logger, cfg *config.OperatorConfiguration
 		Scheme:                  operatorclient.RuntimeScheme,
 		GracefulShutdownTimeout: pointer.Duration(5 * time.Second),
 
-		Host:                   cfg.Server.Webhooks.BindAddress,
-		Port:                   cfg.Server.Webhooks.Port,
-		CertDir:                "/tmp/gardener-operator-cert",
 		HealthProbeBindAddress: net.JoinHostPort(cfg.Server.HealthProbes.BindAddress, strconv.Itoa(cfg.Server.HealthProbes.Port)),
 		MetricsBindAddress:     net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
 
@@ -138,9 +136,15 @@ func run(ctx context.Context, log logr.Logger, cfg *config.OperatorConfiguration
 		LeaseDuration:                 &cfg.LeaderElection.LeaseDuration.Duration,
 		RenewDeadline:                 &cfg.LeaderElection.RenewDeadline.Duration,
 		RetryPeriod:                   &cfg.LeaderElection.RetryPeriod.Duration,
-		Controller: controllerconfigv1alpha1.ControllerConfigurationSpec{
+		Controller: controllerconfig.Controller{
 			RecoverPanic: pointer.Bool(true),
 		},
+
+		WebhookServer: controllerwebhook.NewServer(controllerwebhook.Options{
+			Host:    cfg.Server.Webhooks.BindAddress,
+			Port:    cfg.Server.Webhooks.Port,
+			CertDir: "/tmp/gardener-operator-cert",
+		}),
 	})
 	if err != nil {
 		return err

cmd/gardener-resource-manager/app/app.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	goruntime "runtime"
 	"strconv"
@@ -37,16 +38,16 @@ import (
 	"k8s.io/component-base/version/verflag"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
-	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
-	controllerconfigv1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
+	controllerconfig "sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	controllerwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	"github.com/gardener/gardener/cmd/gardener-resource-manager/app/bootstrappers"
+	"github.com/gardener/gardener/pkg/api/indexer"
 	"github.com/gardener/gardener/pkg/client/kubernetes"
 	"github.com/gardener/gardener/pkg/controllerutils"
 	"github.com/gardener/gardener/pkg/controllerutils/routes"
@@ -56,6 +57,7 @@ import (
 	resourcemanagerclient "github.com/gardener/gardener/pkg/resourcemanager/client"
 	"github.com/gardener/gardener/pkg/resourcemanager/controller"
 	"github.com/gardener/gardener/pkg/resourcemanager/webhook"
+	thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil"
 )
 
 // Name is a const for the name of this component.
@@ -146,9 +148,6 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ResourceManagerConfig
 		Namespace:               *cfg.SourceClientConnection.Namespace,
 		SyncPeriod:              &cfg.SourceClientConnection.CacheResyncPeriod.Duration,
 
-		Host:                   cfg.Server.Webhooks.BindAddress,
-		Port:                   cfg.Server.Webhooks.Port,
-		CertDir:                cfg.Server.Webhooks.TLS.ServerCertDir,
 		HealthProbeBindAddress: net.JoinHostPort(cfg.Server.HealthProbes.BindAddress, strconv.Itoa(cfg.Server.HealthProbes.Port)),
 		MetricsBindAddress:     net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
 
@@ -160,9 +159,15 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ResourceManagerConfig
 		LeaseDuration:                 &cfg.LeaderElection.LeaseDuration.Duration,
 		RenewDeadline:                 &cfg.LeaderElection.RenewDeadline.Duration,
 		RetryPeriod:                   &cfg.LeaderElection.RetryPeriod.Duration,
-		Controller: controllerconfigv1alpha1.ControllerConfigurationSpec{
+		Controller: controllerconfig.Controller{
 			RecoverPanic: pointer.Bool(true),
 		},
+
+		WebhookServer: controllerwebhook.NewServer(controllerwebhook.Options{
+			Host:    cfg.Server.Webhooks.BindAddress,
+			Port:    cfg.Server.Webhooks.Port,
+			CertDir: cfg.Server.Webhooks.TLS.ServerCertDir,
+		}),
 	})
 	if err != nil {
 		return err
@@ -205,27 +210,32 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ResourceManagerConfig
 
 			// use dynamic rest mapper for target cluster, which will automatically rediscover resources on NoMatchErrors
 			// but is rate-limited to not issue to many discovery calls (rate-limit shared across all reconciliations)
-			opts.MapperProvider = func(config *rest.Config) (meta.RESTMapper, error) {
-				return apiutil.NewDynamicRESTMapper(
+			opts.MapperProvider = func(config *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) {
+				// TODO(ary1992): The new rest mapper implementation doesn't return a NoKindMatchError but a ErrGroupDiscoveryFailed
+				// when an API GroupVersion is not present in the cluster. Remove the old restmapper usage once the upstream issue
+				// (https://github.com/kubernetes-sigs/controller-runtime/pull/2425) is fixed.
+				return thirdpartyapiutil.NewDynamicRESTMapper(
 					config,
-					apiutil.WithLazyDiscovery,
-					apiutil.WithLimiter(rate.NewLimiter(rate.Every(1*time.Minute), 1)), // rediscover at maximum every minute
+					thirdpartyapiutil.WithLazyDiscovery,
+					thirdpartyapiutil.WithLimiter(rate.NewLimiter(rate.Every(1*time.Minute), 1)), // rediscover at maximum every minute
 				)
 			}
 
-			opts.Namespace = *cfg.TargetClientConnection.Namespace
+			opts.Cache.Namespaces = []string{*cfg.TargetClientConnection.Namespace}
 			opts.SyncPeriod = &cfg.TargetClientConnection.CacheResyncPeriod.Duration
 
 			if *cfg.TargetClientConnection.DisableCachedClient {
-				opts.NewClient = func(_ cache.Cache, config *rest.Config, opts client.Options, _ ...client.Object) (client.Client, error) {
+				opts.NewClient = func(config *rest.Config, opts client.Options) (client.Client, error) {
 					return client.New(config, opts)
 				}
 			}
 
-			opts.ClientDisableCacheFor = []client.Object{
-				&corev1.Event{},
-				&eventsv1beta1.Event{},
-				&eventsv1.Event{},
+			opts.Client.Cache = &client.CacheOptions{
+				DisableFor: []client.Object{
+					&corev1.Event{},
+					&eventsv1beta1.Event{},
+					&eventsv1.Event{},
+				},
 			}
 		})
 		if err != nil {
@@ -243,6 +253,11 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ResourceManagerConfig
 		}
 	}
 
+	log.Info("Adding field indexes to informers")
+	if err := addAllFieldIndexes(ctx, targetCluster.GetFieldIndexer()); err != nil {
+		return fmt.Errorf("failed adding indexes: %w", err)
+	}
+
 	log.Info("Adding webhook handlers to manager")
 	if err := webhook.AddToManager(mgr, mgr, targetCluster, cfg); err != nil {
 		return fmt.Errorf("failed adding webhook handlers to manager: %w", err)
@@ -260,3 +275,16 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ResourceManagerConfig
 	log.Info("Starting manager")
 	return mgr.Start(ctx)
 }
+
+func addAllFieldIndexes(ctx context.Context, i client.FieldIndexer) error {
+	for _, fn := range []func(context.Context, client.FieldIndexer) error{
+		// core/v1 API group
+		indexer.AddPodNodeName,
+	} {
+		if err := fn(ctx, i); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/gardener-scheduler/app/app.go

@@ -35,7 +35,7 @@ import (
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	controllerconfigv1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
+	controllerconfig "sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -123,13 +123,13 @@ func run(ctx context.Context, log logr.Logger, cfg *config.SchedulerConfiguratio
 		HealthProbeBindAddress: net.JoinHostPort(cfg.Server.HealthProbes.BindAddress, strconv.Itoa(cfg.Server.HealthProbes.Port)),
 		MetricsBindAddress:     net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
 
-		NewCache: cache.BuilderWithOptions(cache.Options{
-			SelectorsByObject: map[client.Object]cache.ObjectSelector{
+		Cache: cache.Options{
+			ByObject: map[client.Object]cache.ByObject{
 				&corev1.ConfigMap{}: {
 					Label: labels.NewSelector().Add(utils.MustNewRequirement(v1beta1constants.SchedulingPurpose, selection.Equals, v1beta1constants.SchedulingPurposeRegionConfig)),
 				},
 			},
-		}),
+		},
 		LeaderElection:                cfg.LeaderElection.LeaderElect,
 		LeaderElectionResourceLock:    cfg.LeaderElection.ResourceLock,
 		LeaderElectionID:              cfg.LeaderElection.ResourceName,
@@ -138,7 +138,7 @@ func run(ctx context.Context, log logr.Logger, cfg *config.SchedulerConfiguratio
 		LeaseDuration:                 &cfg.LeaderElection.LeaseDuration.Duration,
 		RenewDeadline:                 &cfg.LeaderElection.RenewDeadline.Duration,
 		RetryPeriod:                   &cfg.LeaderElection.RetryPeriod.Duration,
-		Controller: controllerconfigv1alpha1.ControllerConfigurationSpec{
+		Controller: controllerconfig.Controller{
 			RecoverPanic: pointer.Bool(true),
 		},
 	})