Upgrade k8s.io/* to v0.28, sigs.k8s.io/controller-runtime to `v…

…0.16` (#8464)

* Vendor k8s.io/* to v0.28.2

* Vendor sigs.k8s.io/controller-runtime to v0.16.2

* Vendor sigs.k8s.io/controller-tools v0.13.0

* Vendor new envtest

* Adapt `MetricsBindAddress` to` Metrics` and `Namespace` to `cache.DefaultNamespaces`

* Adapt `Port`, `Host` and `CertDir` to webhookServer.Options.{`Port`, `Host`,`CertDir`}

* Adapt removed `ClientDisableCacheFor`

* Adapt function signature of `GetInformer` and `GetInformerForKind``

* Adapt profiling

* Fix test

* Adapt `apiserver` changes

* Drop `leader-election-resource-lock` flag for extensions cause only possible value now is leases

For gardener it will be done in separate PR

* No need to add newCache function for secrets and serviceAccounts

* Run `make revendor` and `make generate`

* update todo

charts/gardener/operator/templates/customresouredefintion.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.4
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: gardens.operator.gardener.cloud
 spec:
   group: operator.gardener.cloud

charts/gardener/provider-local/templates/deployment.yaml

@@ -42,7 +42,6 @@ spec:
         image: {{ .Values.image }}
         imagePullPolicy: {{ .Values.imagePullPolicy }}
         args:
-        - --leader-election-resource-lock={{ .Values.leaderElection.resourceLock }}
         - --controlplane-max-concurrent-reconciles={{ .Values.controllers.controlplane.concurrentSyncs }}
         - --dnsrecord-max-concurrent-reconciles={{ .Values.controllers.dnsrecord.concurrentSyncs }}
         # Writing to /etc/hosts is only used for local development, useless when running in pod.

charts/gardener/provider-local/values.yaml

@@ -68,9 +68,6 @@ webhookConfig:
 metricsPort: 8080
 healthPort: 8081
 
-leaderElection:
-  resourceLock: leases
-
 ## settings for metrics, e.g. scraping by seed-prometheus
 ##
 metrics:

cmd/gardener-admission-controller/app/app.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	goruntime "runtime"
 	"strconv"
@@ -33,6 +34,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	controllerwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	"github.com/gardener/gardener/pkg/admissioncontroller/apis/config"
@@ -104,14 +106,25 @@ func run(ctx context.Context, log logr.Logger, cfg *config.AdmissionControllerCo
 		return err
 	}
 
+	var extraHandlers map[string]http.Handler
+	if cfg.Debugging != nil && cfg.Debugging.EnableProfiling {
+		extraHandlers = routes.ProfilingHandlers
+		if cfg.Debugging.EnableContentionProfiling {
+			goruntime.SetBlockProfileRate(1)
+		}
+	}
+
 	log.Info("Setting up manager")
 	mgr, err := manager.New(restConfig, manager.Options{
 		Logger:                  log,
 		Scheme:                  kubernetes.GardenScheme,
 		GracefulShutdownTimeout: pointer.Duration(5 * time.Second),
 
 		HealthProbeBindAddress: net.JoinHostPort(cfg.Server.HealthProbes.BindAddress, strconv.Itoa(cfg.Server.HealthProbes.Port)),
-		MetricsBindAddress:     net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
+		Metrics: metricsserver.Options{
+			BindAddress:   net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
+			ExtraHandlers: extraHandlers,
+		},
 
 		LeaderElection: false,
 
@@ -125,15 +138,6 @@ func run(ctx context.Context, log logr.Logger, cfg *config.AdmissionControllerCo
 		return err
 	}
 
-	if cfg.Debugging != nil && cfg.Debugging.EnableProfiling {
-		if err := (routes.Profiling{}).AddToManager(mgr); err != nil {
-			return fmt.Errorf("failed adding profiling handlers to manager: %w", err)
-		}
-		if cfg.Debugging.EnableContentionProfiling {
-			goruntime.SetBlockProfileRate(1)
-		}
-	}
-
 	log.Info("Setting up health check endpoints")
 	if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil {
 		return err

cmd/gardener-apiserver/app/gardener_apiserver.go

@@ -257,9 +257,18 @@ func (o *Options) config(kubeAPIServerConfig *rest.Config, kubeClient *kubernete
 		}, nil
 	}
 
+	gardenerKubeClient, err := kubernetes.NewForConfig(gardenerAPIServerConfig.ClientConfig)
+	if err != nil {
+		return nil, err
+	}
+	gardenerDynamicClient, err := dynamic.NewForConfig(gardenerAPIServerConfig.ClientConfig)
+	if err != nil {
+		return nil, err
+	}
+
 	if initializers, err := o.Recommended.ExtraAdmissionInitializers(gardenerAPIServerConfig); err != nil {
 		return apiConfig, err
-	} else if err := o.Recommended.Admission.ApplyTo(&gardenerAPIServerConfig.Config, gardenerAPIServerConfig.SharedInformerFactory, gardenerAPIServerConfig.ClientConfig, features.DefaultFeatureGate, initializers...); err != nil {
+	} else if err := o.Recommended.Admission.ApplyTo(&gardenerAPIServerConfig.Config, gardenerAPIServerConfig.SharedInformerFactory, gardenerKubeClient, gardenerDynamicClient, features.DefaultFeatureGate, initializers...); err != nil {
 		return apiConfig, err
 	}
 
@@ -377,11 +386,6 @@ func (o *Options) ApplyTo(config *apiserver.Config) error {
 	gardenerAPIServerConfig := config.GenericConfig
 
 	gardenerVersion := version.Get()
-	// There is an upstream bug which requires OpenAPIConfig and OpenAPIV3Config to be available. Otherwise, there will be a nil pointer exception.
-	// ref: https://github.com/kubernetes/apiserver/blob/b9faf8358c6ec35a3ca611244052efcc394c8e44/pkg/server/genericapiserver.go#L962
-	gardenerAPIServerConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(openapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(api.Scheme))
-	gardenerAPIServerConfig.OpenAPIConfig.Info.Title = "Gardener"
-	gardenerAPIServerConfig.OpenAPIConfig.Info.Version = gardenerVersion.GitVersion
 	gardenerAPIServerConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(openapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(api.Scheme))
 	gardenerAPIServerConfig.OpenAPIV3Config.Info.Title = "Gardener"
 	gardenerAPIServerConfig.OpenAPIV3Config.Info.Version = gardenerVersion.GitVersion
@@ -438,9 +442,5 @@ func (o *Options) ApplyTo(config *apiserver.Config) error {
 		),
 	}
 
-	if err := o.Recommended.Etcd.Complete(config.GenericConfig.StorageObjectCountTracker, config.GenericConfig.DrainedNotify(), config.GenericConfig.AddPostStartHook); err != nil {
-		return err
-	}
-
 	return o.Recommended.Etcd.ApplyWithStorageFactoryTo(storageFactory, &gardenerAPIServerConfig.Config)
 }

cmd/gardener-controller-manager/app/app.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	goruntime "runtime"
 	"strconv"
@@ -36,6 +37,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	"github.com/gardener/gardener/cmd/gardener-controller-manager/app/bootstrappers"
 	"github.com/gardener/gardener/pkg/api/indexer"
@@ -120,14 +122,25 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ControllerManagerConf
 		return err
 	}
 
+	var extraHandlers map[string]http.Handler
+	if cfg.Debugging != nil && cfg.Debugging.EnableProfiling {
+		extraHandlers = routes.ProfilingHandlers
+		if cfg.Debugging.EnableContentionProfiling {
+			goruntime.SetBlockProfileRate(1)
+		}
+	}
+
 	log.Info("Setting up manager")
 	mgr, err := manager.New(restConfig, manager.Options{
 		Logger:                  log,
 		Scheme:                  kubernetes.GardenScheme,
 		GracefulShutdownTimeout: pointer.Duration(5 * time.Second),
 
 		HealthProbeBindAddress: net.JoinHostPort(cfg.Server.HealthProbes.BindAddress, strconv.Itoa(cfg.Server.HealthProbes.Port)),
-		MetricsBindAddress:     net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
+		Metrics: metricsserver.Options{
+			BindAddress:   net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
+			ExtraHandlers: extraHandlers,
+		},
 
 		LeaderElection:                cfg.LeaderElection.LeaderElect,
 		LeaderElectionResourceLock:    cfg.LeaderElection.ResourceLock,
@@ -145,15 +158,6 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ControllerManagerConf
 		return err
 	}
 
-	if cfg.Debugging != nil && cfg.Debugging.EnableProfiling {
-		if err := (routes.Profiling{}).AddToManager(mgr); err != nil {
-			return fmt.Errorf("failed adding profiling handlers to manager: %w", err)
-		}
-		if cfg.Debugging.EnableContentionProfiling {
-			goruntime.SetBlockProfileRate(1)
-		}
-	}
-
 	log.Info("Setting up health check endpoints")
 	if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil {
 		return err

cmd/gardener-extension-provider-local/app/app.go

@@ -35,7 +35,6 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
-	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -88,14 +87,13 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command {
 	var (
 		restOpts = &extensionscmdcontroller.RESTOptions{}
 		mgrOpts  = &extensionscmdcontroller.ManagerOptions{
-			LeaderElection:             true,
-			LeaderElectionResourceLock: resourcelock.LeasesResourceLock,
-			LeaderElectionID:           extensionscmdcontroller.LeaderElectionNameID(local.Name),
-			LeaderElectionNamespace:    os.Getenv("LEADER_ELECTION_NAMESPACE"),
-			WebhookServerPort:          443,
-			WebhookCertDir:             "/tmp/gardener-extensions-cert",
-			MetricsBindAddress:         ":8080",
-			HealthBindAddress:          ":8081",
+			LeaderElection:          true,
+			LeaderElectionID:        extensionscmdcontroller.LeaderElectionNameID(local.Name),
+			LeaderElectionNamespace: os.Getenv("LEADER_ELECTION_NAMESPACE"),
+			WebhookServerPort:       443,
+			WebhookCertDir:          "/tmp/gardener-extensions-cert",
+			MetricsBindAddress:      ":8080",
+			HealthBindAddress:       ":8081",
 		}
 		generalOpts = &extensionscmdcontroller.GeneralOptions{}
 

cmd/gardener-operator/app/app.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	goruntime "runtime"
 	"strconv"
@@ -39,6 +40,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	controllerwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	"github.com/gardener/gardener/cmd/gardener-operator/app/bootstrappers"
@@ -120,14 +122,25 @@ func run(ctx context.Context, log logr.Logger, cfg *config.OperatorConfiguration
 		return err
 	}
 
+	var extraHandlers map[string]http.Handler
+	if cfg.Debugging != nil && cfg.Debugging.EnableProfiling {
+		extraHandlers = routes.ProfilingHandlers
+		if cfg.Debugging.EnableContentionProfiling {
+			goruntime.SetBlockProfileRate(1)
+		}
+	}
+
 	log.Info("Setting up manager")
 	mgr, err := manager.New(restConfig, manager.Options{
 		Logger:                  log,
 		Scheme:                  operatorclient.RuntimeScheme,
 		GracefulShutdownTimeout: pointer.Duration(5 * time.Second),
 
 		HealthProbeBindAddress: net.JoinHostPort(cfg.Server.HealthProbes.BindAddress, strconv.Itoa(cfg.Server.HealthProbes.Port)),
-		MetricsBindAddress:     net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
+		Metrics: metricsserver.Options{
+			BindAddress:   net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
+			ExtraHandlers: extraHandlers,
+		},
 
 		LeaderElection:                cfg.LeaderElection.LeaderElect,
 		LeaderElectionResourceLock:    cfg.LeaderElection.ResourceLock,
@@ -151,15 +164,6 @@ func run(ctx context.Context, log logr.Logger, cfg *config.OperatorConfiguration
 		return err
 	}
 
-	if cfg.Debugging != nil && cfg.Debugging.EnableProfiling {
-		if err := (routes.Profiling{}).AddToManager(mgr); err != nil {
-			return fmt.Errorf("failed adding profiling handlers to manager: %w", err)
-		}
-		if cfg.Debugging.EnableContentionProfiling {
-			goruntime.SetBlockProfileRate(1)
-		}
-	}
-
 	log.Info("Setting up health check endpoints")
 	if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil {
 		return err

cmd/gardener-resource-manager/app/app.go

@@ -45,6 +45,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	controllerwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	"github.com/gardener/gardener/cmd/gardener-resource-manager/app/bootstrappers"
@@ -141,17 +142,28 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ResourceManagerConfig
 		managerScheme = resourcemanagerclient.SourceScheme
 	}
 
+	var extraHandlers map[string]http.Handler
+	if cfg.Debugging != nil && cfg.Debugging.EnableProfiling {
+		extraHandlers = routes.ProfilingHandlers
+		if cfg.Debugging.EnableContentionProfiling {
+			goruntime.SetBlockProfileRate(1)
+		}
+	}
+
 	log.Info("Setting up manager")
 	mgr, err := manager.New(sourceRESTConfig, manager.Options{
 		Logger:                  log,
 		Scheme:                  managerScheme,
 		GracefulShutdownTimeout: pointer.Duration(5 * time.Second),
 		Cache: cache.Options{
-			Namespaces: cfg.SourceClientConnection.Namespaces,
-			SyncPeriod: &cfg.SourceClientConnection.CacheResyncPeriod.Duration,
+			DefaultNamespaces: getCacheConfig(cfg.SourceClientConnection.Namespaces),
+			SyncPeriod:        &cfg.SourceClientConnection.CacheResyncPeriod.Duration,
 		},
 		HealthProbeBindAddress: net.JoinHostPort(cfg.Server.HealthProbes.BindAddress, strconv.Itoa(cfg.Server.HealthProbes.Port)),
-		MetricsBindAddress:     net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
+		Metrics: metricsserver.Options{
+			BindAddress:   net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)),
+			ExtraHandlers: extraHandlers,
+		},
 
 		LeaderElection:                cfg.LeaderElection.LeaderElect,
 		LeaderElectionResourceLock:    cfg.LeaderElection.ResourceLock,
@@ -175,15 +187,6 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ResourceManagerConfig
 		return err
 	}
 
-	if cfg.Debugging != nil && cfg.Debugging.EnableProfiling {
-		if err := (routes.Profiling{}).AddToManager(mgr); err != nil {
-			return fmt.Errorf("failed adding profiling handlers to manager: %w", err)
-		}
-		if cfg.Debugging.EnableContentionProfiling {
-			goruntime.SetBlockProfileRate(1)
-		}
-	}
-
 	log.Info("Setting up health check endpoints")
 	sourceClientSet, err := kubernetesclientset.NewForConfig(sourceRESTConfig)
 	if err != nil {
@@ -223,7 +226,7 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ResourceManagerConfig
 				)
 			}
 
-			opts.Cache.Namespaces = cfg.TargetClientConnection.Namespaces
+			opts.Cache.DefaultNamespaces = getCacheConfig(cfg.TargetClientConnection.Namespaces)
 			opts.Cache.SyncPeriod = &cfg.TargetClientConnection.CacheResyncPeriod.Duration
 
 			opts.Client.Cache = &client.CacheOptions{
@@ -284,3 +287,13 @@ func addAllFieldIndexes(ctx context.Context, i client.FieldIndexer) error {
 
 	return nil
 }
+
+func getCacheConfig(namespaces []string) map[string]cache.Config {
+	cacheConfig := map[string]cache.Config{}
+
+	for _, namespace := range namespaces {
+		cacheConfig[namespace] = cache.Config{}
+	}
+
+	return cacheConfig
+}