Merge pull request #10920 from muang0/readiness-generation-check

Verify generation in readiness checks

pkg/kube/ready.go

@@ -183,11 +183,30 @@ func (c *ReadyChecker) IsReady(ctx context.Context, v *resource.Info) (bool, err
 		if !c.statefulSetReady(sts) {
 			return false, nil
 		}
-	case *corev1.ReplicationController, *extensionsv1beta1.ReplicaSet, *appsv1beta2.ReplicaSet, *appsv1.ReplicaSet:
-		ok, err = c.podsReadyForObject(ctx, v.Namespace, value)
-	}
-	if !ok || err != nil {
-		return false, err
+	case *corev1.ReplicationController:
+		rc, err := c.client.CoreV1().ReplicationControllers(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		if !c.replicationControllerReady(rc) {
+			return false, nil
+		}
+		ready, err := c.podsReadyForObject(ctx, v.Namespace, value)
+		if !ready || err != nil {
+			return false, err
+		}
+	case *extensionsv1beta1.ReplicaSet, *appsv1beta2.ReplicaSet, *appsv1.ReplicaSet:
+		rs, err := c.client.AppsV1().ReplicaSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		if !c.replicaSetReady(rs) {
+			return false, nil
+		}
+		ready, err := c.podsReadyForObject(ctx, v.Namespace, value)
+		if !ready || err != nil {
+			return false, err
+		}
 	}
 	return true, nil
 }
@@ -276,6 +295,16 @@ func (c *ReadyChecker) volumeReady(v *corev1.PersistentVolumeClaim) bool {
 }
 
 func (c *ReadyChecker) deploymentReady(rs *appsv1.ReplicaSet, dep *appsv1.Deployment) bool {
+	// Verify the replicaset readiness
+	if !c.replicaSetReady(rs) {
+		return false
+	}
+	// Verify the generation observed by the deployment controller matches the spec generation
+	if dep.Status.ObservedGeneration != dep.ObjectMeta.Generation {
+		c.log("Deployment is not ready: %s/%s. observedGeneration (%d) does not match spec generation (%d).", dep.Namespace, dep.Name, dep.Status.ObservedGeneration, dep.ObjectMeta.Generation)
+		return false
+	}
+
 	expectedReady := *dep.Spec.Replicas - deploymentutil.MaxUnavailable(*dep)
 	if !(rs.Status.ReadyReplicas >= expectedReady) {
 		c.log("Deployment is not ready: %s/%s. %d out of %d expected pods are ready", dep.Namespace, dep.Name, rs.Status.ReadyReplicas, expectedReady)
@@ -285,6 +314,12 @@ func (c *ReadyChecker) deploymentReady(rs *appsv1.ReplicaSet, dep *appsv1.Deploy
 }
 
 func (c *ReadyChecker) daemonSetReady(ds *appsv1.DaemonSet) bool {
+	// Verify the generation observed by the daemonSet controller matches the spec generation
+	if ds.Status.ObservedGeneration != ds.ObjectMeta.Generation {
+		c.log("DaemonSet is not ready: %s/%s. observedGeneration (%d) does not match spec generation (%d).", ds.Namespace, ds.Name, ds.Status.ObservedGeneration, ds.ObjectMeta.Generation)
+		return false
+	}
+
 	// If the update strategy is not a rolling update, there will be nothing to wait for
 	if ds.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType {
 		return true
@@ -355,22 +390,22 @@ func (c *ReadyChecker) crdReady(crd apiextv1.CustomResourceDefinition) bool {
 }
 
 func (c *ReadyChecker) statefulSetReady(sts *appsv1.StatefulSet) bool {
+	// Verify the generation observed by the statefulSet controller matches the spec generation
+	if sts.Status.ObservedGeneration != sts.ObjectMeta.Generation {
+		c.log("StatefulSet is not ready: %s/%s. observedGeneration (%d) does not match spec generation (%d).", sts.Namespace, sts.Name, sts.Status.ObservedGeneration, sts.ObjectMeta.Generation)
+		return false
+	}
+
 	// If the update strategy is not a rolling update, there will be nothing to wait for
 	if sts.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType {
 		c.log("StatefulSet skipped ready check: %s/%s. updateStrategy is %v", sts.Namespace, sts.Name, sts.Spec.UpdateStrategy.Type)
 		return true
 	}
 
-	// Make sure the status is up-to-date with the StatefulSet changes
-	if sts.Status.ObservedGeneration < sts.Generation {
-		c.log("StatefulSet is not ready: %s/%s. update has not yet been observed", sts.Namespace, sts.Name)
-		return false
-	}
-
 	// Dereference all the pointers because StatefulSets like them
 	var partition int
 	// 1 is the default for replicas if not set
-	var replicas = 1
+	replicas := 1
 	// For some reason, even if the update strategy is a rolling update, the
 	// actual rollingUpdate field can be nil. If it is, we can safely assume
 	// there is no partition value
@@ -409,6 +444,24 @@ func (c *ReadyChecker) statefulSetReady(sts *appsv1.StatefulSet) bool {
 	return true
 }
 
+func (c *ReadyChecker) replicationControllerReady(rc *corev1.ReplicationController) bool {
+	// Verify the generation observed by the replicationController controller matches the spec generation
+	if rc.Status.ObservedGeneration != rc.ObjectMeta.Generation {
+		c.log("ReplicationController is not ready: %s/%s. observedGeneration (%d) does not match spec generation (%d).", rc.Namespace, rc.Name, rc.Status.ObservedGeneration, rc.ObjectMeta.Generation)
+		return false
+	}
+	return true
+}
+
+func (c *ReadyChecker) replicaSetReady(rs *appsv1.ReplicaSet) bool {
+	// Verify the generation observed by the replicaSet controller matches the spec generation
+	if rs.Status.ObservedGeneration != rs.ObjectMeta.Generation {
+		c.log("ReplicaSet is not ready: %s/%s. observedGeneration (%d) does not match spec generation (%d).", rs.Namespace, rs.Name, rs.Status.ObservedGeneration, rs.ObjectMeta.Generation)
+		return false
+	}
+	return true
+}
+
 func getPods(ctx context.Context, client kubernetes.Interface, namespace, selector string) ([]corev1.Pod, error) {
 	list, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
 		LabelSelector: selector,