diff --git a/pkg/controller/controllerutil/controllerutil.go b/pkg/controller/controllerutil/controllerutil.go
index c8e30bd173..706d48f8ec 100644
--- a/pkg/controller/controllerutil/controllerutil.go
+++ b/pkg/controller/controllerutil/controllerutil.go
@@ -184,11 +184,16 @@ func RemoveControllerReference(owner, object metav1.Object, scheme *runtime.Sche
 		APIVersion: gvk.GroupVersion().String(),
 		Name:       owner.GetName(),
 		Kind:       gvk.Kind,
-		Controller: ptr.To(true),
 	})
+
 	if index == -1 {
 		return fmt.Errorf("%T does not have an controller reference for %T", object, owner)
 	}
+
+	if ownerRefs[index].Controller != ptr.To(true) && owner.GetName() != ownerRefs[index].Name {
+		return fmt.Errorf("%T owner is not the controller reference for %T", object, owner)
+	}
+
 	ownerRefs = append(ownerRefs[:index], ownerRefs[index+1:]...)
 	object.SetOwnerReferences(ownerRefs)
 	return nil
@@ -239,7 +244,6 @@ func referSameObject(a, b metav1.OwnerReference) bool {
 	if err != nil {
 		return false
 	}
-
 	return aGV.Group == bGV.Group && a.Kind == b.Kind && a.Name == b.Name
 }
 
diff --git a/pkg/controller/controllerutil/controllerutil_test.go b/pkg/controller/controllerutil/controllerutil_test.go
index 7e9d9f96c4..845bf773a9 100644
--- a/pkg/controller/controllerutil/controllerutil_test.go
+++ b/pkg/controller/controllerutil/controllerutil_test.go
@@ -181,6 +181,7 @@ var _ = Describe("Controllerutil", func() {
 			Expect(controllerutil.RemoveOwnerReference(obj, rs, scheme.Scheme)).To(HaveOccurred())
 			Expect(rs.GetOwnerReferences()).To(HaveLen(1))
 		})
+
 		It("should error when trying to remove an owner that doesn't exist", func() {
 			rs := &appsv1.ReplicaSet{
 				ObjectMeta: metav1.ObjectMeta{},
@@ -229,6 +230,20 @@ var _ = Describe("Controllerutil", func() {
 			Expect(controllerutil.RemoveControllerReference(dep, rs, scheme.Scheme)).To(HaveOccurred())
 		})
 
+		It("should error when RemoveControllerReference passed in owner is not the owner", func() {
+			rs := &appsv1.ReplicaSet{
+				ObjectMeta: metav1.ObjectMeta{},
+			}
+			dep := &extensionsv1beta1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"},
+			}
+			dep2 := &extensionsv1beta1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "foo-2", UID: "foo-uid-42"},
+			}
+			Expect(controllerutil.SetControllerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred())
+			Expect(controllerutil.RemoveControllerReference(dep2, rs, scheme.Scheme)).To(HaveOccurred())
+		})
+
 		It("should not error when RemoveControllerReference owner's controller is set to true", func() {
 			rs := &appsv1.ReplicaSet{
 				ObjectMeta: metav1.ObjectMeta{},