Prevent a deadlock in expiration (#22374)

* Prevent a deadlock that occurs when Restore grabs lease lock then pending lock, while a revocationJob's OnFailure grabs those lock in the reverse order.

* Fix lock

* Add CL

* Grab lock before markLeaseIrrevocable

---------

Co-authored-by: hc-github-team-secure-vault-core <github-team-secure-vault-core@hashicorp.com>

changelog/22374.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+expiration: Fix a deadlock that could occur when a revocation failure happens while restoring leases on startup.
+```

vault/expiration.go

@@ -240,8 +240,8 @@ func (r *revocationJob) OnFailure(err error) {
 	r.m.core.metricSink.IncrCounterWithLabels([]string{"expire", "lease_expiration", "error"}, 1, []metrics.Label{metricsutil.NamespaceLabel(r.ns)})
 
 	r.m.pendingLock.Lock()
-	defer r.m.pendingLock.Unlock()
 	pendingRaw, ok := r.m.pending.Load(r.leaseID)
+	r.m.pendingLock.Unlock()
 	if !ok {
 		r.m.logger.Warn("failed to find lease in pending map for revocation retry", "lease_id", r.leaseID)
 		return
@@ -269,15 +269,19 @@ func (r *revocationJob) OnFailure(err error) {
 			return
 		}
 
+		r.m.pendingLock.Lock()
 		r.m.markLeaseIrrevocable(r.nsCtx, le, err)
+		r.m.pendingLock.Unlock()
 		return
 	} else {
 		r.m.logger.Error("failed to revoke lease", "lease_id", r.leaseID, "error", err,
 			"attempts", pending.revokesAttempted, "next_attempt", newTimer)
 	}
 
 	pending.timer.Reset(newTimer)
+	r.m.pendingLock.Lock()
 	r.m.pending.Store(r.leaseID, pending)
+	r.m.pendingLock.Unlock()
 }
 
 func expireLeaseStrategyFairsharing(ctx context.Context, m *ExpirationManager, leaseID string, ns *namespace.Namespace) {