[Merged by Bors] - Add jitter to spread out requests to get poet proof and submit challenge

activation/activation.go

@@ -46,7 +46,15 @@
 	}
 }
 
-const defaultPoetRetryInterval = 5 * time.Second
+const (
+	defaultPoetRetryInterval = 5 * time.Second
+
+	// Jitter added to the wait time before building a nipost challenge.
+	// It's expressed as % of poet grace period which translates to:
+	// mainnet (grace period 1h) -> 36s
+	// systest (grace period 10s) -> 0.1s
+	maxNipostChallengeBuildJitter = 1.0
+)
 
 // Config defines configuration for Builder.
 type Config struct {
@@ -432,8 +440,8 @@
 			ErrATXChallengeExpired, current, -until)
 	}
 	metrics.PublishOntimeWindowLatency.Observe(until.Seconds())
-	if until > b.poetCfg.GracePeriod {
-		wait := until - b.poetCfg.GracePeriod
+	wait := timeToWaitToBuildNipostChallenge(until, b.poetCfg.GracePeriod)
+	if wait >= 0 {
 		b.log.WithContext(ctx).With().Debug("waiting for fresh atxs",
 			log.Duration("till poet round", until),
 			log.Uint32("current epoch", current.Uint32()),
@@ -721,3 +729,8 @@
 	atx.SmesherID = signer.NodeID()
 	return atx.Initialize()
 }
+
+func timeToWaitToBuildNipostChallenge(untilRoundStart, gracePeriod time.Duration) time.Duration {
+	jitter := randomDurationInRange(time.Duration(0), gracePeriod*maxNipostChallengeBuildJitter/100.0)
+	return untilRoundStart + jitter - gracePeriod
+}

activation/activation_test.go

@@ -1265,3 +1265,36 @@ func TestWaitPositioningAtx(t *testing.T) {
 		})
 	}
 }
+
+func TestWaitingToBuildNipostChallengeWithJitter(t *testing.T) {
+	t.Run("before grace period", func(t *testing.T) {
+		//          ┌──grace period──┐
+		//          │                │
+		// ───▲─────|──────|─────────|----> time
+		//    │     └jitter|         └round start
+		//   now
+		wait := timeToWaitToBuildNipostChallenge(2*time.Hour, time.Hour)
+		require.Greater(t, wait, time.Hour)
+		require.LessOrEqual(t, wait, time.Hour+time.Second*36)
+	})
+	t.Run("after grace period, within max jitter value", func(t *testing.T) {
+		//          ┌──grace period──┐
+		//          │                │
+		// ─────────|──▲────|────────|----> time
+		//          └ji│tter|        └round start
+		//            now
+		wait := timeToWaitToBuildNipostChallenge(time.Hour-time.Second*10, time.Hour)
+		require.GreaterOrEqual(t, wait, -time.Second*10)
+		// jitter is 1% = 36s for 1h grace period
+		require.LessOrEqual(t, wait, time.Second*(36-10))
+	})
+	t.Run("after jitter max value", func(t *testing.T) {
+		//          ┌──grace period──┐
+		//          │                │
+		// ─────────|──────|──▲──────|----> time
+		//          └jitter|  │       └round start
+		//                   now
+		wait := timeToWaitToBuildNipostChallenge(time.Hour-time.Second*37, time.Hour)
+		require.Less(t, wait, time.Duration(0))
+	})
+}

activation/nipost.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"math/rand"
 	"time"
 
 	"github.com/spacemeshos/merkle-tree"
@@ -22,6 +23,24 @@ import (
 	"github.com/spacemeshos/go-spacemesh/signing"
 )
 
+const (
+	// Jitter values to avoid all nodes querying the poet at the same time.
+	// Note: the jitter values are represented as a percentage of cycle gap.
+	// mainnet cycle-gap: 12h
+	// systest cycle-gap: 30s.
+
+	// Minimum jitter value before querying for the proof.
+	// Gives the poet service time to generate proof after a round ends (~8s on mainnet).
+	// mainnet -> 8.64s
+	// systest -> 0.36s.
+	minPoetGetProofJitter = 0.02
+
+	// The maximum jitter value before querying for the proof.
+	// mainnet -> 17.28s
+	// systest -> 0.72s.
+	maxPoetGetProofJitter = 0.04
+)
+
 //go:generate mockgen -package=activation -destination=./nipost_mocks.go -source=./nipost.go PoetProvingServiceClient
 
 // PoetProvingServiceClient provides a gateway to a trust-less public proving service, which may serve many PoET
@@ -385,10 +404,7 @@ func (nb *NIPostBuilder) getBestProof(ctx context.Context, challenge types.Hash3
 			continue
 		}
 		round := r.PoetRound.ID
-		// Time to wait before querying for the proof
-		// The additional second is an optimization to be nicer to poet
-		// and don't accidentally ask it to soon and have to retry.
-		waitTime := time.Until(r.PoetRound.End.IntoTime()) + time.Second
+		waitTime := calcGetProofWaitTime(time.Until(r.PoetRound.End.IntoTime()), nb.poetCfg.CycleGap)
 		eg.Go(func() error {
 			logger.With().Info("waiting till poet round end", log.Duration("wait time", waitTime))
 			select {
@@ -479,3 +495,16 @@ func constructMerkleProof(challenge types.Hash32, members []types.Member) (*type
 		Nodes:     nodesH32,
 	}, nil
 }
+
+func randomDurationInRange(min, max time.Duration) time.Duration {
+	return min + time.Duration(rand.Int63n(int64(max-min+1)))
+}
+
+// Calculate the time to wait before querying for the proof
+// We add a jitter to avoid all nodes querying for the proof at the same time.
+func calcGetProofWaitTime(tillRoundEnd, cycleGap time.Duration) (waitTime time.Duration) {
+	minJitter := time.Duration(float64(cycleGap) * minPoetGetProofJitter / 100.0)
+	maxJitter := time.Duration(float64(cycleGap) * maxPoetGetProofJitter / 100.0)
+	jitter := randomDurationInRange(minJitter, maxJitter)
+	return tillRoundEnd + jitter
+}

activation/nipost_test.go

@@ -1054,3 +1054,39 @@ func FuzzBuilderStateConsistency(f *testing.F) {
 func FuzzBuilderStateSafety(f *testing.F) {
 	tester.FuzzSafety[types.NIPostBuilderState](f)
 }
+
+func TestRandomDurationInRange(t *testing.T) {
+	t.Parallel()
+	test := func(min, max time.Duration) {
+		for i := 0; i < 100; i++ {
+			waittime := randomDurationInRange(min, max)
+			require.LessOrEqual(t, waittime, max)
+			require.GreaterOrEqual(t, waittime, min)
+		}
+	}
+	t.Run("min = 0", func(t *testing.T) {
+		t.Parallel()
+		test(0, 7*time.Second)
+	})
+	t.Run("min != 0", func(t *testing.T) {
+		t.Parallel()
+		test(5*time.Second, 7*time.Second)
+	})
+}
+
+func TestCalculatingGetProofWaitTime(t *testing.T) {
+	t.Parallel()
+	t.Run("past round end", func(t *testing.T) {
+		t.Parallel()
+		waitTime := calcGetProofWaitTime(-time.Hour, time.Hour*12)
+		require.Less(t, waitTime, time.Duration(0))
+	})
+	t.Run("before round end", func(t *testing.T) {
+		t.Parallel()
+		cycleGap := 12 * time.Hour
+		waitTime := calcGetProofWaitTime(time.Hour, cycleGap)
+
+		require.Greater(t, waitTime, time.Hour+time.Duration(float64(cycleGap)*minPoetGetProofJitter/100))
+		require.LessOrEqual(t, waitTime, time.Hour+time.Duration(float64(cycleGap)*maxPoetGetProofJitter/100))
+	})
+}