use tortoise data for ballot eligibility validation (#4935)

related: #4927

this change eliminates database lookups on ballot validation path, with an exception of when lru cache is not sufficient to hold
vrf nonces and activations. in that case we will have to load them from db, that can be optimized later by fitting more in lru cache #4935 or reusing tortoise dataset as well

- fetcher will be asked for ballots only if we don't have them in memory
- reference ballots for eligibility validation will be fetched from tortoise

miner/oracle_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/sql/certificates"
 	"github.com/spacemeshos/go-spacemesh/sql/identities"
 	"github.com/spacemeshos/go-spacemesh/system/mocks"
+	"github.com/spacemeshos/go-spacemesh/tortoise"
 )
 
 const (
@@ -181,11 +182,24 @@ func testMinerOracleAndProposalValidator(t *testing.T, layerSize, layersPerEpoch
 	mbc := mocks.NewMockBeaconCollector(ctrl)
 	vrfVerifier := proposals.NewMockvrfVerifier(ctrl)
 	vrfVerifier.EXPECT().Verify(gomock.Any(), gomock.Any(), gomock.Any()).Return(true).AnyTimes()
+	tmock := proposals.NewMocktortoiseProvider(ctrl)
+	tmock.EXPECT().GetBallot(gomock.Any()).AnyTimes().DoAndReturn(func(id types.BallotID) *tortoise.BallotData {
+		ballot, err := ballots.Get(o.cdb, id)
+		require.NoError(t, err)
+		return &tortoise.BallotData{
+			ID:           ballot.ID(),
+			Layer:        ballot.Layer,
+			ATXID:        ballot.AtxID,
+			Smesher:      ballot.SmesherID,
+			Beacon:       ballot.EpochData.Beacon,
+			Eligiblities: ballot.EpochData.EligibilityCount,
+		}
+	})
 
 	nonceFetcher := proposals.NewMocknonceFetcher(ctrl)
 	nonce := types.VRFPostIndex(rand.Uint64())
 
-	validator := proposals.NewEligibilityValidator(layerSize, layersPerEpoch, 0, o.mClock, o.cdb, mbc, o.log.WithName("blkElgValidator"), vrfVerifier,
+	validator := proposals.NewEligibilityValidator(layerSize, layersPerEpoch, 0, o.mClock, tmock, o.cdb, mbc, o.log.WithName("blkElgValidator"), vrfVerifier,
 		proposals.WithNonceFetcher(nonceFetcher),
 	)
 

proposals/eligibility_validator.go

@@ -10,7 +10,6 @@ import (
 	"github.com/spacemeshos/go-spacemesh/common/types"
 	"github.com/spacemeshos/go-spacemesh/datastore"
 	"github.com/spacemeshos/go-spacemesh/log"
-	"github.com/spacemeshos/go-spacemesh/sql/ballots"
 	"github.com/spacemeshos/go-spacemesh/system"
 )
 
@@ -30,6 +29,7 @@ type Validator struct {
 	minActiveSetWeight uint64
 	avgLayerSize       uint32
 	layersPerEpoch     uint32
+	tortoise           tortoiseProvider
 	cdb                *datastore.CachedDB
 	clock              layerClock
 	beacons            system.BeaconCollector
@@ -49,12 +49,13 @@ func WithNonceFetcher(nf nonceFetcher) ValidatorOpt {
 
 // NewEligibilityValidator returns a new EligibilityValidator.
 func NewEligibilityValidator(
-	avgLayerSize, layersPerEpoch uint32, minActiveSetWeight uint64, clock layerClock, cdb *datastore.CachedDB, bc system.BeaconCollector, lg log.Log, vrfVerifier vrfVerifier, opts ...ValidatorOpt,
+	avgLayerSize, layersPerEpoch uint32, minActiveSetWeight uint64, clock layerClock, tortoise tortoiseProvider, cdb *datastore.CachedDB, bc system.BeaconCollector, lg log.Log, vrfVerifier vrfVerifier, opts ...ValidatorOpt,
 ) *Validator {
 	v := &Validator{
 		minActiveSetWeight: minActiveSetWeight,
 		avgLayerSize:       avgLayerSize,
 		layersPerEpoch:     layersPerEpoch,
+		tortoise:           tortoise,
 		cdb:                cdb,
 		nonceFetcher:       cdb,
 		clock:              clock,
@@ -162,27 +163,24 @@ func (v *Validator) validateReference(ballot *types.Ballot, owned *types.Activat
 
 // validateSecondary executed for non-reference ballots in latest epoch and all ballots in past epochs.
 func (v *Validator) validateSecondary(ballot *types.Ballot, owned *types.ActivationTxHeader) (*types.EpochData, error) {
-	var refballot *types.Ballot
 	if ballot.RefBallot == types.EmptyBallotID {
-		refballot = ballot
-	} else {
-		var err error
-		refballot, err = ballots.Get(v.cdb, ballot.RefBallot)
-		if err != nil {
-			return nil, fmt.Errorf("ref ballot is missing %v: %w", ballot.RefBallot, err)
+		if ballot.EpochData == nil {
+			return nil, fmt.Errorf("%w: ref ballot %v", errMissingEpochData, ballot.ID())
 		}
+		return ballot.EpochData, nil
 	}
-	if refballot.EpochData == nil {
-		return nil, fmt.Errorf("%w: ref ballot %v", errMissingEpochData, refballot.ID())
+	refdata := v.tortoise.GetBallot(ballot.RefBallot)
+	if refdata == nil {
+		return nil, fmt.Errorf("ref ballot is missing %v", ballot.RefBallot)
 	}
-	if refballot.AtxID != ballot.AtxID {
-		return nil, fmt.Errorf("ballot (%v/%v) should be sharing atx with a reference ballot (%v/%v)", ballot.ID(), ballot.AtxID, refballot.ID(), refballot.AtxID)
+	if refdata.ATXID != ballot.AtxID {
+		return nil, fmt.Errorf("ballot (%v/%v) should be sharing atx with a reference ballot (%v/%v)", ballot.ID(), ballot.AtxID, refdata.ID, refdata.ATXID)
 	}
-	if refballot.SmesherID != ballot.SmesherID {
+	if refdata.Smesher != ballot.SmesherID {
 		return nil, fmt.Errorf("mismatched smesher id with refballot in ballot %v", ballot.ID())
 	}
-	if refballot.Layer.GetEpoch() != ballot.Layer.GetEpoch() {
+	if refdata.Layer.GetEpoch() != ballot.Layer.GetEpoch() {
 		return nil, fmt.Errorf("ballot %v targets mismatched epoch %d", ballot.ID(), ballot.Layer.GetEpoch())
 	}
-	return refballot.EpochData, nil
+	return &types.EpochData{Beacon: refdata.Beacon, EligibilityCount: refdata.Eligiblities}, nil
 }

proposals/eligibility_validator_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/log/logtest"
 	"github.com/spacemeshos/go-spacemesh/sql"
 	"github.com/spacemeshos/go-spacemesh/sql/atxs"
-	"github.com/spacemeshos/go-spacemesh/sql/ballots"
+	"github.com/spacemeshos/go-spacemesh/tortoise"
 )
 
 func gatx(id types.ATXID, epoch types.EpochID, smesher types.NodeID, units uint32, nonce types.VRFPostIndex) types.VerifiedActivationTx {
@@ -533,17 +533,33 @@ func TestEligibilityValidator(t *testing.T) {
 			ms := fullMockSet(t)
 			ms.mclock.EXPECT().CurrentLayer().Return(tc.current).AnyTimes()
 			ms.mvrf.EXPECT().Verify(gomock.Any(), gomock.Any(), gomock.Any()).Return(!tc.vrfFailed).AnyTimes()
+			ballots := map[types.BallotID]*types.Ballot{}
+			ms.md.EXPECT().GetBallot(gomock.Any()).DoAndReturn(func(id types.BallotID) *tortoise.BallotData {
+				ballot, exists := ballots[id]
+				if !exists {
+					return nil
+				}
+				return &tortoise.BallotData{
+					ID:           ballot.ID(),
+					Layer:        ballot.Layer,
+					ATXID:        ballot.AtxID,
+					Smesher:      ballot.SmesherID,
+					Beacon:       ballot.EpochData.Beacon,
+					Eligiblities: ballot.EpochData.EligibilityCount,
+				}
+			}).AnyTimes()
 
 			lg := logtest.New(t)
 			db := datastore.NewCachedDB(sql.InMemory(), lg)
-			tv := NewEligibilityValidator(layerAvgSize, layersPerEpoch, tc.minWeight, ms.mclock, db, ms.mbc, lg, ms.mvrf,
+			tv := NewEligibilityValidator(layerAvgSize, layersPerEpoch, tc.minWeight, ms.mclock, ms.md,
+				db, ms.mbc, lg, ms.mvrf,
 				WithNonceFetcher(db),
 			)
 			for _, atx := range tc.atxs {
 				require.NoError(t, atxs.Add(db, &atx))
 			}
 			for _, ballot := range tc.ballots {
-				require.NoError(t, ballots.Add(db, &ballot))
+				ballots[ballot.ID()] = &ballot
 			}
 			if !tc.fail {
 				ms.mbc.EXPECT().ReportBeaconFromBallot(tc.executed.Layer.GetEpoch(), &tc.executed, gomock.Any(), gomock.Any())

proposals/handler.go

@@ -18,7 +18,6 @@ import (
 	"github.com/spacemeshos/go-spacemesh/p2p/pubsub"
 	"github.com/spacemeshos/go-spacemesh/signing"
 	"github.com/spacemeshos/go-spacemesh/sql"
-	"github.com/spacemeshos/go-spacemesh/sql/ballots"
 	"github.com/spacemeshos/go-spacemesh/sql/proposals"
 	"github.com/spacemeshos/go-spacemesh/system"
 	"github.com/spacemeshos/go-spacemesh/tortoise"
@@ -56,7 +55,7 @@ type Handler struct {
 	fetcher    system.Fetcher
 	mesh       meshProvider
 	validator  eligibilityValidator
-	decoder    ballotDecoder
+	tortoise   tortoiseProvider
 	clock      layerClock
 }
 
@@ -109,7 +108,7 @@ func NewHandler(
 	f system.Fetcher,
 	bc system.BeaconCollector,
 	m meshProvider,
-	decoder ballotDecoder,
+	tortoise tortoiseProvider,
 	verifier vrfVerifier,
 	clock layerClock,
 	opts ...Opt,
@@ -122,14 +121,14 @@ func NewHandler(
 		publisher:  p,
 		fetcher:    f,
 		mesh:       m,
-		decoder:    decoder,
+		tortoise:   tortoise,
 		clock:      clock,
 	}
 	for _, opt := range opts {
 		opt(b)
 	}
 	if b.validator == nil {
-		b.validator = NewEligibilityValidator(b.cfg.LayerSize, b.cfg.LayersPerEpoch, b.cfg.MinimalActiveSetWeight, clock, cdb, bc, b.logger, verifier)
+		b.validator = NewEligibilityValidator(b.cfg.LayerSize, b.cfg.LayersPerEpoch, b.cfg.MinimalActiveSetWeight, clock, tortoise, cdb, bc, b.logger, verifier)
 	}
 	return b
 }
@@ -340,15 +339,10 @@ func (h *Handler) handleProposal(ctx context.Context, expHash types.Hash32, peer
 }
 
 func (h *Handler) processBallot(ctx context.Context, logger log.Log, b *types.Ballot) (*types.MalfeasanceProof, error) {
-	t0 := time.Now()
-	if has, err := ballots.Has(h.cdb, b.ID()); err != nil {
-		logger.With().Error("failed to look up ballot", log.Err(err))
-		return nil, fmt.Errorf("lookup ballot %v: %w", b.ID(), err)
-	} else if has {
+	if data := h.tortoise.GetBallot(b.ID()); data != nil {
 		known.Inc()
 		return nil, fmt.Errorf("%w: ballot %s", errKnownBallot, b.ID())
 	}
-	ballotDuration.WithLabelValues(dbLookup).Observe(float64(time.Since(t0)))
 
 	logger.With().Info("new ballot", log.Inline(b))
 
@@ -367,7 +361,7 @@ func (h *Handler) processBallot(ctx context.Context, logger log.Log, b *types.Ba
 		return nil, fmt.Errorf("save ballot: %w", err)
 	}
 	ballotDuration.WithLabelValues(dbSave).Observe(float64(time.Since(t1)))
-	if err := h.decoder.StoreBallot(decoded); err != nil {
+	if err := h.tortoise.StoreBallot(decoded); err != nil {
 		if errors.Is(err, tortoise.ErrBallotExists) {
 			return nil, fmt.Errorf("%w: %s", errKnownBallot, b.ID())
 		}
@@ -395,7 +389,7 @@ func (h *Handler) checkBallotSyntacticValidity(ctx context.Context, logger log.L
 	t2 := time.Now()
 	// ballot can be decoded only if all dependencies (blocks, ballots, atxs) were downloaded
 	// and added to the tortoise.
-	decoded, err := h.decoder.DecodeBallot(b.ToTortoiseData())
+	decoded, err := h.tortoise.DecodeBallot(b.ToTortoiseData())
 	if err != nil {
 		return nil, fmt.Errorf("decode ballot %s: %w", b.ID(), err)
 	}
@@ -499,16 +493,15 @@ func (h *Handler) checkVotesConsistency(ctx context.Context, b *types.Ballot) er
 
 func (h *Handler) checkBallotDataAvailability(ctx context.Context, b *types.Ballot) error {
 	var blts []types.BallotID
-	if b.Votes.Base != types.EmptyBallotID {
+	if b.Votes.Base != types.EmptyBallotID && h.tortoise.GetBallot(b.Votes.Base) == nil {
 		blts = append(blts, b.Votes.Base)
 	}
-	if b.RefBallot != types.EmptyBallotID {
+	if b.RefBallot != types.EmptyBallotID && h.tortoise.GetBallot(b.RefBallot) == nil {
 		blts = append(blts, b.RefBallot)
 	}
 	if err := h.fetcher.GetBallots(ctx, blts); err != nil {
 		return fmt.Errorf("fetch ballots: %w", err)
 	}
-
 	if err := h.fetchReferencedATXs(ctx, b); err != nil {
 		return fmt.Errorf("fetch referenced ATXs: %w", err)
 	}
@@ -520,7 +513,7 @@ func (h *Handler) fetchReferencedATXs(ctx context.Context, b *types.Ballot) erro
 	if b.EpochData != nil {
 		atxs = append(atxs, b.ActiveSet...)
 	}
-	if err := h.fetcher.GetAtxs(ctx, h.decoder.GetMissingActiveSet(b.Layer.GetEpoch(), atxs)); err != nil {
+	if err := h.fetcher.GetAtxs(ctx, h.tortoise.GetMissingActiveSet(b.Layer.GetEpoch(), atxs)); err != nil {
 		return fmt.Errorf("proposal get ATXs: %w", err)
 	}
 	return nil

proposals/handler_test.go

@@ -47,7 +47,7 @@ type mockSet struct {
 	mclock *MocklayerClock
 	mm     *MockmeshProvider
 	mv     *MockeligibilityValidator
-	md     *MockballotDecoder
+	md     *MocktortoiseProvider
 	mvrf   *MockvrfVerifier
 }
 
@@ -77,7 +77,7 @@ func fullMockSet(tb testing.TB) *mockSet {
 		mclock: NewMocklayerClock(ctrl),
 		mm:     NewMockmeshProvider(ctrl),
 		mv:     NewMockeligibilityValidator(ctrl),
-		md:     NewMockballotDecoder(ctrl),
+		md:     NewMocktortoiseProvider(ctrl),
 		mvrf:   NewMockvrfVerifier(ctrl),
 	}
 }
@@ -87,9 +87,27 @@ func createTestHandler(t *testing.T) *testHandler {
 	ms := fullMockSet(t)
 	edVerifier, err := signing.NewEdVerifier()
 	require.NoError(t, err)
+	db := datastore.NewCachedDB(sql.InMemory(), logtest.New(t))
+	ms.md.EXPECT().GetBallot(gomock.Any()).AnyTimes().DoAndReturn(func(id types.BallotID) *tortoise.BallotData {
+		ballot, err := ballots.Get(db, id)
+		if err != nil {
+			return nil
+		}
 
+		data := &tortoise.BallotData{
+			ID:      ballot.ID(),
+			Layer:   ballot.Layer,
+			ATXID:   ballot.AtxID,
+			Smesher: ballot.SmesherID,
+		}
+		if ballot.EpochData != nil {
+			data.Beacon = ballot.EpochData.Beacon
+			data.Eligiblities = ballot.EpochData.EligibilityCount
+		}
+		return data
+	})
 	return &testHandler{
-		Handler: NewHandler(datastore.NewCachedDB(sql.InMemory(), logtest.New(t)), edVerifier, ms.mpub, ms.mf, ms.mbc, ms.mm, ms.md, ms.mvrf, ms.mclock,
+		Handler: NewHandler(db, edVerifier, ms.mpub, ms.mf, ms.mbc, ms.mm, ms.md, ms.mvrf, ms.mclock,
 			WithLogger(logtest.New(t)),
 			WithConfig(Config{
 				LayerSize:      layerAvgSize,

proposals/interface.go

@@ -19,7 +19,8 @@ type eligibilityValidator interface {
 	CheckEligibility(context.Context, *types.Ballot) (bool, error)
 }
 
-type ballotDecoder interface {
+type tortoiseProvider interface {
+	GetBallot(types.BallotID) *tortoise.BallotData
 	GetMissingActiveSet(types.EpochID, []types.ATXID) []types.ATXID
 	DecodeBallot(*types.BallotTortoiseData) (*tortoise.DecodedBallot, error)
 	StoreBallot(*tortoise.DecodedBallot) error