prune old proposals after block is generated (#4993)

## Motivation prune mesh data ## Changes after block is generated for layer N, prune proposals < N
spacemeshos · Sep 11, 2023 · 8b371b1 · 8b371b1
1 parent 61d308f
commit 8b371b1
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 5 deletions.
diff --git a/blocks/generator.go b/blocks/generator.go
@@ -13,10 +13,10 @@ import (
 	"github.com/spacemeshos/go-spacemesh/common/types"
 	"github.com/spacemeshos/go-spacemesh/datastore"
 	"github.com/spacemeshos/go-spacemesh/hare"
-	heligibility "github.com/spacemeshos/go-spacemesh/hare/eligibility"
+	"github.com/spacemeshos/go-spacemesh/hare/eligibility"
 	"github.com/spacemeshos/go-spacemesh/log"
 	"github.com/spacemeshos/go-spacemesh/sql/layers"
-	dbproposals "github.com/spacemeshos/go-spacemesh/sql/proposals"
+	"github.com/spacemeshos/go-spacemesh/sql/proposals"
 	"github.com/spacemeshos/go-spacemesh/system"
 )
 
@@ -167,6 +167,12 @@ func (g *Generator) run() error {
 			if len(g.optimisticOutput) > 0 {
 				g.processOptimisticLayers(maxLayer)
 			}
+			if err := proposals.Delete(g.cdb, out.Layer); err != nil {
+				g.logger.With().Error("failed to delete old proposals",
+					out.Layer,
+					log.Err(err),
+				)
+			}
 		case <-time.After(g.cfg.GenBlockInterval):
 			if len(g.optimisticOutput) > 0 {
 				g.processOptimisticLayers(maxLayer)
@@ -182,7 +188,7 @@ func (g *Generator) getProposals(pids []types.ProposalID) ([]*types.Proposal, er
 		err    error
 	)
 	for _, pid := range pids {
-		if p, err = dbproposals.Get(g.cdb, pid); err != nil {
+		if p, err = proposals.Get(g.cdb, pid); err != nil {
 			return nil, err
 		}
 		result = append(result, p)
@@ -309,7 +315,7 @@ func (g *Generator) saveAndCertify(ctx context.Context, lid types.LayerID, block
 		)
 	}
 
-	if err := g.cert.CertifyIfEligible(ctx, g.logger, lid, hareOutput); err != nil && !errors.Is(err, heligibility.ErrNotActive) {
+	if err := g.cert.CertifyIfEligible(ctx, g.logger, lid, hareOutput); err != nil && !errors.Is(err, eligibility.ErrNotActive) {
 		g.logger.With().Warning("failed to certify block",
 			log.Context(ctx),
 			lid,

diff --git a/blocks/generator_test.go b/blocks/generator_test.go
@@ -337,7 +337,17 @@ func Test_run(t *testing.T) {
 			txIDs := createAndSaveTxs(t, numTXs, tg.cdb)
 			signers, atxes := createATXs(t, tg.cdb, (layerID.GetEpoch() - 1).FirstLayer(), numProposals)
 			activeSet := types.ToATXIDs(atxes)
+			// generate some proposals before this layer
+			oldest := layerID - 10
+			for lid := oldest; lid < layerID; lid++ {
+				createProposals(t, tg.cdb, lid, types.EmptyLayerHash, signers, activeSet, txIDs)
+			}
 			plist := createProposals(t, tg.cdb, layerID, meshHash, signers, activeSet, txIDs)
+			for lid := oldest; lid <= layerID; lid++ {
+				got, err := proposals.GetByLayer(tg.cdb, lid)
+				require.NoError(t, err)
+				require.Len(t, got, len(signers))
+			}
 			pids := types.ToProposalIDs(plist)
 			tg.mockFetch.EXPECT().GetProposals(gomock.Any(), pids)
 
@@ -395,6 +405,13 @@ func Test_run(t *testing.T) {
 			tg.hareCh <- hare.LayerOutput{Ctx: context.Background(), Layer: layerID, Proposals: pids}
 			require.Eventually(t, func() bool { return len(tg.hareCh) == 0 }, time.Second, 100*time.Millisecond)
 			tg.Stop()
+			for lid := oldest; lid < layerID; lid++ {
+				_, err := proposals.GetByLayer(tg.cdb, lid)
+				require.ErrorIs(t, err, sql.ErrNotFound)
+			}
+			got, err := proposals.GetByLayer(tg.cdb, layerID)
+			require.NoError(t, err)
+			require.Len(t, got, len(signers))
 		})
 	}
 }

diff --git a/blocks/utils.go b/blocks/utils.go
@@ -122,7 +122,7 @@ func getProposalMetadata(
 			return nil, fmt.Errorf("get prev mesh hash %w", err)
 		}
 		if ownMeshHash != majorityState.hash {
-			return nil, fmt.Errorf("%w: majority %v, node %v", errNodeHasBadMeshHash, majorityState.hash, ownMeshHash)
+			return nil, fmt.Errorf("%w: majority %v, node %v", errNodeHasBadMeshHash, majorityState.hash.ShortString(), ownMeshHash.ShortString())
 		}
 		logger.With().Debug("consensus on mesh hash. doing optimistic filtering",
 			lid,

diff --git a/sql/proposals/proposals.go b/sql/proposals/proposals.go
@@ -185,3 +185,13 @@ func decodeProposal(stmt *sql.Statement) (*types.Proposal, error) {
 	proposal.SetID(proposalID)
 	return proposal, nil
 }
+
+func Delete(db sql.Executor, lid types.LayerID) error {
+	if _, err := db.Exec(`delete from proposals where layer < ?1;`,
+		func(stmt *sql.Statement) {
+			stmt.BindInt64(1, int64(lid))
+		}, nil); err != nil {
+		return fmt.Errorf("delete %s: %w", lid, err)
+	}
+	return nil
+}
diff --git a/sql/proposals/proposals_test.go b/sql/proposals/proposals_test.go
@@ -33,6 +33,40 @@ func TestAdd(t *testing.T) {
 	require.ErrorIs(t, Add(db, proposal), sql.ErrObjectExists)
 }
 
+func TestDelete(t *testing.T) {
+	db := sql.InMemory()
+	nodeID := types.RandomNodeID()
+	maxLayers := 10
+	numProps := 3
+	for i := 1; i <= maxLayers; i++ {
+		for j := 0; j < numProps; j++ {
+			ballot := types.NewExistingBallot(types.RandomBallotID(), types.RandomEdSignature(), nodeID, types.LayerID(i))
+			require.NoError(t, ballots.Add(db, &ballot))
+			proposal := &types.Proposal{
+				InnerProposal: types.InnerProposal{
+					Ballot:   ballot,
+					TxIDs:    []types.TransactionID{{3, 4}},
+					MeshHash: types.RandomHash(),
+				},
+				Signature: types.RandomEdSignature(),
+			}
+			proposal.SetID(types.RandomProposalID())
+			require.NoError(t, Add(db, proposal))
+		}
+		got, err := GetByLayer(db, types.LayerID(i))
+		require.NoError(t, err)
+		require.Len(t, got, numProps)
+	}
+	require.NoError(t, Delete(db, types.LayerID(maxLayers)))
+	for i := 1; i < maxLayers; i++ {
+		_, err := GetByLayer(db, types.LayerID(i))
+		require.ErrorIs(t, err, sql.ErrNotFound)
+	}
+	got, err := GetByLayer(db, types.LayerID(maxLayers))
+	require.NoError(t, err)
+	require.Len(t, got, numProps)
+}
+
 func TestHas(t *testing.T) {
 	db := sql.InMemory()
 	nodeID := types.RandomNodeID()