remove read/write for ballot.ActiveSet (#5024)

## Motivation
Closes #4984

## Changes
- drop EmitEmptyActiveSet config
- remove all access to Ballot.ActiveSet and look up from db instead
- remove all writes of Ballot.ActiveSet

after this PR, miner will still accept proposal/ballot with non-empty active sets, but will save ballots without active set.

CHANGELOG.md

@@ -20,6 +20,7 @@ Support for old certificate sync protocol is dropped. This update is incompatibl
 
 ### Improvements
 * [#5021](https://github.com/spacemeshos/go-spacemesh/pull/5021) Drop support for old certificate sync protocol.
+* [#5024](https://github.com/spacemeshos/go-spacemesh/pull/5024) Active set will be saved in state separately from ballots.
 
 ## v1.1.5
 
@@ -41,7 +42,7 @@ active set will not be gossipped together with proposals. That was the main netw
   * `postdata_metadata.json` is now updated atomically to prevent corruption of the file.
 * [#4956](https://github.com/spacemeshos/go-spacemesh/pull/4956) Active set is will not be gossipped in every proposal.
   Active set usually contains list of atxs that targets current epoch. As the number of atxs grows this object grows as well.
-* [#4993](https://github.com/spacemeshos/go-spacemesh/pull/4993) Drop proposals after genering a block. This limits growth of the state.
+* [#4993](https://github.com/spacemeshos/go-spacemesh/pull/4993) Drop proposals after generating a block. This limits growth of the state.
 
 ## v1.1.4
 

api/grpcserver/grpcserver_test.go

@@ -50,6 +50,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/signing"
 	"github.com/spacemeshos/go-spacemesh/sql"
 	"github.com/spacemeshos/go-spacemesh/sql/accounts"
+	"github.com/spacemeshos/go-spacemesh/sql/activesets"
 	"github.com/spacemeshos/go-spacemesh/sql/atxs"
 	"github.com/spacemeshos/go-spacemesh/sql/identities"
 	"github.com/spacemeshos/go-spacemesh/system"
@@ -58,7 +59,6 @@ import (
 
 const (
 	labelsPerUnit  = 2048
-	bitsPerLabel   = 8
 	numUnits       = 2
 	genTimeUnix    = 1000000
 	layerDuration  = 10 * time.Second
@@ -71,7 +71,6 @@ const (
 	accountBalance = 8675301
 	accountCounter = 0
 	rewardAmount   = 5551234
-	activesetSize  = 10001
 )
 
 var (
@@ -214,8 +213,7 @@ func TestMain(m *testing.M) {
 	// These create circular dependencies so they have to be initialized
 	// after the global vars
 	ballot1.AtxID = globalAtx.ID()
-	ballot1.EpochData = &types.EpochData{}
-	ballot1.ActiveSet = []types.ATXID{globalAtx.ID(), globalAtx2.ID()}
+	ballot1.EpochData = &types.EpochData{ActiveSetHash: types.ATXIDList{globalAtx.ID(), globalAtx2.ID()}.Hash()}
 
 	globalTx = NewTx(0, addr1, signer1)
 	globalTx2 = NewTx(1, addr2, signer2)
@@ -1043,7 +1041,9 @@ func TestMeshService(t *testing.T) {
 	genesis := time.Unix(genTimeUnix, 0)
 	genTime.EXPECT().GenesisTime().Return(genesis)
 	genTime.EXPECT().CurrentLayer().Return(layerCurrent).AnyTimes()
-	grpcService := NewMeshService(datastore.NewCachedDB(sql.InMemory(), logtest.New(t)), meshAPIMock, conStateAPI, genTime, layersPerEpoch, types.Hash20{}, layerDuration, layerAvgSize, txsPerProposal)
+	db := datastore.NewCachedDB(sql.InMemory(), logtest.New(t))
+	grpcService := NewMeshService(db, meshAPIMock, conStateAPI, genTime, layersPerEpoch, types.Hash20{}, layerDuration, layerAvgSize, txsPerProposal)
+	require.NoError(t, activesets.Add(db, ballot1.EpochData.ActiveSetHash, &types.EpochActiveSet{Set: types.ATXIDList{globalAtx.ID(), globalAtx2.ID()}}))
 	t.Cleanup(launchServer(t, cfg, grpcService))
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
@@ -2163,7 +2163,9 @@ func TestLayerStream_comprehensive(t *testing.T) {
 
 	ctrl := gomock.NewController(t)
 	genTime := NewMockgenesisTimeAPI(ctrl)
-	grpcService := NewMeshService(datastore.NewCachedDB(sql.InMemory(), logtest.New(t)), meshAPIMock, conStateAPI, genTime, layersPerEpoch, types.Hash20{}, layerDuration, layerAvgSize, txsPerProposal)
+	db := datastore.NewCachedDB(sql.InMemory(), logtest.New(t))
+	grpcService := NewMeshService(db, meshAPIMock, conStateAPI, genTime, layersPerEpoch, types.Hash20{}, layerDuration, layerAvgSize, txsPerProposal)
+	require.NoError(t, activesets.Add(db, ballot1.EpochData.ActiveSetHash, &types.EpochActiveSet{Set: types.ATXIDList{globalAtx.ID(), globalAtx2.ID()}}))
 	t.Cleanup(launchServer(t, cfg, grpcService))
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

api/grpcserver/mesh_service.go

@@ -18,6 +18,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/datastore"
 	"github.com/spacemeshos/go-spacemesh/events"
 	"github.com/spacemeshos/go-spacemesh/sql"
+	"github.com/spacemeshos/go-spacemesh/sql/activesets"
 )
 
 // MeshService exposes mesh data such as accounts, blocks, and transactions.
@@ -124,17 +125,19 @@ func (s MeshService) getFilteredTransactions(from types.LayerID, address types.A
 
 func (s MeshService) getFilteredActivations(ctx context.Context, startLayer types.LayerID, addr types.Address) (activations []*types.VerifiedActivationTx, err error) {
 	// We have no way to look up activations by coinbase so we have no choice but to read all of them.
-	// TODO: index activations by layer (and maybe by coinbase)
-	// See https://github.com/spacemeshos/go-spacemesh/issues/2064.
 	var atxids []types.ATXID
 	for l := startLayer; !l.After(s.mesh.LatestLayer()); l = l.Add(1) {
 		layer, err := s.mesh.GetLayer(l)
 		if layer == nil || err != nil {
 			return nil, status.Errorf(codes.Internal, "error retrieving layer data")
 		}
 		for _, b := range layer.Ballots() {
-			if b.EpochData != nil && b.ActiveSet != nil {
-				atxids = append(atxids, b.ActiveSet...)
+			if b.EpochData != nil {
+				actives, err := activesets.Get(s.cdb, b.EpochData.ActiveSetHash)
+				if err != nil {
+					return nil, status.Errorf(codes.Internal, "error retrieving active set %s (%s)", b.ID().String(), b.EpochData.ActiveSetHash.ShortString())
+				}
+				atxids = append(atxids, actives.Set...)
 			}
 		}
 	}
@@ -334,8 +337,12 @@ func (s MeshService) readLayer(ctx context.Context, layerID types.LayerID, layer
 	}
 
 	for _, b := range layer.Ballots() {
-		if b.EpochData != nil && b.ActiveSet != nil {
-			activations = append(activations, b.ActiveSet...)
+		if b.EpochData != nil {
+			actives, err := activesets.Get(s.cdb, b.EpochData.ActiveSetHash)
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "error retrieving active set %s (%s)", b.ID().String(), b.EpochData.ActiveSetHash.ShortString())
+			}
+			activations = append(activations, actives.Set...)
 		}
 	}
 

blocks/generator_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/log/logtest"
 	"github.com/spacemeshos/go-spacemesh/signing"
 	"github.com/spacemeshos/go-spacemesh/sql"
+	"github.com/spacemeshos/go-spacemesh/sql/activesets"
 	"github.com/spacemeshos/go-spacemesh/sql/atxs"
 	"github.com/spacemeshos/go-spacemesh/sql/ballots"
 	"github.com/spacemeshos/go-spacemesh/sql/layers"
@@ -192,10 +193,10 @@ func createProposal(
 					EpochData: &types.EpochData{
 						Beacon:           types.RandomBeacon(),
 						EligibilityCount: uint32(layerSize * epochSize / len(activeSet)),
+						ActiveSetHash:    activeSet.Hash(),
 					},
 				},
 				EligibilityProofs: make([]types.VotingEligibility, numEligibility),
-				ActiveSet:         activeSet,
 			},
 			TxIDs:    txIDs,
 			MeshHash: meshHash,
@@ -207,6 +208,7 @@ func createProposal(
 	require.NoError(t, p.Initialize())
 	require.NoError(t, ballots.Add(db, &p.Ballot))
 	require.NoError(t, proposals.Add(db, p))
+	activesets.Add(db, activeSet.Hash(), &types.EpochActiveSet{Set: activeSet})
 	return p
 }
 

blocks/utils_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/log/logtest"
 	"github.com/spacemeshos/go-spacemesh/signing"
 	"github.com/spacemeshos/go-spacemesh/sql"
+	"github.com/spacemeshos/go-spacemesh/sql/activesets"
 	"github.com/spacemeshos/go-spacemesh/sql/layers"
 )
 
@@ -103,7 +104,7 @@ func Test_getProposalMetadata(t *testing.T) {
 	cfg := Config{OptFilterThreshold: 70}
 	lid := types.LayerID(111)
 	_, atxs := createATXs(t, cdb, (lid.GetEpoch() - 1).FirstLayer(), 10)
-	actives := types.ToATXIDs(atxs)
+	actives := types.ATXIDList(types.ToATXIDs(atxs))
 	props := make([]*types.Proposal, 0, 10)
 	hash1 := types.Hash32{1, 2, 3}
 	hash2 := types.Hash32{3, 2, 1}
@@ -119,11 +120,14 @@ func Test_getProposalMetadata(t *testing.T) {
 		for j := 0; j <= i; j++ {
 			p.EligibilityProofs = append(p.EligibilityProofs, types.VotingEligibility{J: uint32(j + 1)})
 		}
-		p.EpochData = &types.EpochData{}
-		p.ActiveSet = actives
+		p.EpochData = &types.EpochData{ActiveSetHash: actives.Hash()}
 		p.EpochData.EligibilityCount = uint32(i + 1)
 		props = append(props, &p)
 	}
+	require.NoError(t, activesets.Add(cdb, actives.Hash(), &types.EpochActiveSet{
+		Epoch: lid.GetEpoch(),
+		Set:   actives,
+	}))
 	require.NoError(t, layers.SetMeshHash(cdb, lid-1, hash2))
 
 	// only 5 / 10 proposals has the same state

common/types/ballot.go

@@ -289,12 +289,12 @@ func (b *Ballot) IsMalicious() bool {
 // MarshalLogObject implements logging encoder for Ballot.
 func (b *Ballot) MarshalLogObject(encoder log.ObjectEncoder) error {
 	var (
-		activeSetSize = 0
-		beacon        Beacon
+		activeHash Hash32
+		beacon     Beacon
 	)
 
 	if b.EpochData != nil {
-		activeSetSize = len(b.ActiveSet)
+		activeHash = b.EpochData.ActiveSetHash
 		beacon = b.EpochData.Beacon
 	}
 
@@ -309,7 +309,7 @@ func (b *Ballot) MarshalLogObject(encoder log.ObjectEncoder) error {
 	encoder.AddInt("abstain", len(b.Votes.Abstain))
 	encoder.AddString("atx_id", b.AtxID.String())
 	encoder.AddString("ref_ballot", b.RefBallot.String())
-	encoder.AddInt("active_set_size", activeSetSize)
+	encoder.AddString("active set hash", activeHash.ShortString())
 	encoder.AddString("beacon", beacon.ShortString())
 	encoder.AddObject("votes", &b.Votes)
 	return nil

config/mainnet.go

@@ -80,7 +80,6 @@ func MainnetConfig() Config {
 			// 1000 - is assumed minimal number of units
 			// 5000 - half of the expected poet ticks
 			MinimalActiveSetWeight: 1000 * 5000,
-			EmitEmptyActiveSet:     20000,
 		},
 		HARE: hareConfig.Config{
 			N:               200,

hare/eligibility/oracle.go

@@ -19,6 +19,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/miner"
 	"github.com/spacemeshos/go-spacemesh/signing"
 	"github.com/spacemeshos/go-spacemesh/sql"
+	"github.com/spacemeshos/go-spacemesh/sql/activesets"
 	"github.com/spacemeshos/go-spacemesh/sql/ballots"
 	"github.com/spacemeshos/go-spacemesh/system"
 )
@@ -462,11 +463,24 @@ func (o *Oracle) activeSetFromRefBallots(epoch types.EpochID) ([]types.ATXID, er
 			o.Log.With().Debug("beacon mismatch", log.Stringer("local", beacon), log.Object("ballot", ballot))
 			continue
 		}
-		for _, id := range ballot.ActiveSet {
+		actives, err := activesets.Get(o.cdb, ballot.EpochData.ActiveSetHash)
+		if err != nil {
+			o.Log.With().Error("failed to get active set",
+				log.String("actives hash", ballot.EpochData.ActiveSetHash.ShortString()),
+				log.String("ballot ", ballot.ID().String()),
+				log.Err(err),
+			)
+			continue
+		}
+		for _, id := range actives.Set {
 			activeMap[id] = struct{}{}
 		}
 	}
-	o.Log.With().Warning("using tortoise active set", log.Uint32("epoch", epoch.Uint32()), log.Stringer("beacon", beacon))
+	o.Log.With().Warning("using tortoise active set",
+		log.Int("actives size", len(activeMap)),
+		log.Uint32("epoch", epoch.Uint32()),
+		log.Stringer("beacon", beacon),
+	)
 	return maps.Keys(activeMap), nil
 }