update majority calculation for optimistic filtering (#4879)

## Motivation
Closes #4878

## Changes
this change will take effect at the first layer of epoch 3

blocks/generator.go

@@ -44,17 +44,13 @@ type Generator struct {
 
 // Config is the config for Generator.
 type Config struct {
-	LayerSize          uint32
-	LayersPerEpoch     uint32
 	GenBlockInterval   time.Duration
 	BlockGasLimit      uint64
 	OptFilterThreshold int
 }
 
 func defaultConfig() Config {
 	return Config{
-		LayerSize:          50,
-		LayersPerEpoch:     3,
 		GenBlockInterval:   time.Second,
 		BlockGasLimit:      math.MaxUint64,
 		OptFilterThreshold: 90,

blocks/generator_test.go

@@ -44,8 +44,6 @@ const (
 
 func testConfig() Config {
 	return Config{
-		LayerSize:          layerSize,
-		LayersPerEpoch:     epochSize,
 		GenBlockInterval:   10 * time.Millisecond,
 		BlockGasLimit:      math.MaxUint64,
 		OptFilterThreshold: 90,

blocks/utils.go

@@ -69,15 +69,22 @@ func getProposalMetadata(
 	if err != nil {
 		return nil, err
 	}
+	upgrade := lid.Uint32() >= types.OpUpgradeLayer()
+	total := 0
 	for _, p := range proposals {
 		key := p.MeshHash
+		cnt := 1
+		if upgrade {
+			cnt = len(p.EligibilityProofs)
+		}
+		total += cnt
 		if _, ok := meshHashes[key]; !ok {
 			meshHashes[key] = &meshState{
 				hash:  p.MeshHash,
-				count: 1,
+				count: cnt,
 			}
 		} else {
-			meshHashes[key].count++
+			meshHashes[key].count += cnt
 		}
 
 		for _, tid := range p.TxIDs {
@@ -95,12 +102,13 @@ func getProposalMetadata(
 			mtxs = append(mtxs, mtx)
 		}
 	}
-	majority := cfg.OptFilterThreshold * len(proposals)
+	majority := cfg.OptFilterThreshold * total
 	var majorityState *meshState
 	for _, ms := range meshHashes {
 		logger.With().Debug("mesh hash",
 			ms.hash,
 			log.Int("count", ms.count),
+			log.Int("total", total),
 			log.Int("threshold", cfg.OptFilterThreshold),
 			log.Int("num_proposals", len(proposals)))
 		if ms.hash != types.EmptyLayerHash && ms.count*100 >= majority {
@@ -223,7 +231,7 @@ func rewardInfoAndHeight(logger log.Log, cdb *datastore.CachedDB, cfg Config, pr
 			max = atx.BaseTickHeight
 		}
 		ballot := &p.Ballot
-		weightPer, err := proposals.ComputeWeightPerEligibility(cdb, ballot, cfg.LayerSize, cfg.LayersPerEpoch)
+		weightPer, err := proposals.ComputeWeightPerEligibility(cdb, ballot)
 		if err != nil {
 			logger.With().Error("failed to calculate weight per eligibility", p.ID(), log.Err(err))
 			return 0, nil, err

blocks/utils_test.go

@@ -1,16 +1,26 @@
 package blocks
 
 import (
+	"context"
 	"math/rand"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/spacemeshos/go-spacemesh/common/types"
+	"github.com/spacemeshos/go-spacemesh/datastore"
 	"github.com/spacemeshos/go-spacemesh/log/logtest"
 	"github.com/spacemeshos/go-spacemesh/signing"
+	"github.com/spacemeshos/go-spacemesh/sql"
+	"github.com/spacemeshos/go-spacemesh/sql/layers"
 )
 
+func TestMain(m *testing.M) {
+	types.SetLayersPerEpoch(5)
+	os.Exit(m.Run())
+}
+
 func checkInNonceOrder(t *testing.T, tids []types.TransactionID, byTid map[types.TransactionID]*types.MeshTransaction) {
 	accounts := make(map[types.Address]uint64)
 	for _, tid := range tids {
@@ -86,3 +96,44 @@ func Test_getBlockTXs(t *testing.T) {
 	require.NoError(t, err)
 	require.Empty(t, got)
 }
+
+func Test_getProposalMetadata(t *testing.T) {
+	lg := logtest.New(t)
+	cdb := datastore.NewCachedDB(sql.InMemory(), lg)
+	cfg := Config{OptFilterThreshold: 70}
+	lid := types.LayerID(111)
+	_, atxs := createATXs(t, cdb, (lid.GetEpoch() - 1).FirstLayer(), 10)
+	actives := types.ToATXIDs(atxs)
+	props := make([]*types.Proposal, 0, 10)
+	hash1 := types.Hash32{1, 2, 3}
+	hash2 := types.Hash32{3, 2, 1}
+	for i := 0; i < 10; i++ {
+		var p types.Proposal
+		p.Layer = lid
+		p.AtxID = atxs[i].ID()
+		if i < 5 {
+			p.MeshHash = hash1
+		} else {
+			p.MeshHash = hash2
+		}
+		for j := 0; j <= i; j++ {
+			p.EligibilityProofs = append(p.EligibilityProofs, types.VotingEligibility{J: uint32(j + 1)})
+		}
+		p.EpochData = &types.EpochData{}
+		p.ActiveSet = actives
+		p.EpochData.EligibilityCount = uint32(i + 1)
+		props = append(props, &p)
+	}
+	require.NoError(t, layers.SetMeshHash(cdb, lid-1, hash2))
+
+	types.SetOpUpgradeLayer(lid.Uint32() + 1)
+	// only 5 / 10 proposals has the same state, threshold is 70
+	md, err := getProposalMetadata(context.Background(), lg, cdb, cfg, lid, props)
+	require.NoError(t, err)
+	require.False(t, md.optFilter)
+	// eligibility wise 40 / 55 has the same state, threshold is 70
+	types.SetOpUpgradeLayer(0)
+	md, err = getProposalMetadata(context.Background(), lg, cdb, cfg, lid, props)
+	require.NoError(t, err)
+	require.True(t, md.optFilter)
+}

common/types/layer.go

@@ -21,6 +21,9 @@ var (
 	EmptyLayerHash = Hash32{}
 
 	legacyLayer uint32
+
+	// layer at which optimistic filtering majority calculation is upgraded.
+	opUpgradeLayer uint32
 )
 
 // SetLayersPerEpoch sets global parameter of layers per epoch, all conversions from layer to epoch use this param.
@@ -37,6 +40,14 @@ func GetLegacyLayer() uint32 {
 	return legacyLayer
 }
 
+func SetOpUpgradeLayer(layer uint32) {
+	opUpgradeLayer = layer
+}
+
+func OpUpgradeLayer() uint32 {
+	return opUpgradeLayer
+}
+
 func SetEffectiveGenesis(layer uint32) {
 	atomic.StoreUint32(&effectiveGenesis, layer)
 }

node/node.go

@@ -141,6 +141,8 @@ func GetCommand() *cobra.Command {
 			run := func(ctx context.Context) error {
 				types.SetLayersPerEpoch(app.Config.LayersPerEpoch)
 				types.SetLegacyLayers(app.Config.LegacyLayer)
+				// starting on 2023-09-14 20:00:00 +0000 UTC (~1 week into 4th epoch)
+				types.SetOpUpgradeLayer(18000)
 				// ensure all data folders exist
 				if err := os.MkdirAll(app.Config.DataDir(), 0o700); err != nil {
 					return fmt.Errorf("ensure folders exist: %w", err)
@@ -735,8 +737,6 @@ func (app *App) initServices(ctx context.Context) error {
 	app.blockGen = blocks.NewGenerator(app.cachedDB, executor, msh, fetcherWrapped, app.certifier, patrol,
 		blocks.WithContext(ctx),
 		blocks.WithConfig(blocks.Config{
-			LayerSize:          layerSize,
-			LayersPerEpoch:     layersPerEpoch,
 			BlockGasLimit:      app.Config.BlockGasLimit,
 			OptFilterThreshold: app.Config.OptFilterThreshold,
 			GenBlockInterval:   500 * time.Millisecond,

proposals/util/util.go

@@ -60,8 +60,6 @@ func GetNumEligibleSlots(weight, minWeight, totalWeight uint64, committeeSize, l
 func ComputeWeightPerEligibility(
 	cdb *datastore.CachedDB,
 	ballot *types.Ballot,
-	layerSize,
-	layersPerEpoch uint32,
 ) (*big.Rat, error) {
 	var (
 		refBallot = ballot

proposals/util_test.go

@@ -20,7 +20,6 @@ import (
 )
 
 func TestComputeWeightPerEligibility(t *testing.T) {
-	types.SetLayersPerEpoch(layersPerEpoch)
 	signer, err := signing.NewEdSigner(
 		signing.WithKeyFromRand(rand.New(rand.NewSource(1001))),
 	)
@@ -49,15 +48,14 @@ func TestComputeWeightPerEligibility(t *testing.T) {
 	}
 	expectedWeight := big.NewRat(int64(testedATXUnit), int64(eligibleSlots))
 	for _, b := range blts {
-		got, err := ComputeWeightPerEligibility(cdb, b, layerAvgSize, layersPerEpoch)
+		got, err := ComputeWeightPerEligibility(cdb, b)
 		require.NoError(t, err)
 		require.NotNil(t, got)
 		require.Equal(t, 0, got.Cmp(expectedWeight))
 	}
 }
 
 func TestComputeWeightPerEligibility_EmptyRefBallotID(t *testing.T) {
-	types.SetLayersPerEpoch(layersPerEpoch)
 	signer, err := signing.NewEdSigner(
 		signing.WithKeyFromRand(rand.New(rand.NewSource(1001))),
 	)
@@ -68,13 +66,12 @@ func TestComputeWeightPerEligibility_EmptyRefBallotID(t *testing.T) {
 	b := blts[1]
 	b.RefBallot = types.EmptyBallotID
 	cdb := datastore.NewCachedDB(sql.InMemory(), logtest.New(t))
-	got, err := ComputeWeightPerEligibility(cdb, b, layerAvgSize, layersPerEpoch)
+	got, err := ComputeWeightPerEligibility(cdb, b)
 	require.ErrorIs(t, err, putil.ErrBadBallotData)
 	require.Nil(t, got)
 }
 
 func TestComputeWeightPerEligibility_FailToGetRefBallot(t *testing.T) {
-	types.SetLayersPerEpoch(layersPerEpoch)
 	signer, err := signing.NewEdSigner(
 		signing.WithKeyFromRand(rand.New(rand.NewSource(1001))),
 	)
@@ -83,22 +80,21 @@ func TestComputeWeightPerEligibility_FailToGetRefBallot(t *testing.T) {
 	blts := createBallots(t, signer, genActiveSet(), beacon)
 	require.GreaterOrEqual(t, 2, len(blts))
 	cdb := datastore.NewCachedDB(sql.InMemory(), logtest.New(t))
-	got, err := ComputeWeightPerEligibility(cdb, blts[1], layerAvgSize, layersPerEpoch)
+	got, err := ComputeWeightPerEligibility(cdb, blts[1])
 	require.ErrorIs(t, err, sql.ErrNotFound)
 	require.True(t, strings.Contains(err.Error(), "missing ref ballot"))
 	require.Nil(t, got)
 }
 
 func TestComputeWeightPerEligibility_FailATX(t *testing.T) {
-	types.SetLayersPerEpoch(layersPerEpoch)
 	signer, err := signing.NewEdSigner(
 		signing.WithKeyFromRand(rand.New(rand.NewSource(1001))),
 	)
 	require.NoError(t, err)
 	beacon := types.Beacon{1, 1, 1}
 	blts := createBallots(t, signer, genActiveSet(), beacon)
 	cdb := datastore.NewCachedDB(sql.InMemory(), logtest.New(t))
-	got, err := ComputeWeightPerEligibility(cdb, blts[0], layerAvgSize, layersPerEpoch)
+	got, err := ComputeWeightPerEligibility(cdb, blts[0])
 	require.ErrorIs(t, err, sql.ErrNotFound)
 	require.True(t, strings.Contains(err.Error(), "missing atx"))
 	require.Nil(t, got)