Merge branch 'develop' into parametrize-out-of-sync-threshold

node/node.go

@@ -1338,7 +1338,7 @@ func (app *App) setupDBs(ctx context.Context, lg log.Log) error {
 	sqlDB, err := sql.Open("file:"+filepath.Join(dbPath, dbFile),
 		sql.WithConnections(app.Config.DatabaseConnections),
 		sql.WithLatencyMetering(app.Config.DatabaseLatencyMetering),
-		sql.WithV4Migration(util.ExtractActiveSet),
+		sql.WithV5Migration(util.ExtractActiveSet),
 	)
 	if err != nil {
 		return fmt.Errorf("open sqlite db %w", err)

sql/database.go

@@ -61,7 +61,7 @@ type conf struct {
 	enableLatency bool
 
 	// TODO: remove after state is pruned for majority
-	v4Migration func(Executor) error
+	v5Migration func(Executor) error
 }
 
 // WithConnections overwrites number of pooled connections.
@@ -78,9 +78,9 @@ func WithMigrations(migrations Migrations) Opt {
 	}
 }
 
-func WithV4Migration(cb func(Executor) error) Opt {
+func WithV5Migration(cb func(Executor) error) Opt {
 	return func(c *conf) {
-		c.v4Migration = cb
+		c.v5Migration = cb
 	}
 }
 
@@ -145,9 +145,9 @@ func Open(uri string, opts ...Opt) (*Database, error) {
 		if err != nil {
 			return nil, err
 		}
-		if before <= 3 && after == 4 && config.v4Migration != nil {
-			// v4 migration (active set extraction) needs the 3rd migration to execute first
-			if err := config.v4Migration(db); err != nil {
+		if before <= 4 && after == 5 && config.v5Migration != nil {
+			// v5 migration (active set extraction) needs the 4rd migration to execute first
+			if err := config.v5Migration(db); err != nil {
 				return nil, err
 			}
 			if err := Vacuum(db); err != nil {

sql/migrations/0004_v1.1.7.sql

@@ -0,0 +1 @@
+CREATE INDEX ballots_by_atx_by_layer ON ballots (atx, layer asc);

sql/migrations_test.go

@@ -15,5 +15,5 @@ func TestMigrationsAppliedOnce(t *testing.T) {
 		return true
 	})
 	require.NoError(t, err)
-	require.Equal(t, version, 4)
+	require.Equal(t, version, 5)
 }

tortoise/state.go

@@ -56,7 +56,7 @@ type (
 		evicted types.LayerID
 
 		epochs map[types.EpochID]*epochInfo
-		layers map[types.LayerID]*layerInfo
+		layers layerSlice
 		// ballots should not be referenced by other ballots
 		// each ballot stores references (votes) for X previous layers
 		// those X layers may reference another set of ballots that will
@@ -75,7 +75,6 @@ type (
 func newState() *state {
 	return &state{
 		epochs:     map[types.EpochID]*epochInfo{},
-		layers:     map[types.LayerID]*layerInfo{},
 		ballots:    map[types.LayerID][]*ballotInfo{},
 		ballotRefs: map[types.BallotID]*ballotInfo{},
 		malnodes:   map[types.NodeID]struct{}{},
@@ -91,13 +90,7 @@ func (s *state) expectedWeight(cfg Config, target types.LayerID) weight {
 }
 
 func (s *state) layer(lid types.LayerID) *layerInfo {
-	layer, exist := s.layers[lid]
-	if !exist {
-		layersNumber.Inc()
-		layer = &layerInfo{lid: lid, opinions: map[types.Hash32]votes{}}
-		s.layers[lid] = layer
-	}
-	return layer
+	return s.layers.get(s.evicted, lid)
 }
 
 func (s *state) epoch(eid types.EpochID) *epochInfo {
@@ -503,3 +496,24 @@ func decodeVotes(evicted, blid types.LayerID, base *ballotInfo, exceptions types
 	}
 	return decoded, from, nil
 }
+
+type layerSlice struct {
+	data []*layerInfo
+}
+
+func (s *layerSlice) get(offset, index types.LayerID) *layerInfo {
+	i := index - offset - 1
+	lth := types.LayerID(len(s.data))
+	if i < lth {
+		return s.data[i]
+	}
+	last := offset + lth
+	for lid := last + 1; lid <= index; lid++ {
+		s.data = append(s.data, &layerInfo{lid: lid, opinions: map[types.Hash32]votes{}})
+	}
+	return s.data[i]
+}
+
+func (s *layerSlice) pop() {
+	s.data = s.data[1:]
+}

tortoise/state_test.go

@@ -395,3 +395,17 @@ func TestStateDecodeVotes(t *testing.T) {
 		})
 	}
 }
+
+func TestLayersSliceBoundaries(t *testing.T) {
+	slice := layerSlice{}
+	const (
+		total    = 100_000
+		capacity = 10_000
+	)
+	for i := types.LayerID(0); i < total; i++ {
+		slice.get(i, i+capacity)
+		slice.pop()
+		require.True(t, cap(slice.data) < capacity+capacity/2, "slice expected to double on append: cap=%d", cap(slice.data))
+		require.Equal(t, capacity-1, len(slice.data))
+	}
+}

tortoise/tortoise.go

@@ -57,11 +57,8 @@ func newTurtle(logger *zap.Logger, config Config) *turtle {
 	t.evicted = genesis.Sub(1)
 
 	t.epochs[genesis.GetEpoch()] = &epochInfo{atxs: map[types.ATXID]atxInfo{}}
-	t.layers[genesis] = &layerInfo{
-		lid:            genesis,
-		hareTerminated: true,
-		opinions:       map[types.Hash32]votes{},
-	}
+	genlayer := t.layer(genesis)
+	genlayer.hareTerminated = true
 	t.verifying = newVerifying(config, t.state)
 	t.full = newFullTortoise(config, t.state)
 	t.full.counted = genesis
@@ -92,6 +89,7 @@ func (t *turtle) evict(ctx context.Context) {
 		zap.Stringer("from_layer", t.evicted.Add(1)),
 		zap.Stringer("upto_layer", windowStart),
 	)
+	layersNumber.Set(float64(len(t.layers.data)))
 	if !windowStart.After(t.evicted) {
 		return
 	}
@@ -100,21 +98,17 @@ func (t *turtle) evict(ctx context.Context) {
 	}
 	for lid := t.evicted.Add(1); lid.Before(windowStart); lid = lid.Add(1) {
 		for _, ballot := range t.ballots[lid] {
-			ballotsNumber.Dec()
 			delete(t.ballotRefs, ballot.id)
 		}
-		for range t.layers[lid].blocks {
-			blocksNumber.Dec()
-		}
-		layersNumber.Dec()
-		delete(t.layers, lid)
+
+		ballotsNumber.Sub(float64(len(t.ballots[lid])))
+		blocksNumber.Sub(float64(len(t.layer(lid).blocks)))
+		t.layers.pop()
+
 		delete(t.ballots, lid)
 		if lid.OrdinalInEpoch() == types.GetLayersPerEpoch()-1 {
-			layersNumber.Dec()
 			epoch := t.epoch(lid.GetEpoch())
-			for range epoch.atxs {
-				atxsNumber.Dec()
-			}
+			atxsNumber.Sub(float64(len(epoch.atxs)))
 			delete(t.epochs, lid.GetEpoch())
 		}
 	}

tortoise/tortoise_test.go

@@ -1932,9 +1932,7 @@ func TestStateManagement(t *testing.T) {
 
 	evicted := tortoise.trtl.evicted
 	require.Equal(t, verified.Sub(window).Sub(1), evicted)
-	for lid := types.GetEffectiveGenesis(); !lid.After(evicted); lid = lid.Add(1) {
-		require.Empty(t, tortoise.trtl.layers[lid])
-	}
+	require.EqualValues(t, evicted+1, tortoise.trtl.layers.data[0].lid)
 
 	for lid := evicted.Add(1); !lid.After(last); lid = lid.Add(1) {
 		for _, ballot := range tortoise.trtl.ballots[lid] {

tortoise/verifying_test.go

@@ -584,9 +584,12 @@ func TestVerifying_Verify(t *testing.T) {
 			state.verified = verified
 			state.processed = processed
 			state.last = processed
-			state.layers = tc.layers
+			for lid, info := range tc.layers {
+				layer := state.layers.get(state.evicted, lid)
+				*layer = *info
+			}
 			refs := map[types.BlockID]*blockInfo{}
-			for _, layer := range state.layers {
+			for _, layer := range state.layers.data {
 				for _, block := range layer.blocks {
 					refs[block.id] = block
 					state.updateRefHeight(layer, block)