Merge branch 'develop' into proper-latency

.golangci.yml

@@ -603,11 +603,7 @@ issues:
   # exclude:
   #   - abcdef
   # Excluding configuration per-path, per-linter, per-text and per-source
-  exclude-rules:
-    # Exclude some `staticcheck` messages.
-    - linters:
-        - staticcheck
-      text: "SA1019:"
+  # exclude-rules:
   # Independently of option `exclude` we use default exclude patterns,
   # it can be disabled by this option.
   # To list all excluded by default patterns execute `golangci-lint run --help`.

CHANGELOG.md

@@ -19,14 +19,24 @@ Support for old certificate sync protocol is dropped. This update is incompatibl
 ### Features
 
 * [#5031](https://github.com/spacemeshos/go-spacemesh/pull/5031) Nodes will also fetch from PoET 112 for round 4 if they were able to register to PoET 110.
+* [#5067](https://github.com/spacemeshos/go-spacemesh/pull/5067) dbstat virtual table can be read periodically to collect table/index sizes.
+
+In order to enable provide following configuration:
+```json
+"main": {
+    "db-size-metering-interval": "10m"
+}
+```
 
 ### Improvements
 
 * [#4998](https://github.com/spacemeshos/go-spacemesh/pull/4998) First phase of state size reduction.
   Ephemeral data are deleted and state compacted at the time of upgrade. In steady-state, data is pruned periodically.
 * [#5021](https://github.com/spacemeshos/go-spacemesh/pull/5021) Drop support for old certificate sync protocol.
 * [#5024](https://github.com/spacemeshos/go-spacemesh/pull/5024) Active set will be saved in state separately from ballots.
+* [#5032](https://github.com/spacemeshos/go-spacemesh/pull/5032) Ativeset data pruned from ballots.
 * [#5035](https://github.com/spacemeshos/go-spacemesh/pull/5035) Fix possible nil pointer panic when node fails to persist nipost builder state.
+* [#5079](https://github.com/spacemeshos/go-spacemesh/pull/5079) increase atx cache to 50 000 to reduce disk reads.
 
 ## v1.1.5
 

activation/nipost.go

@@ -7,12 +7,12 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
+	"slices"
 	"time"
 
 	"github.com/spacemeshos/merkle-tree"
 	"github.com/spacemeshos/poet/shared"
 	"github.com/spacemeshos/post/proving"
-	"golang.org/x/exp/slices"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/spacemeshos/go-spacemesh/activation/metrics"
@@ -469,14 +469,6 @@ func (nb *NIPostBuilder) addPoETMitigation(ctx context.Context, from, to string,
 
 func (nb *NIPostBuilder) getBestProof(ctx context.Context, challenge types.Hash32, publishEpoch types.EpochID) (types.PoetProofRef, *types.MerkleProof, error) {
 	switch publishEpoch {
-	case 4:
-		// because PoET 111 had a hardware issue when challenges for round 3 were submitted, no node could submit to it
-		// 111 was recovered with the PoET 110 DB, so all successful submissions to 110 should be able to be fetched from there as well
-		// TODO(mafa): remove after next PoET round; https://github.com/spacemeshos/go-spacemesh/issues/4968
-		err := nb.addPoETMitigation(ctx, "https://poet-110.spacemesh.network", "https://poet-111.spacemesh.network", 4)
-		if err != nil {
-			nb.log.With().Error("pub epoch 4 mitigation: failed to add PoET 111 to state for pub epoch 4", log.Err(err))
-		}
 	case 5:
 		// PoET 112 came online after the registration window for round 4 ended, so no node could submit to it
 		// 112 was initialized with the PoET 110 DB, so all successful submissions to 110 should be able to be fetched from there as well

activation/nipost_test.go

@@ -180,7 +180,8 @@ func spawnPoet(tb testing.TB, opts ...HTTPPoetOpt) *HTTPPoetTestHarness {
 		eg.Wait()
 	})
 	eg.Go(func() error {
-		return poetProver.Service.Start(ctx)
+		err := poetProver.Service.Start(ctx)
+		return errors.Join(err, poetProver.Service.Close())
 	})
 
 	return poetProver
@@ -1048,13 +1049,6 @@ func TestNIPostBuilder_Mainnet_PoetRound3_Workaround(t *testing.T) {
 		to    string
 		epoch types.EpochID
 	}{
-		{
-			// TODO(mafa): remove after epoch 4 end; https://github.com/spacemeshos/go-spacemesh/issues/4968
-			name:  "epoch 4: PoET 111 restore with PoET 110",
-			from:  "https://poet-110.spacemesh.network",
-			to:    "https://poet-111.spacemesh.network",
-			epoch: 4,
-		},
 		{
 			// TODO(mafa): remove after epoch 5 end; https://github.com/spacemeshos/go-spacemesh/issues/5030
 			name:  "epoch 5: PoET 112 restore with PoET 110",

activation/poet_test.go

@@ -36,7 +36,8 @@ func TestHTTPPoet(t *testing.T) {
 	r.NotNil(c)
 
 	eg.Go(func() error {
-		return c.Service.Start(ctx)
+		err := c.Service.Start(ctx)
+		return errors.Join(err, c.Service.Close())
 	})
 
 	client, err := NewHTTPPoetClient(c.RestURL().String(), DefaultPoetConfig(), WithLogger(zaptest.NewLogger(t)))

api/grpcserver/activation_service.go

@@ -5,12 +5,12 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/golang/protobuf/ptypes/empty"
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap"
 	pb "github.com/spacemeshos/api/release/go/spacemesh/v1"
 	"go.uber.org/zap"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/emptypb"
 
 	"github.com/spacemeshos/go-spacemesh/common/types"
 	"github.com/spacemeshos/go-spacemesh/events"
@@ -68,7 +68,7 @@ func (s *activationService) Get(ctx context.Context, request *pb.GetRequest) (*p
 	return resp, nil
 }
 
-func (s *activationService) Highest(ctx context.Context, req *empty.Empty) (*pb.HighestResponse, error) {
+func (s *activationService) Highest(ctx context.Context, req *emptypb.Empty) (*pb.HighestResponse, error) {
 	highest, err := s.atxProvider.MaxHeightAtx()
 	if err != nil {
 		return &pb.HighestResponse{

api/grpcserver/activation_service_test.go

@@ -6,12 +6,12 @@ import (
 	"math/rand"
 	"testing"
 
-	"github.com/golang/protobuf/ptypes/empty"
 	pb "github.com/spacemeshos/api/release/go/spacemesh/v1"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/emptypb"
 
 	"github.com/spacemeshos/go-spacemesh/api/grpcserver"
 	"github.com/spacemeshos/go-spacemesh/common/types"
@@ -26,7 +26,7 @@ func Test_Highest_ReturnsGoldenAtxOnError(t *testing.T) {
 	activationService := grpcserver.NewActivationService(atxProvider, goldenAtx)
 
 	atxProvider.EXPECT().MaxHeightAtx().Return(types.EmptyATXID, errors.New("blah"))
-	response, err := activationService.Highest(context.Background(), &empty.Empty{})
+	response, err := activationService.Highest(context.Background(), &emptypb.Empty{})
 	require.NoError(t, err)
 	require.Equal(t, goldenAtx.Bytes(), response.Atx.Id.Id)
 	require.Nil(t, response.Atx.Layer)
@@ -62,7 +62,7 @@ func Test_Highest_ReturnsMaxTickHeight(t *testing.T) {
 	atxProvider.EXPECT().MaxHeightAtx().Return(id, nil)
 	atxProvider.EXPECT().GetFullAtx(id).Return(&atx, nil)
 
-	response, err := activationService.Highest(context.Background(), &empty.Empty{})
+	response, err := activationService.Highest(context.Background(), &emptypb.Empty{})
 	require.NoError(t, err)
 	require.Equal(t, atx.ID().Bytes(), response.Atx.Id.Id)
 	require.Equal(t, atx.PublishEpoch.Uint32(), response.Atx.Layer.Number)

api/grpcserver/admin_service.go

@@ -8,14 +8,14 @@ import (
 	"os"
 	"time"
 
-	"github.com/golang/protobuf/ptypes/empty"
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap"
 	pb "github.com/spacemeshos/api/release/go/spacemesh/v1"
 	"github.com/spf13/afero"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/emptypb"
 
 	"github.com/spacemeshos/go-spacemesh/checkpoint"
 	"github.com/spacemeshos/go-spacemesh/common/types"
@@ -102,10 +102,10 @@ func (a AdminService) CheckpointStream(req *pb.CheckpointStreamRequest, stream p
 	}
 }
 
-func (a AdminService) Recover(ctx context.Context, _ *pb.RecoverRequest) (*empty.Empty, error) {
+func (a AdminService) Recover(ctx context.Context, _ *pb.RecoverRequest) (*emptypb.Empty, error) {
 	ctxzap.Info(ctx, "going to recover from checkpoint")
 	a.recover()
-	return &empty.Empty{}, nil
+	return &emptypb.Empty{}, nil
 }
 
 func (a AdminService) EventsStream(req *pb.EventStreamRequest, stream pb.AdminService_EventsStreamServer) error {
@@ -140,7 +140,7 @@ func (a AdminService) EventsStream(req *pb.EventStreamRequest, stream pb.AdminSe
 	}
 }
 
-func (a AdminService) PeerInfoStream(_ *empty.Empty, stream pb.AdminService_PeerInfoStreamServer) error {
+func (a AdminService) PeerInfoStream(_ *emptypb.Empty, stream pb.AdminService_PeerInfoStreamServer) error {
 	for _, p := range a.p.GetPeers() {
 		select {
 		case <-stream.Context().Done():

api/grpcserver/admin_service_test.go

@@ -57,7 +57,8 @@ func TestAdminService_Checkpoint(t *testing.T) {
 	db := sql.InMemory()
 	createMesh(t, db)
 	svc := NewAdminService(db, t.TempDir(), nil)
-	t.Cleanup(launchServer(t, cfg, svc))
+	cfg, cleanup := launchServer(t, svc)
+	t.Cleanup(cleanup)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -92,7 +93,8 @@ func TestAdminService_Checkpoint(t *testing.T) {
 func TestAdminService_CheckpointError(t *testing.T) {
 	db := sql.InMemory()
 	svc := NewAdminService(db, t.TempDir(), nil)
-	t.Cleanup(launchServer(t, cfg, svc))
+	cfg, cleanup := launchServer(t, svc)
+	t.Cleanup(cleanup)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -111,7 +113,8 @@ func TestAdminService_Recovery(t *testing.T) {
 	svc := NewAdminService(db, t.TempDir(), nil)
 	svc.recover = func() { recoveryCalled.Store(true) }
 
-	t.Cleanup(launchServer(t, cfg, svc))
+	cfg, cleanup := launchServer(t, svc)
+	t.Cleanup(cleanup)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()

api/grpcserver/config.go

@@ -47,8 +47,8 @@ func DefaultConfig() Config {
 // DefaultTestConfig returns the default config for tests.
 func DefaultTestConfig() Config {
 	conf := DefaultConfig()
-	conf.PublicListener = "127.0.0.1:19092"
-	conf.PrivateListener = "127.0.0.1:19093"
-	conf.JSONListener = "127.0.0.1:19094"
+	conf.PublicListener = "127.0.0.1:0"
+	conf.PrivateListener = "127.0.0.1:0"
+	conf.JSONListener = "127.0.0.1:0"
 	return conf
 }

api/grpcserver/debug_service.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/golang/protobuf/ptypes/empty"
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap"
 	pb "github.com/spacemeshos/api/release/go/spacemesh/v1"
 	"go.uber.org/zap"
@@ -78,7 +77,7 @@ func (d DebugService) Accounts(ctx context.Context, in *pb.AccountsRequest) (*pb
 }
 
 // NetworkInfo query provides NetworkInfoResponse.
-func (d DebugService) NetworkInfo(ctx context.Context, _ *empty.Empty) (*pb.NetworkInfoResponse, error) {
+func (d DebugService) NetworkInfo(ctx context.Context, _ *emptypb.Empty) (*pb.NetworkInfoResponse, error) {
 	return &pb.NetworkInfoResponse{Id: d.identity.ID().String()}, nil
 }
 

api/grpcserver/grpc.go

@@ -19,7 +19,7 @@ type ServiceAPI interface {
 
 // Server is a very basic grpc server.
 type Server struct {
-	Listener string
+	listener string
 	logger   log.Logger
 	// BoundAddress contains the address that the server bound to, useful if
 	// the server uses a dynamic port. It is set during startup and can be
@@ -34,7 +34,7 @@ type Server struct {
 func New(listener string, lg log.Logger, opts ...grpc.ServerOption) *Server {
 	opts = append(opts, ServerOptions...)
 	return &Server{
-		Listener:   listener,
+		listener:   listener,
 		logger:     lg,
 		GrpcServer: grpc.NewServer(opts...),
 	}
@@ -43,15 +43,15 @@ func New(listener string, lg log.Logger, opts ...grpc.ServerOption) *Server {
 // Start starts the server.
 func (s *Server) Start() error {
 	s.logger.With().Info("starting grpc server",
-		log.String("address", s.Listener),
+		log.String("address", s.listener),
 		log.Array("services", log.ArrayMarshalerFunc(func(encoder log.ArrayEncoder) error {
 			for svc := range s.GrpcServer.GetServiceInfo() {
 				encoder.AppendString(svc)
 			}
 			return nil
 		})),
 	)
-	lis, err := net.Listen("tcp", s.Listener)
+	lis, err := net.Listen("tcp", s.listener)
 	if err != nil {
 		s.logger.Error("error listening: %v", err)
 		return err
@@ -60,7 +60,7 @@ func (s *Server) Start() error {
 	reflection.Register(s.GrpcServer)
 	s.grp.Go(func() error {
 		if err := s.GrpcServer.Serve(lis); err != nil {
-			s.logger.Error("error stopping grpc server: %v", err)
+			s.logger.Error("error serving grpc server: %v", err)
 			return err
 		}
 		return nil