Problem: memiavl more efficient snapshot setup is not used (#1067)

* Problem: memiavl more efficient snapshot setup is not used

Solution:
- only export state-sync snapshot on a snapshot, which is more efficient than export an arbitrary version

* Update CHANGELOG.md

Signed-off-by: yihuang <huang@crypto.com>

* fix unit test

* try fix mac build

* align to macos-11

* fix integration test

* revert CI change

* Problem: install-nix-action fails with recent macos update

Solution:
- update to v22, see cachix/install-nix-action#184

* go mod tidy

* fix integration test

* better error message

---------

Signed-off-by: yihuang <huang@crypto.com>
Co-authored-by: mmsqe <mavis@crypto.com>

CHANGELOG.md

@@ -47,6 +47,7 @@
 - [#1035](https://github.com/crypto-org-chain/cronos/pull/1035) Support caching in memiavl directly, ignore inter-block cache silently.
 - [#1050](https://github.com/crypto-org-chain/cronos/pull/1050) nativebyteorder mode will check endianness on startup, binaries are built with nativebyteorder by default.
 - [#1064](https://github.com/crypto-org-chain/cronos/pull/1064) Simplify memiavl snapshot switching.
+- [#1067](https://github.com/crypto-org-chain/cronos/pull/1067) memiavl: only export state-sync snapshots on an exist snapshot
 
 *April 13, 2023*
 

app/memiavl.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/cosmos/cosmos-sdk/baseapp"
 	servertypes "github.com/cosmos/cosmos-sdk/server/types"
-	storetypes "github.com/cosmos/cosmos-sdk/store/types"
 
 	"github.com/crypto-org-chain/cronos/memiavl"
 	"github.com/crypto-org-chain/cronos/store/rootmulti"
@@ -25,22 +24,29 @@ const (
 
 func SetupMemIAVL(logger log.Logger, homePath string, appOpts servertypes.AppOptions, baseAppOptions []func(*baseapp.BaseApp)) []func(*baseapp.BaseApp) {
 	if cast.ToBool(appOpts.Get(FlagMemIAVL)) {
-		// cms must be overridden before the other options, because they may use the cms,
-		// make sure the cms aren't be overridden by the other options later on.
-		cms := rootmulti.NewStore(filepath.Join(homePath, "data", "memiavl.db"), logger)
-		cms.SetMemIAVLOptions(memiavl.Options{
+		opts := memiavl.Options{
 			AsyncCommitBuffer:  cast.ToInt(appOpts.Get(FlagAsyncCommitBuffer)),
 			ZeroCopy:           cast.ToBool(appOpts.Get(FlagZeroCopy)),
 			SnapshotKeepRecent: cast.ToUint32(appOpts.Get(FlagSnapshotKeepRecent)),
 			SnapshotInterval:   cast.ToUint32(appOpts.Get(FlagSnapshotInterval)),
 			CacheSize:          cast.ToInt(appOpts.Get(FlagCacheSize)),
-		})
-		baseAppOptions = append([]func(*baseapp.BaseApp){setCMS(cms)}, baseAppOptions...)
+		}
+		// cms must be overridden before the other options, because they may use the cms,
+		// make sure the cms aren't be overridden by the other options later on.
+		baseAppOptions = append([]func(*baseapp.BaseApp){setMemIAVL(homePath, logger, opts)}, baseAppOptions...)
 	}
 
 	return baseAppOptions
 }
 
-func setCMS(cms storetypes.CommitMultiStore) func(*baseapp.BaseApp) {
-	return func(bapp *baseapp.BaseApp) { bapp.SetCMS(cms) }
+func setMemIAVL(homePath string, logger log.Logger, opts memiavl.Options) func(*baseapp.BaseApp) {
+	return func(bapp *baseapp.BaseApp) {
+		// trigger state-sync snapshot creation by memiavl
+		opts.TriggerStateSyncExport = func(height int64) {
+			go bapp.SnapshotManager().SnapshotIfApplicable(height)
+		}
+		cms := rootmulti.NewStore(filepath.Join(homePath, "data", "memiavl.db"), logger)
+		cms.SetMemIAVLOptions(opts)
+		bapp.SetCMS(cms)
+	}
 }

go.mod

@@ -6,7 +6,7 @@ require (
 	cosmossdk.io/errors v1.0.0-beta.7
 	cosmossdk.io/math v1.0.0-rc.0
 	github.com/armon/go-metrics v0.4.1
-	github.com/cosmos/cosmos-sdk v0.46.13-rc.0
+	github.com/cosmos/cosmos-sdk v0.46.14-0.20230619012023-673ee3a54518
 	github.com/cosmos/gogoproto v1.4.7
 	github.com/cosmos/ibc-go/v5 v5.2.1
 	github.com/crypto-org-chain/cronos/memiavl v0.0.2

go.sum

@@ -412,8 +412,8 @@ github.com/cosmos/btcutil v1.0.5 h1:t+ZFcX77LpKtDBhjucvnOH8C2l2ioGsBNEQ3jef8xFk=
 github.com/cosmos/btcutil v1.0.5/go.mod h1:IyB7iuqZMJlthe2tkIFL33xPyzbFYP0XVdS8P5lUPis=
 github.com/cosmos/cosmos-proto v1.0.0-beta.1 h1:iDL5qh++NoXxG8hSy93FdYJut4XfgbShIocllGaXx/0=
 github.com/cosmos/cosmos-proto v1.0.0-beta.1/go.mod h1:8k2GNZghi5sDRFw/scPL8gMSowT1vDA+5ouxL8GjaUE=
-github.com/cosmos/cosmos-sdk v0.46.13-rc.0 h1:WwWcR8ZnuKYfWSQfNRXIzOFT0x5vtCHwzPWOjV5x/Wo=
-github.com/cosmos/cosmos-sdk v0.46.13-rc.0/go.mod h1:maYupex8eSZZhw9jow4D/7qWV9XpYB/fyVu7MIbD3RM=
+github.com/cosmos/cosmos-sdk v0.46.14-0.20230619012023-673ee3a54518 h1:IBJ1EaG3dyJpJ9FYUlpslCP+Rfcz0feRMBr384cIicg=
+github.com/cosmos/cosmos-sdk v0.46.14-0.20230619012023-673ee3a54518/go.mod h1:ABDNny0muxj03rmJcyR0mG7O9QPGktl5e30EVsHvORM=
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
 github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY=
 github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw=

gomod2nix.toml

@@ -137,8 +137,8 @@ schema = 3
     version = "v1.0.0-beta.1"
     hash = "sha256-oATkuj+fM5eBn+ywO+w/tL0AFSIEkx0J3Yz+VhVe0QA="
   [mod."github.com/cosmos/cosmos-sdk"]
-    version = "v0.46.13-rc.0"
-    hash = "sha256-TESRQDJ4/YXgCUIU9udOErhem6AupqOnzzev0i18MZs="
+    version = "v0.46.14-0.20230619012023-673ee3a54518"
+    hash = "sha256-9GDhGWyc/DW81P7uYq7H157Nl5xiERq/SzLtGiptHso="
   [mod."github.com/cosmos/go-bip39"]
     version = "v1.0.0"
     hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA="

integration_tests/test_basic.py

@@ -264,9 +264,11 @@ def test_local_statesync(cronos):
     wait_for_block(cli0, 6)
 
     sync_info = cli0.status()["SyncInfo"]
-    height = int(sync_info["latest_block_height"])
     cronos.supervisorctl("stop", "cronos_777-1-node0")
     tarball = cli0.data_dir / "snapshot.tar.gz"
+    height = int(sync_info["latest_block_height"])
+    # round down to multplies of memiavl.snapshot-interval
+    height -= height % 5
 
     if height not in set(item.height for item in cli0.list_snapshot()):
         cli0.export_snapshot(height)

memiavl/db.go

@@ -45,6 +45,9 @@ type DB struct {
 	snapshotInterval uint32
 	// make sure only one snapshot rewrite is running
 	pruneSnapshotLock sync.Mutex
+	// it's more efficient to export snapshot versions, so we only support that by default
+	supportExportNonSnapshotVersion bool
+	triggerStateSyncExport          func(height int64)
 
 	// invariant: the LastIndex always match the current version of MultiTree
 	wal         *wal.Log
@@ -73,6 +76,9 @@ type Options struct {
 	InitialStores      []string
 	SnapshotKeepRecent uint32
 	SnapshotInterval   uint32
+	// it's more efficient to export snapshot versions, we can filter out the non-snapshot versions
+	SupportExportNonSnapshotVersion bool
+	TriggerStateSyncExport          func(height int64)
 	// load the target version instead of latest version
 	TargetVersion uint32
 	// Buffer size for the asynchronous commit queue, -1 means synchronous commit,
@@ -135,13 +141,15 @@ func Load(dir string, opts Options) (*DB, error) {
 	}
 
 	db := &DB{
-		MultiTree:          *mtree,
-		logger:             opts.Logger,
-		dir:                dir,
-		wal:                wal,
-		walChanSize:        opts.AsyncCommitBuffer,
-		snapshotKeepRecent: opts.SnapshotKeepRecent,
-		snapshotInterval:   opts.SnapshotInterval,
+		MultiTree:                       *mtree,
+		logger:                          opts.Logger,
+		dir:                             dir,
+		wal:                             wal,
+		walChanSize:                     opts.AsyncCommitBuffer,
+		snapshotKeepRecent:              opts.SnapshotKeepRecent,
+		snapshotInterval:                opts.SnapshotInterval,
+		supportExportNonSnapshotVersion: opts.SupportExportNonSnapshotVersion,
+		triggerStateSyncExport:          opts.TriggerStateSyncExport,
 	}
 
 	if db.logger == nil {
@@ -242,6 +250,11 @@ func (db *DB) checkBackgroundSnapshotRewrite() error {
 		db.logger.Info("switched to new snapshot", "version", db.MultiTree.Version())
 
 		db.pruneSnapshots()
+
+		// trigger state-sync snapshot export
+		if db.triggerStateSyncExport != nil {
+			db.triggerStateSyncExport(db.SnapshotVersion())
+		}
 	default:
 	}
 
@@ -609,6 +622,10 @@ func currentPath(root string) string {
 	return filepath.Join(root, "current")
 }
 
+func snapshotPath(root string, version uint32) string {
+	return filepath.Join(root, snapshotName(version))
+}
+
 func currentTmpPath(root string) string {
 	return filepath.Join(root, "current-tmp")
 }

memiavl/export.go

@@ -20,13 +20,31 @@ func (db *DB) Snapshot(height uint64, protoWriter protoio.Writer) error {
 	if height > math.MaxUint32 {
 		return fmt.Errorf("height overflows uint32: %d", height)
 	}
+	version := uint32(height)
 
-	mtree, err := Load(db.dir, Options{
-		TargetVersion: uint32(height),
-		ZeroCopy:      true,
-	})
-	if err != nil {
-		return errors.Wrapf(err, "invalid snapshot height: %d", height)
+	var mtree *MultiTree
+	if db.supportExportNonSnapshotVersion {
+		db, err := Load(db.dir, Options{
+			TargetVersion: version,
+			ZeroCopy:      true,
+		})
+		if err != nil {
+			return errors.Wrapf(err, "invalid height: %d", height)
+		}
+
+		mtree = &db.MultiTree
+	} else {
+		curVersion, err := currentVersion(db.dir)
+		if err != nil {
+			return fmt.Errorf("failed to load current version: %w", err)
+		}
+		if int64(version) > curVersion {
+			return fmt.Errorf("snapshot is not created yet: height: %d", version)
+		}
+		mtree, err = LoadMultiTree(snapshotPath(db.dir, version), true, 0)
+		if err != nil {
+			return errors.Wrapf(err, "snapshot don't exists: height: %d", version)
+		}
 	}
 
 	for _, tree := range mtree.trees {

memiavl/go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/tidwall/wal v1.1.7
 	golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0
 	golang.org/x/sync v0.1.0
+	golang.org/x/sys v0.7.0
 )
 
 require (
@@ -78,7 +79,6 @@ require (
 	go.uber.org/atomic v1.10.0 // indirect
 	golang.org/x/crypto v0.7.0 // indirect
 	golang.org/x/net v0.9.0 // indirect
-	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
 	google.golang.org/grpc v1.54.0 // indirect

memiavl/snapshot_test.go

@@ -132,9 +132,10 @@ func TestSnapshotImportExport(t *testing.T) {
 
 func TestDBSnapshotRestore(t *testing.T) {
 	db, err := Load(t.TempDir(), Options{
-		CreateIfMissing:   true,
-		InitialStores:     []string{"test"},
-		AsyncCommitBuffer: -1,
+		CreateIfMissing:                 true,
+		InitialStores:                   []string{"test"},
+		AsyncCommitBuffer:               -1,
+		SupportExportNonSnapshotVersion: true,
 	})
 	require.NoError(t, err)
 

store/go.mod

@@ -14,15 +14,20 @@ require (
 )
 
 require (
+	github.com/DataDog/zstd v1.4.5 // indirect
 	github.com/VictoriaMetrics/metrics v1.23.1 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
 	github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.8.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230117234908-f91ad392a02b // indirect
+	github.com/cockroachdb/redact v1.0.8 // indirect
+	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
 	github.com/cosmos/gogoproto v1.4.7 // indirect
-	github.com/cosmos/gorocksdb v1.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
 	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
@@ -41,8 +46,11 @@ require (
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
 	github.com/klauspost/compress v1.16.0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263 // indirect
 	github.com/ledgerwatch/log/v3 v3.7.0 // indirect
+	github.com/linxGnu/grocksdb v1.7.16 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
@@ -53,6 +61,7 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/sasha-s/go-deadlock v0.3.1 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/testify v1.8.2 // indirect
@@ -88,4 +97,6 @@ replace (
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	// use cometbft
 	github.com/tendermint/tendermint => github.com/cometbft/cometbft v0.34.29
+	// https://github.com/crypto-org-chain/tm-db/tree/release/v0.6.x
+	github.com/tendermint/tm-db => github.com/crypto-org-chain/tm-db v0.6.8-0.20230424032152-87c7e7f4fb61
 )