From 4b06e4f25e9a97880ddf834b571b05a59fa1290c Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 11 Jul 2023 21:43:23 +0800 Subject: [PATCH] core/state: value diff tracking in StateDB (#27349) This change makes the StateDB track the state key value diff of a block transition. We already tracked current account and storage values for the purpose of updating the state snapshot. With this PR, we now also track the original (pre-transition) values of accounts and storage slots. --- core/state/dump.go | 2 +- core/state/journal.go | 14 +- core/state/metrics.go | 7 + core/state/snapshot/generate.go | 2 +- core/state/snapshot/generate_test.go | 2 +- core/state/state_object.go | 124 +++++--- core/state/state_test.go | 16 +- core/state/statedb.go | 405 ++++++++++++++++++------- core/state/statedb_fuzz_test.go | 373 +++++++++++++++++++++++ core/state/statedb_test.go | 2 +- core/types/state_account.go | 23 ++ eth/protocols/snap/sync_test.go | 8 +- light/postprocess.go | 4 +- light/trie.go | 25 +- tests/fuzzers/stacktrie/trie_fuzzer.go | 2 +- tests/fuzzers/trie/trie-fuzzer.go | 2 +- trie/database.go | 9 +- trie/iterator_test.go | 18 +- trie/secure_trie_test.go | 2 +- trie/sync_test.go | 6 +- trie/tracer_test.go | 18 +- trie/trie_test.go | 20 +- trie/trienode/node.go | 24 +- trie/triestate/state.go | 28 ++ 24 files changed, 908 insertions(+), 228 deletions(-) create mode 100644 core/state/statedb_fuzz_test.go create mode 100644 trie/triestate/state.go diff --git a/core/state/dump.go b/core/state/dump.go index 2d9c65e473e63..3ea987731fe38 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -171,7 +171,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] } else { address = &addr } - obj := newObject(s, addr, data) + obj := newObject(s, addr, &data) if !conf.SkipCode { account.Code = obj.Code(s.db) } diff --git a/core/state/journal.go b/core/state/journal.go index cf8ef3e3875cd..95a218b3a3498 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -95,6 +95,10 @@ type ( prevdestruct bool prevAccount []byte prevStorage map[common.Hash][]byte + + prevAccountOriginExist bool + prevAccountOrigin []byte + prevStorageOrigin map[common.Hash][]byte } suicideChange struct { account *common.Address @@ -163,10 +167,16 @@ func (ch resetObjectChange) revert(s *StateDB) { delete(s.stateObjectsDestruct, ch.prev.address) } if ch.prevAccount != nil { - s.snapAccounts[ch.prev.addrHash] = ch.prevAccount + s.accounts[ch.prev.addrHash] = ch.prevAccount } if ch.prevStorage != nil { - s.snapStorage[ch.prev.addrHash] = ch.prevStorage + s.storages[ch.prev.addrHash] = ch.prevStorage + } + if ch.prevAccountOriginExist { + s.accountsOrigin[ch.prev.addrHash] = ch.prevAccountOrigin + } + if ch.prevStorageOrigin != nil { + s.storagesOrigin[ch.prev.addrHash] = ch.prevStorageOrigin } } diff --git a/core/state/metrics.go b/core/state/metrics.go index e702ef3a81a60..64c651461e731 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -27,4 +27,11 @@ var ( storageTriesUpdatedMeter = metrics.NewRegisteredMeter("state/update/storagenodes", nil) accountTrieDeletedMeter = metrics.NewRegisteredMeter("state/delete/accountnodes", nil) storageTriesDeletedMeter = metrics.NewRegisteredMeter("state/delete/storagenodes", nil) + + slotDeletionMaxCount = metrics.NewRegisteredGauge("state/delete/storage/max/slot", nil) + slotDeletionMaxSize = metrics.NewRegisteredGauge("state/delete/storage/max/size", nil) + slotDeletionTimer = metrics.NewRegisteredResettingTimer("state/delete/storage/timer", nil) + slotDeletionCount = metrics.NewRegisteredMeter("state/delete/storage/slot", nil) + slotDeletionSize = metrics.NewRegisteredMeter("state/delete/storage/size", nil) + slotDeletionSkip = metrics.NewRegisteredGauge("state/delete/storage/skip", nil) ) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index f5db0ff921b30..f80525fcd29be 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -366,7 +366,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi return false, nil, err } if nodes != nil { - tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) tdb.Commit(root, false) } resolver = func(owner common.Hash, path []byte, hash common.Hash) []byte { diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 20a389e477617..8996877ceb810 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -203,7 +203,7 @@ func (t *testHelper) Commit() common.Hash { if nodes != nil { t.nodes.Merge(nodes) } - t.triedb.Update(root, types.EmptyRootHash, t.nodes) + t.triedb.Update(root, types.EmptyRootHash, t.nodes, nil) t.triedb.Commit(root, false) return root } diff --git a/core/state/state_object.go b/core/state/state_object.go index b443d12f126c7..2529ec69290f1 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -57,29 +57,35 @@ func (s Storage) Copy() Storage { // stateObject represents an Ethereum account which is being modified. // // The usage pattern is as follows: -// First you need to obtain a state object. -// Account values can be accessed and modified through the object. -// Finally, call commitTrie to write the modified storage trie into a database. +// - First you need to obtain a state object. +// - Account values as well as storages can be accessed and modified through the object. +// - Finally, call commit to return the changes of storage trie and update account data. type stateObject struct { - address common.Address - addrHash common.Hash // hash of ethereum address of the account - data types.StateAccount db *StateDB + address common.Address // address of ethereum account + addrHash common.Hash // hash of ethereum address of the account + origin *types.StateAccount // Account original data without any change applied, nil means it was not existent + data types.StateAccount // Account data with all mutations applied in the scope of block // Write caches. trie Trie // storage trie, which becomes non-nil on first access code Code // contract bytecode, which gets set when code is loaded - originStorage Storage // Storage cache of original entries to dedup rewrites, reset for every transaction + originStorage Storage // Storage cache of original entries to dedup rewrites pendingStorage Storage // Storage entries that need to be flushed to disk, at the end of an entire block - dirtyStorage Storage // Storage entries that have been modified in the current transaction execution + dirtyStorage Storage // Storage entries that have been modified in the current transaction execution, reset for every transaction // Cache flags. - // When an object is marked suicided it will be deleted from the trie - // during the "update" phase of the state transition. dirtyCode bool // true if the code was updated - suicided bool - deleted bool + + // Flag whether the account was marked as suicided. The suicided account + // is still accessible in the scope of same transaction. + suicided bool + + // Flag whether the account was marked as deleted. The suicided account + // or the account is considered as empty will be marked as deleted at + // the end of transaction and no longer accessible anymore. + deleted bool } // empty returns whether the account is considered empty. @@ -88,21 +94,17 @@ func (s *stateObject) empty() bool { } // newObject creates a state object. -func newObject(db *StateDB, address common.Address, data types.StateAccount) *stateObject { - if data.Balance == nil { - data.Balance = new(big.Int) - } - if data.CodeHash == nil { - data.CodeHash = types.EmptyCodeHash.Bytes() - } - if data.Root == (common.Hash{}) { - data.Root = types.EmptyRootHash +func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *stateObject { + origin := acct + if acct == nil { + acct = types.NewEmptyStateAccount() } return &stateObject{ db: db, address: address, addrHash: crypto.Keccak256Hash(address[:]), - data: data, + origin: origin, + data: *acct, originStorage: make(Storage), pendingStorage: make(Storage), dirtyStorage: make(Storage), @@ -135,10 +137,8 @@ func (s *stateObject) touch() { func (s *stateObject) getTrie(db Database) (Trie, error) { if s.trie == nil { // Try fetching from prefetcher first - // We don't prefetch empty tries if s.data.Root != types.EmptyRootHash && s.db.prefetcher != nil { - // When the miner is creating the pending state, there is no - // prefetcher + // When the miner is creating the pending state, there is no prefetcher s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root) } if s.trie == nil { @@ -277,6 +277,7 @@ func (s *stateObject) updateTrie(db Database) (Trie, error) { // The snapshot storage map for the object var ( storage map[common.Hash][]byte + origin map[common.Hash][]byte hasher = s.db.hasher ) tr, err := s.getTrie(db) @@ -291,6 +292,7 @@ func (s *stateObject) updateTrie(db Database) (Trie, error) { if value == s.originStorage[key] { continue } + prev := s.originStorage[key] s.originStorage[key] = value // rlp-encoded value to be used by the snapshot @@ -311,17 +313,34 @@ func (s *stateObject) updateTrie(db Database) (Trie, error) { } s.db.StorageUpdated += 1 } - // If state snapshotting is active, cache the data til commit - if s.db.snap != nil { - if storage == nil { - // Retrieve the old storage map, if available, create a new one otherwise - if storage = s.db.snapStorage[s.addrHash]; storage == nil { - storage = make(map[common.Hash][]byte) - s.db.snapStorage[s.addrHash] = storage - } + // Cache the mutated storage slots until commit + if storage == nil { + if storage = s.db.storages[s.addrHash]; storage == nil { + storage = make(map[common.Hash][]byte) + s.db.storages[s.addrHash] = storage + } + } + khash := crypto.HashData(hasher, key[:]) + storage[khash] = snapshotVal // snapshotVal will be nil if it's deleted + + // Cache the original value of mutated storage slots + if origin == nil { + if origin = s.db.storagesOrigin[s.addrHash]; origin == nil { + origin = make(map[common.Hash][]byte) + s.db.storagesOrigin[s.addrHash] = origin + } + } + // Track the original value of slot only if it's mutated first time + if _, ok := origin[khash]; !ok { + if prev == (common.Hash{}) { + origin[khash] = nil // nil if it was not present previously + } else { + // Encoding []byte cannot fail, ok to ignore the error. + b, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(prev[:])) + origin[khash] = b } - storage[crypto.HashData(hasher, key[:])] = snapshotVal // will be nil if it's deleted } + // Cache the items for preloading usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure } if s.db.prefetcher != nil { @@ -351,15 +370,15 @@ func (s *stateObject) updateRoot(db Database) { s.data.Root = tr.Hash() } -// commitTrie submits the storage changes into the storage trie and re-computes -// the root. Besides, all trie changes will be collected in a nodeset and returned. -func (s *stateObject) commitTrie(db Database) (*trienode.NodeSet, error) { +// commit returns the changes made in storage trie and updates the account data. +func (s *stateObject) commit(db Database) (*trienode.NodeSet, error) { tr, err := s.updateTrie(db) if err != nil { return nil, err } // If nothing changed, don't bother with committing anything if tr == nil { + s.origin = s.data.Copy() return nil, nil } // Track the amount of time wasted on committing the storage trie @@ -371,6 +390,9 @@ func (s *stateObject) commitTrie(db Database) (*trienode.NodeSet, error) { return nil, err } s.data.Root = root + + // Update original account data after commit + s.origin = s.data.Copy() return nodes, nil } @@ -410,18 +432,24 @@ func (s *stateObject) setBalance(amount *big.Int) { } func (s *stateObject) deepCopy(db *StateDB) *stateObject { - stateObject := newObject(db, s.address, s.data) + obj := &stateObject{ + db: db, + address: s.address, + addrHash: s.addrHash, + origin: s.origin, + data: s.data, + } if s.trie != nil { - stateObject.trie = db.db.CopyTrie(s.trie) - } - stateObject.code = s.code - stateObject.dirtyStorage = s.dirtyStorage.Copy() - stateObject.originStorage = s.originStorage.Copy() - stateObject.pendingStorage = s.pendingStorage.Copy() - stateObject.suicided = s.suicided - stateObject.dirtyCode = s.dirtyCode - stateObject.deleted = s.deleted - return stateObject + obj.trie = db.db.CopyTrie(s.trie) + } + obj.code = s.code + obj.dirtyStorage = s.dirtyStorage.Copy() + obj.originStorage = s.originStorage.Copy() + obj.pendingStorage = s.pendingStorage.Copy() + obj.suicided = s.suicided + obj.dirtyCode = s.dirtyCode + obj.deleted = s.deleted + return obj } // diff --git a/core/state/state_test.go b/core/state/state_test.go index 555e1f7454e34..b6c11728bee46 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -30,22 +30,22 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -type stateTest struct { +type stateEnv struct { db ethdb.Database state *StateDB } -func newStateTest() *stateTest { +func newStateEnv() *stateEnv { db := rawdb.NewMemoryDatabase() sdb, _ := New(types.EmptyRootHash, NewDatabase(db), nil) - return &stateTest{db: db, state: sdb} + return &stateEnv{db: db, state: sdb} } func TestDump(t *testing.T) { db := rawdb.NewMemoryDatabase() tdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) sdb, _ := New(types.EmptyRootHash, tdb, nil) - s := &stateTest{db: db, state: sdb} + s := &stateEnv{db: db, state: sdb} // generate a few entries obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01})) @@ -99,7 +99,7 @@ func TestIterativeDump(t *testing.T) { db := rawdb.NewMemoryDatabase() tdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) sdb, _ := New(types.EmptyRootHash, tdb, nil) - s := &stateTest{db: db, state: sdb} + s := &stateEnv{db: db, state: sdb} // generate a few entries obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01})) @@ -133,7 +133,7 @@ func TestIterativeDump(t *testing.T) { } func TestNull(t *testing.T) { - s := newStateTest() + s := newStateEnv() address := common.HexToAddress("0x823140710bf13990e4500136726d8b55") s.state.CreateAccount(address) //value := common.FromHex("0x823140710bf13990e4500136726d8b55") @@ -155,7 +155,7 @@ func TestSnapshot(t *testing.T) { var storageaddr common.Hash data1 := common.BytesToHash([]byte{42}) data2 := common.BytesToHash([]byte{43}) - s := newStateTest() + s := newStateEnv() // snapshot the genesis state genesis := s.state.Snapshot() @@ -186,7 +186,7 @@ func TestSnapshot(t *testing.T) { } func TestSnapshotEmpty(t *testing.T) { - s := newStateTest() + s := newStateEnv() s.state.RevertToSnapshot(s.state.Snapshot()) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 998fb176df947..71a863add7015 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/trie/triestate" ) type revision struct { @@ -69,21 +70,26 @@ type StateDB struct { prefetcher *triePrefetcher trie Trie hasher crypto.KeccakState + snaps *snapshot.Tree // Nil if snapshot is not available + snap snapshot.Snapshot // Nil if snapshot is not available // originalRoot is the pre-state root, before any changes were made. // It will be updated when the Commit is called. originalRoot common.Hash - snaps *snapshot.Tree - snap snapshot.Snapshot - snapAccounts map[common.Hash][]byte - snapStorage map[common.Hash]map[common.Hash][]byte + // These maps hold the state changes (including the corresponding + // original value) that occurred in this **block**. + accounts map[common.Hash][]byte // The mutated accounts in 'slim RLP' encoding + storages map[common.Hash]map[common.Hash][]byte // The mutated slots in prefix-zero trimmed rlp format + accountsOrigin map[common.Hash][]byte // The original value of mutated accounts in 'slim RLP' encoding + storagesOrigin map[common.Hash]map[common.Hash][]byte // The original value of mutated slots in prefix-zero trimmed rlp format - // This map holds 'live' objects, which will get modified while processing a state transition. + // This map holds 'live' objects, which will get modified while processing + // a state transition. stateObjects map[common.Address]*stateObject - stateObjectsPending map[common.Address]struct{} // State objects finalized but not yet written to the trie - stateObjectsDirty map[common.Address]struct{} // State objects modified in the current execution - stateObjectsDestruct map[common.Address]struct{} // State objects destructed in the block + stateObjectsPending map[common.Address]struct{} // State objects finalized but not yet written to the trie + stateObjectsDirty map[common.Address]struct{} // State objects modified in the current execution + stateObjectsDestruct map[common.Address]*types.StateAccount // State objects destructed in the block along with its previous value // DB error. // State objects are used by the consensus core and VM which are @@ -97,11 +103,13 @@ type StateDB struct { // The refund counter, also used by state transitioning. refund uint64 + // The tx context and all occurred logs in the scope of transaction. thash common.Hash txIndex int logs map[common.Hash][]*types.Log logSize uint + // Preimages occurred seen by VM in the scope of block. preimages map[common.Hash][]byte // Per-transaction access list @@ -147,10 +155,14 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) trie: tr, originalRoot: root, snaps: snaps, + accounts: make(map[common.Hash][]byte), + storages: make(map[common.Hash]map[common.Hash][]byte), + accountsOrigin: make(map[common.Hash][]byte), + storagesOrigin: make(map[common.Hash]map[common.Hash][]byte), stateObjects: make(map[common.Address]*stateObject), stateObjectsPending: make(map[common.Address]struct{}), stateObjectsDirty: make(map[common.Address]struct{}), - stateObjectsDestruct: make(map[common.Address]struct{}), + stateObjectsDestruct: make(map[common.Address]*types.StateAccount), logs: make(map[common.Hash][]*types.Log), preimages: make(map[common.Hash][]byte), journal: newJournal(), @@ -159,10 +171,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) hasher: crypto.NewKeccakState(), } if sdb.snaps != nil { - if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil { - sdb.snapAccounts = make(map[common.Hash][]byte) - sdb.snapStorage = make(map[common.Hash]map[common.Hash][]byte) - } + sdb.snap = sdb.snaps.Snapshot(root) } return sdb, nil } @@ -445,14 +454,20 @@ func (s *StateDB) SetState(addr common.Address, key, value common.Hash) { } // SetStorage replaces the entire storage for the specified account with given -// storage. This function should only be used for debugging. +// storage. This function should only be used for debugging and the mutations +// must be discarded afterwards. func (s *StateDB) SetStorage(addr common.Address, storage map[common.Hash]common.Hash) { // SetStorage needs to wipe existing storage. We achieve this by pretending // that the account self-destructed earlier in this block, by flagging // it in stateObjectsDestruct. The effect of doing so is that storage lookups // will not hit disk, since it is assumed that the disk-data is belonging // to a previous incarnation of the object. - s.stateObjectsDestruct[addr] = struct{}{} + // + // TODO(rjl493456442) this function should only be supported by 'unwritable' + // state and all mutations made should all be discarded afterwards. + if _, ok := s.stateObjectsDestruct[addr]; !ok { + s.stateObjectsDestruct[addr] = nil + } stateObject := s.GetOrNewStateObject(addr) for k, v := range storage { stateObject.SetState(s.db, k, v) @@ -476,7 +491,6 @@ func (s *StateDB) Suicide(addr common.Address) bool { }) stateObject.markSuicided() stateObject.data.Balance = new(big.Int) - return true } @@ -522,13 +536,21 @@ func (s *StateDB) updateStateObject(obj *stateObject) { if err := s.trie.UpdateAccount(addr, &obj.data); err != nil { s.setError(fmt.Errorf("updateStateObject (%x) error: %v", addr[:], err)) } - - // If state snapshotting is active, cache the data til commit. Note, this - // update mechanism is not symmetric to the deletion, because whereas it is - // enough to track account updates at commit time, deletions need tracking - // at transaction boundary level to ensure we capture state clearing. - if s.snap != nil { - s.snapAccounts[obj.addrHash] = types.SlimAccountRLP(obj.data) + // Cache the data until commit. Note, this update mechanism is not symmetric + // to the deletion, because whereas it is enough to track account updates + // at commit time, deletions need tracking at transaction boundary level to + // ensure we capture state clearing. + s.accounts[obj.addrHash] = types.SlimAccountRLP(obj.data) + + // Track the original value of mutated account, nil means it was not present. + // Skip if it has been tracked (because updateStateObject may be called + // multiple times in a block). + if _, ok := s.accountsOrigin[obj.addrHash]; !ok { + if obj.origin == nil { + s.accountsOrigin[obj.addrHash] = nil + } else { + s.accountsOrigin[obj.addrHash] = types.SlimAccountRLP(*obj.origin) + } } } @@ -607,7 +629,7 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { } } // Insert into the live set - obj := newObject(s, addr, *data) + obj := newObject(s, addr, data) s.setStateObject(obj) return obj } @@ -629,38 +651,36 @@ func (s *StateDB) GetOrNewStateObject(addr common.Address) *stateObject { // the given address, it is overwritten and returned as the second return value. func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) { prev = s.getDeletedStateObject(addr) // Note, prev might have been deleted, we need that! - newobj = newObject(s, addr, types.StateAccount{}) + newobj = newObject(s, addr, nil) if prev == nil { s.journal.append(createObjectChange{account: &addr}) } else { // The original account should be marked as destructed and all cached // account and storage data should be cleared as well. Note, it must - // be done here, otherwise the destruction event of original one will - // be lost. + // be done here, otherwise the destruction event of "original account" + // will be lost. _, prevdestruct := s.stateObjectsDestruct[prev.address] if !prevdestruct { - s.stateObjectsDestruct[prev.address] = struct{}{} + s.stateObjectsDestruct[prev.address] = prev.origin } - var ( - account []byte - storage map[common.Hash][]byte - ) // There may be some cached account/storage data already since IntermediateRoot // will be called for each transaction before byzantium fork which will always // cache the latest account/storage data. - if s.snap != nil { - account = s.snapAccounts[prev.addrHash] - storage = s.snapStorage[prev.addrHash] - delete(s.snapAccounts, prev.addrHash) - delete(s.snapStorage, prev.addrHash) - } + prevAccount, ok := s.accountsOrigin[prev.addrHash] s.journal.append(resetObjectChange{ - account: &addr, - prev: prev, - prevdestruct: prevdestruct, - prevAccount: account, - prevStorage: storage, + account: &addr, + prev: prev, + prevdestruct: prevdestruct, + prevAccount: s.accounts[prev.addrHash], + prevStorage: s.storages[prev.addrHash], + prevAccountOriginExist: ok, + prevAccountOrigin: prevAccount, + prevStorageOrigin: s.storagesOrigin[prev.addrHash], }) + delete(s.accounts, prev.addrHash) + delete(s.storages, prev.addrHash) + delete(s.accountsOrigin, prev.addrHash) + delete(s.storagesOrigin, prev.addrHash) } s.setStateObject(newobj) if prev != nil && !prev.deleted { @@ -731,16 +751,27 @@ func (s *StateDB) Copy() *StateDB { db: s.db, trie: s.db.CopyTrie(s.trie), originalRoot: s.originalRoot, + accounts: make(map[common.Hash][]byte), + storages: make(map[common.Hash]map[common.Hash][]byte), + accountsOrigin: make(map[common.Hash][]byte), + storagesOrigin: make(map[common.Hash]map[common.Hash][]byte), stateObjects: make(map[common.Address]*stateObject, len(s.journal.dirties)), stateObjectsPending: make(map[common.Address]struct{}, len(s.stateObjectsPending)), stateObjectsDirty: make(map[common.Address]struct{}, len(s.journal.dirties)), - stateObjectsDestruct: make(map[common.Address]struct{}, len(s.stateObjectsDestruct)), + stateObjectsDestruct: make(map[common.Address]*types.StateAccount, len(s.stateObjectsDestruct)), refund: s.refund, logs: make(map[common.Hash][]*types.Log, len(s.logs)), logSize: s.logSize, preimages: make(map[common.Hash][]byte, len(s.preimages)), journal: newJournal(), hasher: crypto.NewKeccakState(), + + // In order for the block producer to be able to use and make additions + // to the snapshot tree, we need to copy that as well. Otherwise, any + // block mined by ourselves will cause gaps in the tree, and force the + // miner to operate trie-backed only. + snaps: s.snaps, + snap: s.snap, } // Copy the dirty states, logs, and preimages for addr := range s.journal.dirties { @@ -774,10 +805,18 @@ func (s *StateDB) Copy() *StateDB { } state.stateObjectsDirty[addr] = struct{}{} } - // Deep copy the destruction flag. - for addr := range s.stateObjectsDestruct { - state.stateObjectsDestruct[addr] = struct{}{} + // Deep copy the destruction markers. + for addr, value := range s.stateObjectsDestruct { + state.stateObjectsDestruct[addr] = value } + // Deep copy the state changes made in the scope of block + // along with their original values. + state.accounts = copyAccounts(s.accounts) + state.storages = copyStorages(s.storages) + state.accountsOrigin = copyAccounts(state.accountsOrigin) + state.storagesOrigin = copyStorages(state.storagesOrigin) + + // Deep copy the logs occurred in the scope of block for hash, logs := range s.logs { cpy := make([]*types.Log, len(logs)) for i, l := range logs { @@ -786,6 +825,7 @@ func (s *StateDB) Copy() *StateDB { } state.logs[hash] = cpy } + // Deep copy the preimages occurred in the scope of block for hash, preimage := range s.preimages { state.preimages[hash] = preimage } @@ -804,28 +844,6 @@ func (s *StateDB) Copy() *StateDB { if s.prefetcher != nil { state.prefetcher = s.prefetcher.copy() } - if s.snaps != nil { - // In order for the miner to be able to use and make additions - // to the snapshot tree, we need to copy that as well. - // Otherwise, any block mined by ourselves will cause gaps in the tree, - // and force the miner to operate trie-backed only - state.snaps = s.snaps - state.snap = s.snap - - // deep copy needed - state.snapAccounts = make(map[common.Hash][]byte, len(s.snapAccounts)) - for k, v := range s.snapAccounts { - state.snapAccounts[k] = v - } - state.snapStorage = make(map[common.Hash]map[common.Hash][]byte, len(s.snapStorage)) - for k, v := range s.snapStorage { - temp := make(map[common.Hash][]byte, len(v)) - for kk, vv := range v { - temp[kk] = vv - } - state.snapStorage[k] = temp - } - } return state } @@ -878,17 +896,18 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { obj.deleted = true // We need to maintain account deletions explicitly (will remain - // set indefinitely). - s.stateObjectsDestruct[obj.address] = struct{}{} - - // If state snapshotting is active, also mark the destruction there. + // set indefinitely). Note only the first occurred self-destruct + // event is tracked. + if _, ok := s.stateObjectsDestruct[obj.address]; !ok { + s.stateObjectsDestruct[obj.address] = obj.origin + } // Note, we can't do this only at the end of a block because multiple // transactions within the same block might self destruct and then // resurrect an account; but the snapshotter needs both events. - if s.snap != nil { - delete(s.snapAccounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a resurrect) - delete(s.snapStorage, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a resurrect) - } + delete(s.accounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a resurrect) + delete(s.storages, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a resurrect) + delete(s.accountsOrigin, obj.addrHash) // Clear out any previously updated account data (may be recreated via a resurrect) + delete(s.storagesOrigin, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a resurrect) } else { obj.finalise(true) // Prefetch slots in the background } @@ -986,6 +1005,137 @@ func (s *StateDB) clearJournalAndRefund() { s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entries } +// deleteStorage iterates the storage trie belongs to the account and mark all +// slots inside as deleted. +func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, map[common.Hash][]byte, *trienode.NodeSet, error) { + start := time.Now() + tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root) + if err != nil { + return false, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) + } + it, err := tr.NodeIterator(nil) + if err != nil { + return false, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) + } + var ( + set = trienode.NewNodeSet(addrHash) + slots = make(map[common.Hash][]byte) + stateSize common.StorageSize + nodeSize common.StorageSize + ) + for it.Next(true) { + // arbitrary stateSize limit, make it configurable + if stateSize+nodeSize > 512*1024*1024 { + log.Info("Skip large storage deletion", "address", addr.Hex(), "states", stateSize, "nodes", nodeSize) + if metrics.EnabledExpensive { + slotDeletionSkip.Inc(1) + } + return true, nil, nil, nil + } + if it.Leaf() { + slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob()) + stateSize += common.StorageSize(common.HashLength + len(it.LeafBlob())) + continue + } + if it.Hash() == (common.Hash{}) { + continue + } + nodeSize += common.StorageSize(len(it.Path()) + len(it.NodeBlob())) + set.AddNode(it.Path(), trienode.NewWithPrev(common.Hash{}, nil, it.NodeBlob())) + } + if err := it.Error(); err != nil { + return false, nil, nil, err + } + if metrics.EnabledExpensive { + if int64(len(slots)) > slotDeletionMaxCount.Value() { + slotDeletionMaxCount.Update(int64(len(slots))) + } + if int64(stateSize+nodeSize) > slotDeletionMaxSize.Value() { + slotDeletionMaxSize.Update(int64(stateSize + nodeSize)) + } + slotDeletionTimer.UpdateSince(start) + slotDeletionCount.Mark(int64(len(slots))) + slotDeletionSize.Mark(int64(stateSize + nodeSize)) + } + return false, slots, set, nil +} + +// handleDestruction processes all destruction markers and deletes the account +// and associated storage slots if necessary. There are four possible situations +// here: +// +// - the account was not existent and be marked as destructed +// +// - the account was not existent and be marked as destructed, +// however, it's resurrected later in the same block. +// +// - the account was existent and be marked as destructed +// +// - the account was existent and be marked as destructed, +// however it's resurrected later in the same block. +// +// In case (a), nothing needs be deleted, nil to nil transition can be ignored. +// +// In case (b), nothing needs be deleted, nil is used as the original value for +// newly created account and storages +// +// In case (c), **original** account along with its storages should be deleted, +// with their values be tracked as original value. +// +// In case (d), **original** account along with its storages should be deleted, +// with their values be tracked as original value. +func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.Hash]struct{}, error) { + incomplete := make(map[common.Hash]struct{}) + for addr, prev := range s.stateObjectsDestruct { + // The original account was non-existing, and it's marked as destructed + // in the scope of block. It can be case (a) or (b). + // - for (a), skip it without doing anything. + // - for (b), track account's original value as nil. It may overwrite + // the data cached in s.accountsOrigin set by 'updateStateObject'. + addrHash := crypto.Keccak256Hash(addr[:]) + if prev == nil { + if _, ok := s.accounts[addrHash]; ok { + s.accountsOrigin[addrHash] = nil // case (b) + } + continue + } + // It can overwrite the data in s.accountsOrigin set by 'updateStateObject'. + s.accountsOrigin[addrHash] = types.SlimAccountRLP(*prev) // case (c) or (d) + + // Short circuit if the storage was empty. + if prev.Root == types.EmptyRootHash { + continue + } + // Remove storage slots belong to the account. + aborted, slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) + if err != nil { + return nil, fmt.Errorf("failed to delete storage, err: %w", err) + } + // The storage is too huge to handle, skip it but mark as incomplete. + // For case (d), the account is resurrected might with a few slots + // created. In this case, wipe the entire storage state diff because + // of aborted deletion. + if aborted { + incomplete[addrHash] = struct{}{} + delete(s.storagesOrigin, addrHash) + continue + } + if s.storagesOrigin[addrHash] == nil { + s.storagesOrigin[addrHash] = slots + } else { + // It can overwrite the data in s.storagesOrigin[addrHash] set by + // 'object.updateTrie'. + for key, val := range slots { + s.storagesOrigin[addrHash][key] = val + } + } + if err := nodes.Merge(set); err != nil { + return nil, err + } + } + return incomplete, nil +} + // Commit writes the state to the underlying in-memory trie database. // Once the state is committed, tries cached in stateDB (including account // trie, storage tries) will no longer be functional. A new state instance @@ -1008,38 +1158,39 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { nodes = trienode.NewMergedNodeSet() codeWriter = s.db.DiskDB().NewBatch() ) + // Handle all state deletions first + incomplete, err := s.handleDestruction(nodes) + if err != nil { + return common.Hash{}, err + } + // Handle all state updates afterwards for addr := range s.stateObjectsDirty { - if obj := s.stateObjects[addr]; !obj.deleted { - // Write any contract code associated with the state object - if obj.code != nil && obj.dirtyCode { - s.trie.UpdateContractCode(obj.Address(), common.BytesToHash(obj.CodeHash()), obj.code) - rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code) - obj.dirtyCode = false - } - // Write any storage changes in the state object to its storage trie - set, err := obj.commitTrie(s.db) - if err != nil { + obj := s.stateObjects[addr] + if obj.deleted { + continue + } + // Write any contract code associated with the state object + if obj.code != nil && obj.dirtyCode { + rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code) + s.trie.UpdateContractCode(obj.Address(), common.BytesToHash(obj.CodeHash()), obj.code) + obj.dirtyCode = false + } + // Write any storage changes in the state object to its storage trie + set, err := obj.commit(s.db) + if err != nil { + return common.Hash{}, err + } + // Merge the dirty nodes of storage trie into global set. It is possible + // that the account was destructed and then resurrected in the same block. + // In this case, the node set is shared by both accounts. + if set != nil { + if err := nodes.Merge(set); err != nil { return common.Hash{}, err } - // Merge the dirty nodes of storage trie into global set. - if set != nil { - if err := nodes.Merge(set); err != nil { - return common.Hash{}, err - } - updates, deleted := set.Size() - storageTrieNodesUpdated += updates - storageTrieNodesDeleted += deleted - } + updates, deleted := set.Size() + storageTrieNodesUpdated += updates + storageTrieNodesDeleted += deleted } - // If the contract is destructed, the storage is still left in the - // database as dangling data. Theoretically it's should be wiped from - // database as well, but in hash-based-scheme it's extremely hard to - // determine that if the trie nodes are also referenced by other storage, - // and in path-based-scheme some technical challenges are still unsolved. - // Although it won't affect the correctness but please fix it TODO(rjl493456442). - } - if len(s.stateObjectsDirty) > 0 { - s.stateObjectsDirty = make(map[common.Address]struct{}) } if codeWriter.ValueSize() > 0 { if err := codeWriter.Write(); err != nil { @@ -1081,7 +1232,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { start := time.Now() // Only update if there's a state transition (skip empty Clique blocks) if parent := s.snap.Root(); parent != root { - if err := s.snaps.Update(root, parent, s.convertAccountSet(s.stateObjectsDestruct), s.snapAccounts, s.snapStorage); err != nil { + if err := s.snaps.Update(root, parent, s.convertAccountSet(s.stateObjectsDestruct), s.accounts, s.storages); err != nil { log.Warn("Failed to update snapshot tree", "from", parent, "to", root, "err", err) } // Keep 128 diff layers in the memory, persistent layer is 129th. @@ -1095,10 +1246,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { if metrics.EnabledExpensive { s.SnapshotCommits += time.Since(start) } - s.snap, s.snapAccounts, s.snapStorage = nil, nil, nil - } - if len(s.stateObjectsDestruct) > 0 { - s.stateObjectsDestruct = make(map[common.Address]struct{}) + s.snap = nil } if root == (common.Hash{}) { root = types.EmptyRootHash @@ -1109,7 +1257,12 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { } if root != origin { start := time.Now() - if err := s.db.TrieDB().Update(root, origin, nodes); err != nil { + set := &triestate.Set{ + Accounts: s.accountsOrigin, + Storages: s.storagesOrigin, + Incomplete: incomplete, + } + if err := s.db.TrieDB().Update(root, origin, nodes, set); err != nil { return common.Hash{}, err } s.originalRoot = root @@ -1117,6 +1270,13 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { s.TrieDBCommits += time.Since(start) } } + // Clear all internal flags at the end of commit operation. + s.accounts = make(map[common.Hash][]byte) + s.storages = make(map[common.Hash]map[common.Hash][]byte) + s.accountsOrigin = make(map[common.Hash][]byte) + s.storagesOrigin = make(map[common.Hash]map[common.Hash][]byte) + s.stateObjectsDirty = make(map[common.Address]struct{}) + s.stateObjectsDestruct = make(map[common.Address]*types.StateAccount) return root, nil } @@ -1197,7 +1357,7 @@ func (s *StateDB) SlotInAccessList(addr common.Address, slot common.Hash) (addre } // convertAccountSet converts a provided account set from address keyed to hash keyed. -func (s *StateDB) convertAccountSet(set map[common.Address]struct{}) map[common.Hash]struct{} { +func (s *StateDB) convertAccountSet(set map[common.Address]*types.StateAccount) map[common.Hash]struct{} { ret := make(map[common.Hash]struct{}, len(set)) for addr := range set { obj, exist := s.stateObjects[addr] @@ -1209,3 +1369,24 @@ func (s *StateDB) convertAccountSet(set map[common.Address]struct{}) map[common. } return ret } + +// copyAccounts returns a deep-copied account set of the provided one. +func copyAccounts(set map[common.Hash][]byte) map[common.Hash][]byte { + copied := make(map[common.Hash][]byte, len(set)) + for key, val := range set { + copied[key] = common.CopyBytes(val) + } + return copied +} + +// copyStorages returns a deep-copied storage set of the provided one. +func copyStorages(set map[common.Hash]map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte { + copied := make(map[common.Hash]map[common.Hash][]byte, len(set)) + for addr, subset := range set { + copied[addr] = make(map[common.Hash][]byte, len(subset)) + for key, val := range subset { + copied[addr][key] = common.CopyBytes(val) + } + } + return copied +} diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go new file mode 100644 index 0000000000000..d014b2b3cca2b --- /dev/null +++ b/core/state/statedb_fuzz_test.go @@ -0,0 +1,373 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package state + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math" + "math/big" + "math/rand" + "reflect" + "strings" + "testing" + "testing/quick" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triestate" +) + +// A stateTest checks that the state changes are correctly captured. Instances +// of this test with pseudorandom content are created by Generate. +// +// The test works as follows: +// +// A list of states are created by applying actions. The state changes between +// each state instance are tracked and be verified. +type stateTest struct { + addrs []common.Address // all account addresses + actions [][]testAction // modifications to the state, grouped by block + chunk int // The number of actions per chunk + err error // failure details are reported through this field +} + +// newStateTestAction creates a random action that changes state. +func newStateTestAction(addr common.Address, r *rand.Rand, index int) testAction { + actions := []testAction{ + { + name: "SetBalance", + fn: func(a testAction, s *StateDB) { + s.SetBalance(addr, big.NewInt(a.args[0])) + }, + args: make([]int64, 1), + }, + { + name: "SetNonce", + fn: func(a testAction, s *StateDB) { + s.SetNonce(addr, uint64(a.args[0])) + }, + args: make([]int64, 1), + }, + { + name: "SetState", + fn: func(a testAction, s *StateDB) { + var key, val common.Hash + binary.BigEndian.PutUint16(key[:], uint16(a.args[0])) + binary.BigEndian.PutUint16(val[:], uint16(a.args[1])) + s.SetState(addr, key, val) + }, + args: make([]int64, 2), + }, + { + name: "SetCode", + fn: func(a testAction, s *StateDB) { + code := make([]byte, 16) + binary.BigEndian.PutUint64(code, uint64(a.args[0])) + binary.BigEndian.PutUint64(code[8:], uint64(a.args[1])) + s.SetCode(addr, code) + }, + args: make([]int64, 2), + }, + { + name: "CreateAccount", + fn: func(a testAction, s *StateDB) { + s.CreateAccount(addr) + }, + }, + { + name: "Suicide", + fn: func(a testAction, s *StateDB) { + s.Suicide(addr) + }, + }, + } + var nonRandom = index != -1 + if index == -1 { + index = r.Intn(len(actions)) + } + action := actions[index] + var names []string + if !action.noAddr { + names = append(names, addr.Hex()) + } + for i := range action.args { + if nonRandom { + action.args[i] = rand.Int63n(10000) + 1 // set balance to non-zero + } else { + action.args[i] = rand.Int63n(10000) + } + names = append(names, fmt.Sprint(action.args[i])) + } + action.name += " " + strings.Join(names, ", ") + return action +} + +// Generate returns a new snapshot test of the given size. All randomness is +// derived from r. +func (*stateTest) Generate(r *rand.Rand, size int) reflect.Value { + addrs := make([]common.Address, 5) + for i := range addrs { + addrs[i][0] = byte(i) + } + actions := make([][]testAction, rand.Intn(5)+1) + + for i := 0; i < len(actions); i++ { + actions[i] = make([]testAction, size) + for j := range actions[i] { + if j == 0 { + // Always include a set balance action to make sure + // the state changes are not empty. + actions[i][j] = newStateTestAction(common.HexToAddress("0xdeadbeef"), r, 0) + continue + } + actions[i][j] = newStateTestAction(addrs[r.Intn(len(addrs))], r, -1) + } + } + chunk := int(math.Sqrt(float64(size))) + if size > 0 && chunk == 0 { + chunk = 1 + } + return reflect.ValueOf(&stateTest{ + addrs: addrs, + actions: actions, + chunk: chunk, + }) +} + +func (test *stateTest) String() string { + out := new(bytes.Buffer) + for i, actions := range test.actions { + fmt.Fprintf(out, "---- block %d ----\n", i) + for j, action := range actions { + if j%test.chunk == 0 { + fmt.Fprintf(out, "---- transaction %d ----\n", j/test.chunk) + } + fmt.Fprintf(out, "%4d: %s\n", j%test.chunk, action.name) + } + } + return out.String() +} + +func (test *stateTest) run() bool { + var ( + roots []common.Hash + accountList []map[common.Hash][]byte + storageList []map[common.Hash]map[common.Hash][]byte + onCommit = func(states *triestate.Set) { + accountList = append(accountList, copyAccounts(states.Accounts)) + storageList = append(storageList, copyStorages(states.Storages)) + } + disk = rawdb.NewMemoryDatabase() + tdb = trie.NewDatabaseWithConfig(disk, &trie.Config{OnCommit: onCommit}) + sdb = NewDatabaseWithNodeDB(disk, tdb) + byzantium = rand.Intn(2) == 0 + ) + for i, actions := range test.actions { + root := types.EmptyRootHash + if i != 0 { + root = roots[len(roots)-1] + } + state, err := New(root, sdb, nil) + if err != nil { + panic(err) + } + for i, action := range actions { + if i%test.chunk == 0 && i != 0 { + if byzantium { + state.Finalise(true) // call finalise at the transaction boundary + } else { + state.IntermediateRoot(true) // call intermediateRoot at the transaction boundary + } + } + action.fn(action, state) + } + if byzantium { + state.Finalise(true) // call finalise at the transaction boundary + } else { + state.IntermediateRoot(true) // call intermediateRoot at the transaction boundary + } + nroot, err := state.Commit(true) // call commit at the block boundary + if err != nil { + panic(err) + } + if nroot == root { + return true // filter out non-change state transition + } + roots = append(roots, nroot) + } + for i := 0; i < len(test.actions); i++ { + root := types.EmptyRootHash + if i != 0 { + root = roots[i-1] + } + test.err = test.verify(root, roots[i], tdb, accountList[i], storageList[i]) + if test.err != nil { + return false + } + } + return true +} + +// verifyAccountCreation this function is called once the state diff says that +// specific account was not present. A serial of checks will be performed to +// ensure the state diff is correct, includes: +// +// - the account was indeed not present in trie +// - the account is present in new trie, nil->nil is regarded as invalid +// - the slots transition is correct +func (test *stateTest) verifyAccountCreation(next common.Hash, db *trie.Database, otr, ntr *trie.Trie, addrHash common.Hash, slots map[common.Hash][]byte) error { + // Verify account change + oBlob, err := otr.Get(addrHash.Bytes()) + if err != nil { + return err + } + nBlob, err := ntr.Get(addrHash.Bytes()) + if err != nil { + return err + } + if len(oBlob) != 0 { + return fmt.Errorf("unexpected account in old trie, %x", addrHash) + } + if len(nBlob) == 0 { + return fmt.Errorf("missing account in new trie, %x", addrHash) + } + + // Verify storage changes + var nAcct types.StateAccount + if err := rlp.DecodeBytes(nBlob, &nAcct); err != nil { + return err + } + // Account has no slot, empty slot set is expected + if nAcct.Root == types.EmptyRootHash { + if len(slots) != 0 { + return fmt.Errorf("unexpected slot changes %x", addrHash) + } + return nil + } + // Account has slots, ensure all new slots are contained + st, err := trie.New(trie.StorageTrieID(next, addrHash, nAcct.Root), db) + if err != nil { + return err + } + for key, val := range slots { + st.Update(key.Bytes(), val) + } + if st.Hash() != types.EmptyRootHash { + return errors.New("invalid slot changes") + } + return nil +} + +// verifyAccountUpdate this function is called once the state diff says that +// specific account was present. A serial of checks will be performed to +// ensure the state diff is correct, includes: +// +// - the account was indeed present in trie +// - the account in old trie matches the provided value +// - the slots transition is correct +func (test *stateTest) verifyAccountUpdate(next common.Hash, db *trie.Database, otr, ntr *trie.Trie, addrHash common.Hash, origin []byte, slots map[common.Hash][]byte) error { + // Verify account change + oBlob, err := otr.Get(addrHash.Bytes()) + if err != nil { + return err + } + nBlob, err := ntr.Get(addrHash.Bytes()) + if err != nil { + return err + } + if len(oBlob) == 0 { + return fmt.Errorf("missing account in old trie, %x", addrHash) + } + full, err := types.FullAccountRLP(origin) + if err != nil { + return err + } + if !bytes.Equal(full, oBlob) { + return fmt.Errorf("account value is not matched, %x", addrHash) + } + + // Decode accounts + var ( + oAcct types.StateAccount + nAcct types.StateAccount + nRoot common.Hash + ) + if err := rlp.DecodeBytes(oBlob, &oAcct); err != nil { + return err + } + if len(nBlob) == 0 { + nRoot = types.EmptyRootHash + } else { + if err := rlp.DecodeBytes(nBlob, &nAcct); err != nil { + return err + } + nRoot = nAcct.Root + } + + // Verify storage + st, err := trie.New(trie.StorageTrieID(next, addrHash, nRoot), db) + if err != nil { + return err + } + for key, val := range slots { + st.Update(key.Bytes(), val) + } + if st.Hash() != oAcct.Root { + return errors.New("invalid slot changes") + } + return nil +} + +func (test *stateTest) verify(root common.Hash, next common.Hash, db *trie.Database, accountsOrigin map[common.Hash][]byte, storagesOrigin map[common.Hash]map[common.Hash][]byte) error { + otr, err := trie.New(trie.StateTrieID(root), db) + if err != nil { + return err + } + ntr, err := trie.New(trie.StateTrieID(next), db) + if err != nil { + return err + } + for addrHash, account := range accountsOrigin { + var err error + if len(account) == 0 { + err = test.verifyAccountCreation(next, db, otr, ntr, addrHash, storagesOrigin[addrHash]) + } else { + err = test.verifyAccountUpdate(next, db, otr, ntr, addrHash, accountsOrigin[addrHash], storagesOrigin[addrHash]) + } + if err != nil { + return err + } + } + return nil +} + +func TestStateChanges(t *testing.T) { + config := &quick.Config{MaxCount: 1000} + err := quick.Check((*stateTest).run, config) + if cerr, ok := err.(*quick.CheckError); ok { + test := cerr.In[0].(*stateTest) + t.Errorf("%v:\n%s", test.err, test) + } else if err != nil { + t.Error(err) + } +} diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 44eace01d89f5..8290a154ff37b 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -485,7 +485,7 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error { } func TestTouchDelete(t *testing.T) { - s := newStateTest() + s := newStateEnv() s.state.GetOrNewStateObject(common.Address{}) root, _ := s.state.Commit(false) s.state, _ = New(root, s.state.db, s.state.snaps) diff --git a/core/types/state_account.go b/core/types/state_account.go index b721929e42cef..314f4943ecab3 100644 --- a/core/types/state_account.go +++ b/core/types/state_account.go @@ -35,6 +35,29 @@ type StateAccount struct { CodeHash []byte } +// NewEmptyStateAccount constructs an empty state account. +func NewEmptyStateAccount() *StateAccount { + return &StateAccount{ + Balance: new(big.Int), + Root: EmptyRootHash, + CodeHash: EmptyCodeHash.Bytes(), + } +} + +// Copy returns a deep-copied state account object. +func (acct *StateAccount) Copy() *StateAccount { + var balance *big.Int + if acct.Balance != nil { + balance = new(big.Int).Set(acct.Balance) + } + return &StateAccount{ + Nonce: acct.Nonce, + Balance: balance, + Root: acct.Root, + CodeHash: common.CopyBytes(acct.CodeHash), + } +} + // SlimAccount is a modified version of an Account, where the root is replaced // with a byte slice. This format can be used to represent full-consensus format // or slim format which replaces the empty root and code hash as nil byte slice. diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index c1875cc721af4..1c5f103cb58b1 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1387,7 +1387,7 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) { // Commit the state changes into db and re-create the trie // for accessing later. root, nodes, _ := accTrie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) accTrie, _ = trie.New(trie.StateTrieID(root), db) return db.Scheme(), accTrie, entries @@ -1449,7 +1449,7 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) { // Commit the state changes into db and re-create the trie // for accessing later. root, nodes, _ := accTrie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) accTrie, _ = trie.New(trie.StateTrieID(root), db) return db.Scheme(), accTrie, entries @@ -1498,7 +1498,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) nodes.Merge(set) // Commit gathered dirty nodes into database - db.Update(root, types.EmptyRootHash, nodes) + db.Update(root, types.EmptyRootHash, nodes, nil) // Re-create tries with new root accTrie, _ = trie.New(trie.StateTrieID(root), db) @@ -1563,7 +1563,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (strin nodes.Merge(set) // Commit gathered dirty nodes into database - db.Update(root, types.EmptyRootHash, nodes) + db.Update(root, types.EmptyRootHash, nodes, nil) // Re-create tries with new root accTrie, err := trie.New(trie.StateTrieID(root), db) diff --git a/light/postprocess.go b/light/postprocess.go index 7ef797ff47c17..8905694211456 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -220,7 +220,7 @@ func (c *ChtIndexerBackend) Commit() error { } // Commit trie changes into trie database in case it's not nil. if nodes != nil { - if err := c.triedb.Update(root, c.originRoot, trienode.NewWithNodeSet(nodes)); err != nil { + if err := c.triedb.Update(root, c.originRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { return err } if err := c.triedb.Commit(root, false); err != nil { @@ -473,7 +473,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { } // Commit trie changes into trie database in case it's not nil. if nodes != nil { - if err := b.triedb.Update(root, b.originRoot, trienode.NewWithNodeSet(nodes)); err != nil { + if err := b.triedb.Update(root, b.originRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { return err } if err := b.triedb.Commit(root, false); err != nil { diff --git a/light/trie.go b/light/trie.go index 43fccd5e361c7..4967cc74e5ba8 100644 --- a/light/trie.go +++ b/light/trie.go @@ -121,19 +121,22 @@ func (t *odrTrie) GetStorage(_ common.Address, key []byte) ([]byte, error) { } func (t *odrTrie) GetAccount(address common.Address) (*types.StateAccount, error) { - var res types.StateAccount - key := crypto.Keccak256(address.Bytes()) + var ( + enc []byte + key = crypto.Keccak256(address.Bytes()) + ) err := t.do(key, func() (err error) { - value, err := t.trie.Get(key) - if err != nil { - return err - } - if value == nil { - return nil - } - return rlp.DecodeBytes(value, &res) + enc, err = t.trie.Get(key) + return err }) - return &res, err + if err != nil || len(enc) == 0 { + return nil, err + } + acct := new(types.StateAccount) + if err := rlp.DecodeBytes(enc, acct); err != nil { + return nil, err + } + return acct, nil } func (t *odrTrie) UpdateAccount(address common.Address, acc *types.StateAccount) error { diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index a6b1b3bc7975e..deb0c92c277ef 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -176,7 +176,7 @@ func (f *fuzzer) fuzz() int { panic(err) } if nodes != nil { - dbA.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + dbA.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) } // Flush memdb -> disk (sponge) dbA.Commit(rootA, false) diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index 004fb64372c96..2f9b97390b631 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -170,7 +170,7 @@ func runRandTest(rt randTest) error { return err } if nodes != nil { - if err := triedb.Update(hash, origin, trienode.NewWithNodeSet(nodes)); err != nil { + if err := triedb.Update(hash, origin, trienode.NewWithNodeSet(nodes), nil); err != nil { return err } } diff --git a/trie/database.go b/trie/database.go index faaa1550e077d..85681d7597470 100644 --- a/trie/database.go +++ b/trie/database.go @@ -24,12 +24,16 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/trie/triestate" ) // Config defines all necessary options for database. type Config struct { Cache int // Memory allowance (MB) to use for caching trie nodes in memory Preimages bool // Flag whether the preimage of trie key is recorded + + // Testing hooks + OnCommit func(states *triestate.Set) // Hook invoked when commit is performed } // backend defines the methods needed to access/update trie nodes in different @@ -114,7 +118,10 @@ func (db *Database) Reader(blockRoot common.Hash) (Reader, error) { // given set in order to update state from the specified parent to the specified // root. The held pre-images accumulated up to this point will be flushed in case // the size exceeds the threshold. -func (db *Database) Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet) error { +func (db *Database) Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet, states *triestate.Set) error { + if db.config != nil && db.config.OnCommit != nil { + db.config.OnCommit(states) + } if db.preimages != nil { db.preimages.commit(false) } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 8f9849abbd12f..1ffb4bfbe8c04 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -63,7 +63,7 @@ func TestIterator(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) found := make(map[string]string) @@ -255,7 +255,7 @@ func TestDifferenceIterator(t *testing.T) { triea.MustUpdate([]byte(val.k), []byte(val.v)) } rootA, nodesA, _ := triea.Commit(false) - dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)) + dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) dbb := NewDatabase(rawdb.NewMemoryDatabase()) @@ -264,7 +264,7 @@ func TestDifferenceIterator(t *testing.T) { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } rootB, nodesB, _ := trieb.Commit(false) - dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB)) + dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB), nil) trieb, _ = New(TrieID(rootB), dbb) found := make(map[string]string) @@ -297,7 +297,7 @@ func TestUnionIterator(t *testing.T) { triea.MustUpdate([]byte(val.k), []byte(val.v)) } rootA, nodesA, _ := triea.Commit(false) - dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)) + dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) dbb := NewDatabase(rawdb.NewMemoryDatabase()) @@ -306,7 +306,7 @@ func TestUnionIterator(t *testing.T) { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } rootB, nodesB, _ := trieb.Commit(false) - dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB)) + dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB), nil) trieb, _ = New(TrieID(rootB), dbb) di, _ := NewUnionIterator([]NodeIterator{triea.MustNodeIterator(nil), trieb.MustNodeIterator(nil)}) @@ -368,7 +368,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { tr.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := tr.Commit(false) - tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) if !memonly { tdb.Commit(root, false) } @@ -484,7 +484,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool, scheme strin break } } - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) if !memonly { triedb.Commit(root, false) } @@ -605,7 +605,7 @@ func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { trie.MustUpdate(key, val) } root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) triedb.Commit(root, false) // Return the generated trie @@ -648,7 +648,7 @@ func testIteratorNodeBlob(t *testing.T, scheme string) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) triedb.Commit(root, false) var found = make(map[common.Hash][]byte) diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 4c70cbaa44b98..d756cb522394b 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -61,7 +61,7 @@ func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { } } root, nodes, _ := trie.Commit(false) - if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil { + if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } // Re-create the trie based on the new state diff --git a/trie/sync_test.go b/trie/sync_test.go index cfb7261a043aa..437cca7312d1d 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -57,7 +57,7 @@ func makeTestTrie(scheme string) (ethdb.Database, *Database, *StateTrie, map[str } } root, nodes, _ := trie.Commit(false) - if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil { + if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } if err := triedb.Commit(root, false); err != nil { @@ -740,7 +740,7 @@ func testSyncMovingTarget(t *testing.T, scheme string) { diff[string(key)] = val } root, nodes, _ := srcTrie.Commit(false) - if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil { + if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(err) } if err := srcDb.Commit(root, false); err != nil { @@ -765,7 +765,7 @@ func testSyncMovingTarget(t *testing.T, scheme string) { reverted[k] = val } root, nodes, _ = srcTrie.Commit(false) - if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil { + if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(err) } if err := srcDb.Commit(root, false); err != nil { diff --git a/trie/tracer_test.go b/trie/tracer_test.go index ca9d64766b169..a68cf3abaf1b6 100644 --- a/trie/tracer_test.go +++ b/trie/tracer_test.go @@ -71,7 +71,7 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { insertSet := copySet(trie.tracer.inserts) // copy before commit deleteSet := copySet(trie.tracer.deletes) // copy before commit root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) seen := setKeys(iterNodes(db, root)) if !compareSet(insertSet, seen) { @@ -137,7 +137,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -152,7 +152,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), randBytes(32)) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes)) + db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -170,7 +170,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate(key, randBytes(32)) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes)) + db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -185,7 +185,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(key), nil) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes)) + db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -200,7 +200,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), nil) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes)) + db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -219,7 +219,7 @@ func TestAccessListLeak(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) var cases = []struct { op func(tr *Trie) @@ -269,7 +269,7 @@ func TestTinyTree(t *testing.T) { trie.MustUpdate([]byte(val.k), randBytes(32)) } root, set, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(set)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(set), nil) parent := root trie, _ = New(TrieID(root), db) @@ -278,7 +278,7 @@ func TestTinyTree(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, set, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(set)) + db.Update(root, parent, trienode.NewWithNodeSet(set), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, set); err != nil { diff --git a/trie/trie_test.go b/trie/trie_test.go index 3997c4565dd9f..8e7066eadea32 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -89,7 +89,7 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) { updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) if !memonly { triedb.Commit(root, false) @@ -203,7 +203,7 @@ func TestGet(t *testing.T) { return } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) } } @@ -275,7 +275,7 @@ func TestReplication(t *testing.T) { updateString(trie, val.k, val.v) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) // create a new trie on top of the database and check that lookups work. trie2, err := New(TrieID(root), db) @@ -294,7 +294,7 @@ func TestReplication(t *testing.T) { // recreate the trie after commit if nodes != nil { - db.Update(hash, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(hash, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) } trie2, err = New(TrieID(hash), db) if err != nil { @@ -503,7 +503,7 @@ func runRandTest(rt randTest) bool { case opCommit: root, nodes, _ := tr.Commit(true) if nodes != nil { - triedb.Update(root, origin, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, origin, trienode.NewWithNodeSet(nodes), nil) } newtr, err := New(TrieID(root), triedb) if err != nil { @@ -838,7 +838,7 @@ func TestCommitSequence(t *testing.T) { } // Flush trie -> database root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) // Flush memdb -> disk (sponge) db.Commit(root, false) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { @@ -879,7 +879,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { } // Flush trie -> database root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) // Flush memdb -> disk (sponge) db.Commit(root, false) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { @@ -919,7 +919,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { // Flush trie -> database root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) db.Commit(root, false) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -967,7 +967,7 @@ func TestCommitSequenceSmallRoot(t *testing.T) { // Flush trie -> database root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) db.Commit(root, false) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -1139,7 +1139,7 @@ func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [] } h := trie.Hash() root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) b.StartTimer() triedb.Dereference(h) b.StopTimer() diff --git a/trie/trienode/node.go b/trie/trienode/node.go index b233f517f0402..d99f04bd5e5c7 100644 --- a/trie/trienode/node.go +++ b/trie/trienode/node.go @@ -123,6 +123,26 @@ func (set *NodeSet) AddNode(path []byte, n *WithPrev) { set.Nodes[string(path)] = n } +// Merge adds a set of nodes into the set. +func (set *NodeSet) Merge(owner common.Hash, nodes map[string]*WithPrev) error { + if set.Owner != owner { + return fmt.Errorf("nodesets belong to different owner are not mergeable %x-%x", set.Owner, owner) + } + for path, node := range nodes { + prev, ok := set.Nodes[path] + if ok { + // overwrite happens, revoke the counter + if prev.IsDeleted() { + set.deletes -= 1 + } else { + set.updates -= 1 + } + } + set.AddNode([]byte(path), node) + } + return nil +} + // AddLeaf adds the provided leaf node into set. TODO(rjl493456442) how can // we get rid of it? func (set *NodeSet) AddLeaf(parent common.Hash, blob []byte) { @@ -190,9 +210,9 @@ func NewWithNodeSet(set *NodeSet) *MergedNodeSet { // Merge merges the provided dirty nodes of a trie into the set. The assumption // is held that no duplicated set belonging to the same trie will be merged twice. func (set *MergedNodeSet) Merge(other *NodeSet) error { - _, present := set.Sets[other.Owner] + subset, present := set.Sets[other.Owner] if present { - return fmt.Errorf("duplicate trie for owner %#x", other.Owner) + return subset.Merge(other.Owner, other.Nodes) } set.Sets[other.Owner] = other return nil diff --git a/trie/triestate/state.go b/trie/triestate/state.go new file mode 100644 index 0000000000000..e5d0b87cb761a --- /dev/null +++ b/trie/triestate/state.go @@ -0,0 +1,28 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package triestate + +import "github.com/ethereum/go-ethereum/common" + +// Set represents a collection of mutated states during a state transition. +// The value refers to the original content of state before the transition +// is made. Nil means that the state was not present previously. +type Set struct { + Accounts map[common.Hash][]byte // Mutated account set, nil means the account was not present + Storages map[common.Hash]map[common.Hash][]byte // Mutated storage set, nil means the slot was not present + Incomplete map[common.Hash]struct{} // Indicator whether the storage slot is incomplete due to large deletion +}