Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

cmd, core: resolve scheme from a read-write database #28313

Merged
merged 4 commits into from
Oct 11, 2023
Merged

cmd, core: resolve scheme from a read-write database #28313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b42f162
cmd, core: resolve scheme from a read-write database
rjl493456442 Oct 11, 2023
e92c0bd
cmd, core, eth: move the scheme check in the ethereum constructor
rjl493456442 Oct 11, 2023
128d342
cmd/geth: dump should in ro mode
rjl493456442 Oct 11, 2023
fc37121
cmd: reverts
rjl493456442 Oct 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/geth/chaincmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -474,7 +474,7 @@ func dump(ctx *cli.Context) error {
if err != nil {
return err
}
triedb := utils.MakeTrieDatabase(ctx, db, true, false) // always enable preimage lookup
triedb := utils.MakeTrieDatabase(ctx, db, true, true) // always enable preimage lookup
defer triedb.Close()

state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil)
Expand Down
50 changes: 4 additions & 46 deletions cmd/utils/flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,6 @@ var (
StateSchemeFlag = &cli.StringFlag{
Name: "state.scheme",
Usage: "Scheme to use for storing ethereum state ('hash' or 'path')",
Value: rawdb.HashScheme,
Category: flags.StateCategory,
}
StateHistoryFlag = &cli.Uint64Flag{
Expand Down Expand Up @@ -1721,15 +1720,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
if ctx.IsSet(StateHistoryFlag.Name) {
cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name)
}
// Parse state scheme, abort the process if it's not compatible.
chaindb := tryMakeReadOnlyDatabase(ctx, stack)
scheme, err := ParseStateScheme(ctx, chaindb)
chaindb.Close()
if err != nil {
Fatalf("%v", err)
if ctx.IsSet(StateSchemeFlag.Name) {
cfg.StateScheme = ctx.String(StateSchemeFlag.Name)
}
cfg.StateScheme = scheme

// Parse transaction history flag, if user is still using legacy config
// file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'.
if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit {
Expand Down Expand Up @@ -2165,7 +2158,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
}
scheme, err := ParseStateScheme(ctx, chainDb)
scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), chainDb)
if err != nil {
Fatalf("%v", err)
}
Expand Down Expand Up @@ -2224,47 +2217,12 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
return preloads
}

// ParseStateScheme resolves scheme identifier from CLI flag. If the provided
// state scheme is not compatible with the one of persistent scheme, an error
// will be returned.
//
// - none: use the scheme consistent with persistent state, or fallback
// to hash-based scheme if state is empty.
// - hash: use hash-based scheme or error out if not compatible with
// persistent state scheme.
// - path: use path-based scheme or error out if not compatible with
// persistent state scheme.
func ParseStateScheme(ctx *cli.Context, disk ethdb.Database) (string, error) {
// If state scheme is not specified, use the scheme consistent
// with persistent state, or fallback to hash mode if database
// is empty.
stored := rawdb.ReadStateScheme(disk)
if !ctx.IsSet(StateSchemeFlag.Name) {
if stored == "" {
// use default scheme for empty database, flip it when
// path mode is chosen as default
log.Info("State schema set to default", "scheme", "hash")
return rawdb.HashScheme, nil
}
log.Info("State scheme set to already existing", "scheme", stored)
return stored, nil // reuse scheme of persistent scheme
}
// If state scheme is specified, ensure it's compatible with
// persistent state.
scheme := ctx.String(StateSchemeFlag.Name)
if stored == "" || scheme == stored {
log.Info("State scheme set by user", "scheme", scheme)
return scheme, nil
}
return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, scheme)
}

// MakeTrieDatabase constructs a trie database based on the configured scheme.
func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) *trie.Database {
config := &trie.Config{
Preimages: preimage,
}
scheme, err := ParseStateScheme(ctx, disk)
scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), disk)
if err != nil {
Fatalf("%v", err)
}
Expand Down
45 changes: 6 additions & 39 deletions core/genesis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error {
return nil
}

// deriveHash computes the state root according to the genesis specification.
func (ga *GenesisAlloc) deriveHash() (common.Hash, error) {
// hash computes the state root according to the genesis specification.
func (ga *GenesisAlloc) hash() (common.Hash, error) {
// Create an ephemeral in-memory database for computing hash,
// all the derived states will be discarded to not pollute disk.
db := state.NewDatabase(rawdb.NewMemoryDatabase())
Expand All @@ -142,9 +142,9 @@ func (ga *GenesisAlloc) deriveHash() (common.Hash, error) {
return statedb.Commit(0, false)
}

// flush is very similar with deriveHash, but the main difference is
// all the generated states will be persisted into the given database.
// Also, the genesis state specification will be flushed as well.
// flush is very similar with hash, but the main difference is all the generated
// states will be persisted into the given database. Also, the genesis state
// specification will be flushed as well.
func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash) error {
statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil)
if err != nil {
Expand Down Expand Up @@ -179,39 +179,6 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhas
return nil
}

// CommitGenesisState loads the stored genesis state with the given block
// hash and commits it into the provided trie database.
func CommitGenesisState(db ethdb.Database, triedb *trie.Database, blockhash common.Hash) error {
var alloc GenesisAlloc
blob := rawdb.ReadGenesisStateSpec(db, blockhash)
if len(blob) != 0 {
if err := alloc.UnmarshalJSON(blob); err != nil {
return err
}
} else {
// Genesis allocation is missing and there are several possibilities:
// the node is legacy which doesn't persist the genesis allocation or
// the persisted allocation is just lost.
// - supported networks(mainnet, testnets), recover with defined allocations
// - private network, can't recover
var genesis *Genesis
switch blockhash {
case params.MainnetGenesisHash:
genesis = DefaultGenesisBlock()
case params.GoerliGenesisHash:
genesis = DefaultGoerliGenesisBlock()
case params.SepoliaGenesisHash:
genesis = DefaultSepoliaGenesisBlock()
}
if genesis != nil {
alloc = genesis.Alloc
} else {
return errors.New("not found")
}
}
return alloc.flush(db, triedb, blockhash)
}

// GenesisAccount is an account in the state of the genesis block.
type GenesisAccount struct {
Code []byte `json:"code,omitempty"`
Expand Down Expand Up @@ -444,7 +411,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {

// ToBlock returns the genesis block according to genesis specification.
func (g *Genesis) ToBlock() *types.Block {
root, err := g.Alloc.deriveHash()
root, err := g.Alloc.hash()
if err != nil {
panic(err)
}
Expand Down
2 changes: 1 addition & 1 deletion core/genesis_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ func TestReadWriteGenesisAlloc(t *testing.T) {
{1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}},
{2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}},
}
hash, _ = alloc.deriveHash()
hash, _ = alloc.hash()
)
blob, _ := json.Marshal(alloc)
rawdb.WriteGenesisStateSpec(db, hash, blob)
Expand Down
35 changes: 35 additions & 0 deletions core/rawdb/accessors_trie.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,3 +305,38 @@ func ReadStateScheme(db ethdb.Reader) string {
}
return HashScheme
}

// ParseStateScheme checks if the specified state scheme is compatible with
// the stored state.
//
// - If the provided scheme is none, use the scheme consistent with persistent
// state, or fallback to hash-based scheme if state is empty.
//
// - If the provided scheme is hash, use hash-based scheme or error out if not
// compatible with persistent state scheme.
//
// - If the provided scheme is path: use path-based scheme or error out if not
// compatible with persistent state scheme.
func ParseStateScheme(provided string, disk ethdb.Database) (string, error) {
// If state scheme is not specified, use the scheme consistent
// with persistent state, or fallback to hash mode if database
// is empty.
stored := ReadStateScheme(disk)
if provided == "" {
if stored == "" {
// use default scheme for empty database, flip it when
// path mode is chosen as default
log.Info("State schema set to default", "scheme", "hash")
return HashScheme, nil
}
log.Info("State scheme set to already existing", "scheme", stored)
return stored, nil // reuse scheme of persistent scheme
}
// If state scheme is specified, ensure it's compatible with
// persistent state.
if stored == "" || provided == stored {
log.Info("State scheme set by user", "scheme", provided)
return provided, nil
}
return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, provided)
}
8 changes: 6 additions & 2 deletions eth/backend.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
if err != nil {
return nil, err
}
scheme, err := rawdb.ParseStateScheme(config.StateScheme, chainDb)
if err != nil {
return nil, err
}
// Try to recover offline state pruning only in hash-based.
if config.StateScheme == rawdb.HashScheme {
if scheme == rawdb.HashScheme {
if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb); err != nil {
log.Error("Failed to recover state", "error", err)
}
Expand Down Expand Up @@ -194,7 +198,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
SnapshotLimit: config.SnapshotCache,
Preimages: config.Preimages,
StateHistory: config.StateHistory,
StateScheme: config.StateScheme,
StateScheme: scheme,
}
)
// Override the chain config with provided settings.
Expand Down
8 changes: 5 additions & 3 deletions eth/ethconfig/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"github.com/ethereum/go-ethereum/consensus/clique"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/txpool/legacypool"
"github.com/ethereum/go-ethereum/eth/downloader"
Expand Down Expand Up @@ -64,7 +63,6 @@ var Defaults = Config{
TxLookupLimit: 2350000,
TransactionHistory: 2350000,
StateHistory: params.FullImmutabilityThreshold,
StateScheme: rawdb.HashScheme,
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note, it's necessary to remove the default value, otherwise hash mode is always picked if user doesn't specify via cli flag.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, we could leave it, if we do something like this

func ParseStateScheme(provided, default string, disk ethdb.Database) (string, error) {

and call it via

provided = ""
if ctx.IsSet(StateSchemeFlag.Name){ 
  provided = ctx.String(StateSchemeFlag.Name) 
}
scheme, err := rawdb.ParseStateScheme(provided, ctx.String(StateSchemeFlag.Name), disk)

I don't know if it's worth it though

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW, perhaps it's better to not have HashScheme set in configs unless teh user explicitly does so. Otherwise someone who doesn't care and just did a dumpconfig will forever be stuck on Hash since it will keep forcing it. Otherwise if they don't care and dumpconfig omits it, they would switch to path when the default flips too.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I mean when they resync. I can imagine people keeping configs around longer than their chain.

LightPeers: 100,
DatabaseCache: 512,
TrieCleanCache: 154,
Expand Down Expand Up @@ -105,7 +103,11 @@ type Config struct {
TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved.
StateScheme string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top

// State scheme represents the scheme used to store ethereum states and trie
// nodes on top. It can be 'hash', 'path', or none which means use the scheme
// consistent with persistent state.
StateScheme string `toml:",omitempty"`

// RequiredBlocks is a set of block number -> hash mappings which must be in the
// canonical chain of all remote peers. Setting the option makes geth verify the
Expand Down