From 9d2db926b9455df0555f674ec343db3011359932 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 10 Apr 2025 19:16:27 +0200 Subject: [PATCH 1/5] core/state: introduce the TransitionState object --- consensus/beacon/consensus.go | 16 ++-- core/blockchain.go | 7 +- core/chain_makers.go | 5 +- core/genesis.go | 55 ++++++----- core/genesis_test.go | 2 +- core/rawdb/accessors_overlay.go | 30 ++++++ core/rawdb/schema.go | 8 ++ core/state/database.go | 165 ++++++++++++++++++++++++++++++-- core/state/reader.go | 4 +- core/state/statedb.go | 3 +- core/verkle_witness_test.go | 6 +- params/config.go | 27 ------ 12 files changed, 248 insertions(+), 80 deletions(-) create mode 100644 core/rawdb/accessors_overlay.go diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index cc9f44e460a5..5c9434057907 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -354,9 +354,9 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types. // FinalizeAndAssemble implements consensus.Engine, setting the final state and // assembling the block. -func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { +func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, statedb *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { if !beacon.IsPoSHeader(header) { - return beacon.ethone.FinalizeAndAssemble(chain, header, state, body, receipts) + return beacon.ethone.FinalizeAndAssemble(chain, header, statedb, body, receipts) } shanghai := chain.Config().IsShanghai(header.Number, header.Time) if shanghai { @@ -370,10 +370,10 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea } } // Finalize and assemble the block. - beacon.Finalize(chain, header, state, body) + beacon.Finalize(chain, header, statedb, body) // Assign the final state root to header. - header.Root = state.IntermediateRoot(true) + header.Root = statedb.IntermediateRoot(true) // Assemble the final block. block := types.NewBlock(header, body, receipts, trie.NewStackTrie(nil)) @@ -381,19 +381,21 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea // Create the block witness and attach to block. // This step needs to happen as late as possible to catch all access events. if chain.Config().IsVerkle(header.Number, header.Time) { - keys := state.AccessEvents().Keys() + // TODO(gballet) move this to the end of the overlay conversion function in a subsequent PR + statedb.Database().SaveTransitionState(header.Root, &state.TransitionState{Ended: true}) + keys := statedb.AccessEvents().Keys() // Open the pre-tree to prove the pre-state against parent := chain.GetHeaderByNumber(header.Number.Uint64() - 1) if parent == nil { return nil, fmt.Errorf("nil parent header for block %d", header.Number) } - preTrie, err := state.Database().OpenTrie(parent.Root) + preTrie, err := statedb.Database().OpenTrie(parent.Root) if err != nil { return nil, fmt.Errorf("error opening pre-state tree root: %w", err) } vktPreTrie, okpre := preTrie.(*trie.VerkleTrie) - vktPostTrie, okpost := state.GetTrie().(*trie.VerkleTrie) + vktPostTrie, okpost := statedb.GetTrie().(*trie.VerkleTrie) // The witness is only attached iff both parent and current block are // using verkle tree. diff --git a/core/blockchain.go b/core/blockchain.go index d56996dadbee..c08b05e7f47f 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -283,12 +283,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis if cacheConfig == nil { cacheConfig = defaultCacheConfig } - // Open trie database with provided config - enableVerkle, err := EnableVerkleAtGenesis(db, genesis) - if err != nil { - return nil, err - } - triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(enableVerkle)) + triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(genesis.IsVerkle())) // Write the supplied genesis to the database if it has not been initialized // yet. The corresponding chain config will be returned, either from the diff --git a/core/chain_makers.go b/core/chain_makers.go index 7a258dc05f85..a3825137c6f9 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -536,8 +536,10 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine return block, b.receipts } + sdb := state.NewDatabase(trdb, nil) + for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), state.NewDatabase(trdb, nil)) + statedb, err := state.New(parent.Root(), sdb) if err != nil { panic(err) } @@ -575,6 +577,7 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (common.Hash, ethdb.Database, []*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { db := rawdb.NewMemoryDatabase() + saveVerkleTransitionStatusAtVerlkeGenesis(db) cacheConfig := DefaultCacheConfigWithScheme(rawdb.PathScheme) cacheConfig.SnapshotLimit = 0 triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) diff --git a/core/genesis.go b/core/genesis.go index 95782a827a47..ab80f3ededfb 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -18,6 +18,7 @@ package core import ( "bytes" + "encoding/gob" "encoding/json" "errors" "fmt" @@ -145,6 +146,9 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) { emptyRoot = types.EmptyVerkleHash } db := rawdb.NewMemoryDatabase() + if isVerkle { + saveVerkleTransitionStatusAtVerlkeGenesis(db) + } statedb, err := state.New(emptyRoot, state.NewDatabase(triedb.NewDatabase(db, config), nil)) if err != nil { return common.Hash{}, err @@ -276,6 +280,24 @@ func (o *ChainOverrides) apply(cfg *params.ChainConfig) error { return cfg.CheckConfigForkOrder() } +// saveVerkleTransitionStatusAtVerlkeGenesis saves a conversion marker +// representing a converted state, which is used in devnets that activate +// verkle at genesis. +func saveVerkleTransitionStatusAtVerlkeGenesis(db ethdb.Database) { + saveVerkleTransitionStatus(db, common.Hash{}, &state.TransitionState{Ended: true}) +} + +func saveVerkleTransitionStatus(db ethdb.Database, root common.Hash, ts *state.TransitionState) { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + err := enc.Encode(ts) + if err != nil { + log.Error("failed to encode transition state", "err", err) + return + } + rawdb.WriteVerkleTransitionState(db, root, buf.Bytes()) +} + // SetupGenesisBlock writes or updates the genesis block in db. // The block that will be used is: // @@ -299,6 +321,11 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g if genesis != nil && genesis.Config == nil { return nil, common.Hash{}, nil, errGenesisNoConfig } + // In case of verkle-at-genesis, we need to ensure that the conversion + // markers are indicating that the conversion has completed. + if genesis != nil && genesis.Config.VerkleTime != nil && *genesis.Config.VerkleTime == genesis.Timestamp { + saveVerkleTransitionStatusAtVerlkeGenesis(db) + } // Commit the genesis if the database is empty ghash := rawdb.ReadCanonicalHash(db, 0) if (ghash == common.Hash{}) { @@ -443,7 +470,7 @@ func (g *Genesis) chainConfigOrDefault(ghash common.Hash, stored *params.ChainCo // IsVerkle indicates whether the state is already stored in a verkle // tree at genesis time. func (g *Genesis) IsVerkle() bool { - return g.Config.IsVerkleGenesis() + return g.Config.VerkleTime != nil && *g.Config.VerkleTime == g.Timestamp } // ToBlock returns the genesis block according to genesis specification. @@ -547,6 +574,9 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo if err != nil { return nil, err } + if g.IsVerkle() { + saveVerkleTransitionStatus(db, block.Root(), &state.TransitionState{Ended: true}) + } batch := db.NewBatch() rawdb.WriteGenesisStateSpec(batch, block.Hash(), blob) rawdb.WriteBlock(batch, block) @@ -569,29 +599,6 @@ func (g *Genesis) MustCommit(db ethdb.Database, triedb *triedb.Database) *types. return block } -// EnableVerkleAtGenesis indicates whether the verkle fork should be activated -// at genesis. This is a temporary solution only for verkle devnet testing, where -// verkle fork is activated at genesis, and the configured activation date has -// already passed. -// -// In production networks (mainnet and public testnets), verkle activation always -// occurs after the genesis block, making this function irrelevant in those cases. -func EnableVerkleAtGenesis(db ethdb.Database, genesis *Genesis) (bool, error) { - if genesis != nil { - if genesis.Config == nil { - return false, errGenesisNoConfig - } - return genesis.Config.EnableVerkleAtGenesis, nil - } - if ghash := rawdb.ReadCanonicalHash(db, 0); ghash != (common.Hash{}) { - chainCfg := rawdb.ReadChainConfig(db, ghash) - if chainCfg != nil { - return chainCfg.EnableVerkleAtGenesis, nil - } - } - return false, nil -} - // DefaultGenesisBlock returns the Ethereum main net genesis block. func DefaultGenesisBlock() *Genesis { return &Genesis{ diff --git a/core/genesis_test.go b/core/genesis_test.go index 726bda86bb00..837ec7cdd83c 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -286,7 +286,6 @@ func TestVerkleGenesisCommit(t *testing.T) { OsakaTime: &verkleTime, VerkleTime: &verkleTime, TerminalTotalDifficulty: big.NewInt(0), - EnableVerkleAtGenesis: true, Ethash: nil, Clique: nil, BlobScheduleConfig: ¶ms.BlobScheduleConfig{ @@ -314,6 +313,7 @@ func TestVerkleGenesisCommit(t *testing.T) { } db := rawdb.NewMemoryDatabase() + saveVerkleTransitionStatusAtVerlkeGenesis(db) triedb := triedb.NewDatabase(db, triedb.VerkleDefaults) block := genesis.MustCommit(db, triedb) if !bytes.Equal(block.Root().Bytes(), expected) { diff --git a/core/rawdb/accessors_overlay.go b/core/rawdb/accessors_overlay.go new file mode 100644 index 000000000000..364cc889d1ec --- /dev/null +++ b/core/rawdb/accessors_overlay.go @@ -0,0 +1,30 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +func ReadVerkleTransitionState(db ethdb.KeyValueReader, hash common.Hash) ([]byte, error) { + return db.Get(transitionStateKey(hash)) +} + +func WriteVerkleTransitionState(db ethdb.KeyValueWriter, hash common.Hash, state []byte) error { + return db.Put(transitionStateKey(hash), state) +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index fa125cecc053..a4af6557d9ae 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -147,6 +147,9 @@ var ( preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) preimageHitsCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) preimageMissCounter = metrics.NewRegisteredCounter("db/preimage/miss", nil) + + // Verkle transition information + VerkleTransitionStatePrefix = []byte("verkle-transition-state-") ) // LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary @@ -362,3 +365,8 @@ func filterMapBlockLVKey(number uint64) []byte { binary.BigEndian.PutUint64(key[l:], number) return key } + +// transitionStateKey = transitionStatusKey + hash +func transitionStateKey(hash common.Hash) []byte { + return append(VerkleTransitionStatePrefix, hash.Bytes()...) +} diff --git a/core/state/database.go b/core/state/database.go index faf4954650bf..082dbd82575d 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -17,7 +17,10 @@ package state import ( + "bytes" + "encoding/gob" "fmt" + "reflect" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" @@ -26,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/utils" @@ -62,6 +66,12 @@ type Database interface { // Snapshot returns the underlying state snapshot. Snapshot() *snapshot.Tree + + // SaveTransitionState saves the tree transition progress markers to the database. + SaveTransitionState(common.Hash, *TransitionState) + + // LoadTransitionState loads the tree transition progress markers from the database. + LoadTransitionState(common.Hash) *TransitionState } // Trie is a Ethereum Merkle Patricia trie. @@ -141,6 +151,50 @@ type Trie interface { IsVerkle() bool } +// TransitionState is a structure that holds the progress markers of the +// translation process. +type TransitionState struct { + CurrentAccountAddress *common.Address // addresss of the last translated account + CurrentSlotHash common.Hash // hash of the last translated storage slot + CurrentPreimageOffset int64 // next byte to read from the preimage file + Started, Ended bool + + // Mark whether the storage for an account has been processed. This is useful if the + // maximum number of leaves of the conversion is reached before the whole storage is + // processed. + StorageProcessed bool + + BaseRoot common.Hash // hash of the last read-only MPT base tree +} + +// InTransition returns true if the translation process is in progress. +func (ts *TransitionState) InTransition() bool { + return ts != nil && ts.Started && !ts.Ended +} + +// Transitioned returns true if the translation process has been completed. +func (ts *TransitionState) Transitioned() bool { + return ts != nil && ts.Ended +} + +// Copy returns a deep copy of the TransitionState object. +func (ts *TransitionState) Copy() *TransitionState { + ret := &TransitionState{ + Started: ts.Started, + Ended: ts.Ended, + CurrentSlotHash: ts.CurrentSlotHash, + CurrentPreimageOffset: ts.CurrentPreimageOffset, + StorageProcessed: ts.StorageProcessed, + } + + if ts.CurrentAccountAddress != nil { + ret.CurrentAccountAddress = &common.Address{} + copy(ret.CurrentAccountAddress[:], ts.CurrentAccountAddress[:]) + } + + return ret +} + // CachingDB is an implementation of Database interface. It leverages both trie and // state snapshot to provide functionalities for state access. It's meant to be a // long-live object and has a few caches inside for sharing between blocks. @@ -151,17 +205,21 @@ type CachingDB struct { codeCache *lru.SizeConstrainedCache[common.Hash, []byte] codeSizeCache *lru.Cache[common.Hash, int] pointCache *utils.PointCache + + // Transition-specific fields + TransitionStatePerRoot lru.BasicLRU[common.Hash, *TransitionState] } // NewDatabase creates a state database with the provided data sources. func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { return &CachingDB{ - disk: triedb.Disk(), - triedb: triedb, - snap: snap, - codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), - pointCache: utils.NewPointCache(pointCacheSize), + disk: triedb.Disk(), + triedb: triedb, + snap: snap, + codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), + codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), + pointCache: utils.NewPointCache(pointCacheSize), + TransitionStatePerRoot: lru.NewBasicLRU[common.Hash, *TransitionState](1000), } } @@ -193,9 +251,10 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { readers = append(readers, newFlatReader(reader)) // state reader is optional } } + ts := db.LoadTransitionState(stateRoot) // Set up the trie reader, which is expected to always be available // as the gatekeeper unless the state is corrupted. - tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache) + tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache, ts) if err != nil { return nil, err } @@ -210,7 +269,11 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { // OpenTrie opens the main account trie at a specific root hash. func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { - if db.triedb.IsVerkle() { + ts := db.LoadTransitionState(root) + if ts.InTransition() { + panic("transition isn't supported yet") + } + if ts.Transitioned() { return trie.NewVerkleTrie(root, db.triedb, db.pointCache) } tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb) @@ -222,10 +285,11 @@ func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *CachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, self Trie) (Trie, error) { + ts := db.LoadTransitionState(stateRoot) // In the verkle case, there is only one tree. But the two-tree structure // is hardcoded in the codebase. So we need to return the same trie in this // case. - if db.triedb.IsVerkle() { + if ts.InTransition() || ts.Transitioned() { return self, nil } tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root), db.triedb) @@ -277,3 +341,86 @@ func mustCopyTrie(t Trie) Trie { panic(fmt.Errorf("unknown trie type %T", t)) } } + +// SaveTransitionState saves the transition state to the cache and commits +// it to the database if it's not already in the cache. +func (db *CachingDB) SaveTransitionState(root common.Hash, ts *TransitionState) { + if ts == nil { + panic("nil transition state") + } + + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + err := enc.Encode(ts) + if err != nil { + log.Error("failed to encode transition state", "err", err) + return + } + + if !db.TransitionStatePerRoot.Contains(root) { + // Copy so that the address pointer isn't updated after + // it has been saved. + db.TransitionStatePerRoot.Add(root, ts.Copy()) + rawdb.WriteVerkleTransitionState(db.TrieDB().Disk(), root, buf.Bytes()) + } else { + // Check that the state is consistent with what is in the cache, + // which is not strictly necessary but a good sanity check. Can + // be removed when the transition is stable. + cachedState, _ := db.TransitionStatePerRoot.Get(root) + if !reflect.DeepEqual(cachedState, ts) { + fmt.Println("transition state mismatch", "cached state", cachedState, "new state", ts) + panic("transition state mismatch") + } + } + + log.Debug("saving transition state", "storage processed", ts.StorageProcessed, "addr", ts.CurrentAccountAddress, "slot hash", ts.CurrentSlotHash, "root", root, "ended", ts.Ended, "started", ts.Started) +} + +func (db *CachingDB) LoadTransitionState(root common.Hash) *TransitionState { + // Try to get the transition state from the cache and + // the DB if it's not there. + ts, ok := db.TransitionStatePerRoot.Get(root) + if !ok { + // Not in the cache, try getting it from the DB + data, _ := rawdb.ReadVerkleTransitionState(db.TrieDB().Disk(), root) + // if err != nil && errors.Is(err, triedb.ErrNotFound) { + // log.Error("failed to read transition state", "err", err) + // return nil + // } + + // if a state could be read from the db, attempt to decode it + if len(data) > 0 { + var ( + newts TransitionState + buf = bytes.NewBuffer(data[:]) + dec = gob.NewDecoder(buf) + ) + // Decode transition state + err := dec.Decode(&newts) + if err != nil { + log.Error("failed to decode transition state", "err", err) + return nil + } + ts = &newts + } + + // Fallback that should only happen before the transition + if ts == nil { + // Initialize the first transition state, with the "ended" + // field set to true if the database was created + // as a verkle database. + log.Debug("no transition state found, starting fresh", "is verkle", db.triedb.IsVerkle()) + // Start with a fresh state + ts = &TransitionState{Ended: db.triedb.IsVerkle()} + } + + db.TransitionStatePerRoot.Add(root, ts) + } + + // Copy so that the CurrentAddress pointer in the map + // doesn't get overwritten. + // db.CurrentTransitionState = ts.Copy() + + log.Debug("loaded transition state", "storage processed", ts.StorageProcessed, "addr", ts.CurrentAccountAddress, "slot hash", ts.CurrentSlotHash, "root", root, "ended", ts.Ended, "started", ts.Started) + return ts +} diff --git a/core/state/reader.go b/core/state/reader.go index a0f15dfcc8b5..ae2bf0945d12 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -207,12 +207,12 @@ type trieReader struct { // trieReader constructs a trie reader of the specific state. An error will be // returned if the associated trie specified by root is not existent. -func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache) (*trieReader, error) { +func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache, ts *TransitionState) (*trieReader, error) { var ( tr Trie err error ) - if !db.IsVerkle() { + if !ts.Transitioned() && !ts.InTransition() { tr, err = trie.NewStateTrie(trie.StateTrieID(root), db) } else { tr, err = trie.NewVerkleTrie(root, db, cache) diff --git a/core/state/statedb.go b/core/state/statedb.go index e3f5b9e1a0a8..175f67155d35 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -181,7 +181,7 @@ func New(root common.Hash, db Database) (*StateDB, error) { accessList: newAccessList(), transientStorage: newTransientStorage(), } - if db.TrieDB().IsVerkle() { + if tr.IsVerkle() { sdb.accessEvents = NewAccessEvents(db.PointCache()) } return sdb, nil @@ -1275,6 +1275,7 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag return nil, err } } + s.db.SaveTransitionState(ret.root, &TransitionState{Ended: true}) if !ret.empty() { // If snapshotting is enabled, update the snapshot tree with this new version if snap := s.db.Snapshot(); snap != nil && snap.Snapshot(ret.originRoot) != nil { diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index de2280ced1c7..353c43101504 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -20,6 +20,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "fmt" "math/big" "slices" "testing" @@ -58,7 +59,6 @@ var ( ShanghaiTime: u64(0), VerkleTime: u64(0), TerminalTotalDifficulty: common.Big0, - EnableVerkleAtGenesis: true, BlobScheduleConfig: ¶ms.BlobScheduleConfig{ Verkle: params.DefaultPragueBlobConfig, }, @@ -82,7 +82,6 @@ var ( ShanghaiTime: u64(0), VerkleTime: u64(0), TerminalTotalDifficulty: common.Big0, - EnableVerkleAtGenesis: true, BlobScheduleConfig: ¶ms.BlobScheduleConfig{ Verkle: params.DefaultPragueBlobConfig, }, @@ -202,6 +201,9 @@ func TestProcessVerkle(t *testing.T) { t.Log("verified verkle proof, inserting blocks into the chain") + for i, b := range chain { + fmt.Printf("%d %x\n", i, b.Root()) + } endnum, err := blockchain.InsertChain(chain) if err != nil { t.Fatalf("block %d imported with error: %v", endnum, err) diff --git a/params/config.go b/params/config.go index 8f9e02583bfa..b7e839ff8aff 100644 --- a/params/config.go +++ b/params/config.go @@ -415,19 +415,6 @@ type ChainConfig struct { DepositContractAddress common.Address `json:"depositContractAddress,omitempty"` - // EnableVerkleAtGenesis is a flag that specifies whether the network uses - // the Verkle tree starting from the genesis block. If set to true, the - // genesis state will be committed using the Verkle tree, eliminating the - // need for any Verkle transition later. - // - // This is a temporary flag only for verkle devnet testing, where verkle is - // activated at genesis, and the configured activation date has already passed. - // - // In production networks (mainnet and public testnets), verkle activation - // always occurs after the genesis block, making this flag irrelevant in - // those cases. - EnableVerkleAtGenesis bool `json:"enableVerkleAtGenesis,omitempty"` - // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` Clique *CliqueConfig `json:"clique,omitempty"` @@ -651,20 +638,6 @@ func (c *ChainConfig) IsVerkle(num *big.Int, time uint64) bool { return c.IsLondon(num) && isTimestampForked(c.VerkleTime, time) } -// IsVerkleGenesis checks whether the verkle fork is activated at the genesis block. -// -// Verkle mode is considered enabled if the verkle fork time is configured, -// regardless of whether the local time has surpassed the fork activation time. -// This is a temporary workaround for verkle devnet testing, where verkle is -// activated at genesis, and the configured activation date has already passed. -// -// In production networks (mainnet and public testnets), verkle activation -// always occurs after the genesis block, making this function irrelevant in -// those cases. -func (c *ChainConfig) IsVerkleGenesis() bool { - return c.EnableVerkleAtGenesis -} - // IsEIP4762 returns whether eip 4762 has been activated at given block. func (c *ChainConfig) IsEIP4762(num *big.Int, time uint64) bool { return c.IsVerkle(num, time) From 7352993da0f937b757733885b3db644316934c0f Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:16:30 +0200 Subject: [PATCH 2/5] fix failing tests --- consensus/beacon/consensus.go | 2 +- core/genesis.go | 2 +- core/state/database.go | 6 ------ core/state/statedb.go | 1 - 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 5c9434057907..84fbabac58ca 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -382,7 +382,7 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea // This step needs to happen as late as possible to catch all access events. if chain.Config().IsVerkle(header.Number, header.Time) { // TODO(gballet) move this to the end of the overlay conversion function in a subsequent PR - statedb.Database().SaveTransitionState(header.Root, &state.TransitionState{Ended: true}) + statedb.Database().(*state.CachingDB).SaveTransitionState(header.Root, &state.TransitionState{Ended: true}) keys := statedb.AccessEvents().Keys() // Open the pre-tree to prove the pre-state against diff --git a/core/genesis.go b/core/genesis.go index ab80f3ededfb..da01a420c4ad 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -470,7 +470,7 @@ func (g *Genesis) chainConfigOrDefault(ghash common.Hash, stored *params.ChainCo // IsVerkle indicates whether the state is already stored in a verkle // tree at genesis time. func (g *Genesis) IsVerkle() bool { - return g.Config.VerkleTime != nil && *g.Config.VerkleTime == g.Timestamp + return g != nil && g.Config != nil && g.Config.VerkleTime != nil && *g.Config.VerkleTime == g.Timestamp } // ToBlock returns the genesis block according to genesis specification. diff --git a/core/state/database.go b/core/state/database.go index 082dbd82575d..b5f92df79c34 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -66,12 +66,6 @@ type Database interface { // Snapshot returns the underlying state snapshot. Snapshot() *snapshot.Tree - - // SaveTransitionState saves the tree transition progress markers to the database. - SaveTransitionState(common.Hash, *TransitionState) - - // LoadTransitionState loads the tree transition progress markers from the database. - LoadTransitionState(common.Hash) *TransitionState } // Trie is a Ethereum Merkle Patricia trie. diff --git a/core/state/statedb.go b/core/state/statedb.go index 175f67155d35..6d1ef45a7bde 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1275,7 +1275,6 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag return nil, err } } - s.db.SaveTransitionState(ret.root, &TransitionState{Ended: true}) if !ret.empty() { // If snapshotting is enabled, update the snapshot tree with this new version if snap := s.db.Snapshot(); snap != nil && snap.Snapshot(ret.originRoot) != nil { From ab660198656ea331deea95b020b6b562c1a1f49d Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 15 May 2025 11:58:41 +0200 Subject: [PATCH 3/5] review feedback (with gob still) --- consensus/beacon/consensus.go | 3 +- core/genesis.go | 7 ++-- core/overlay/state_transition.go | 63 ++++++++++++++++++++++++++++++++ core/state/database.go | 61 ++++--------------------------- core/state/reader.go | 3 +- 5 files changed, 78 insertions(+), 59 deletions(-) create mode 100644 core/overlay/state_transition.go diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 84fbabac58ca..cdf68d8e5f11 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" + "github.com/ethereum/go-ethereum/core/overlay" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" @@ -382,7 +383,7 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea // This step needs to happen as late as possible to catch all access events. if chain.Config().IsVerkle(header.Number, header.Time) { // TODO(gballet) move this to the end of the overlay conversion function in a subsequent PR - statedb.Database().(*state.CachingDB).SaveTransitionState(header.Root, &state.TransitionState{Ended: true}) + statedb.Database().(*state.CachingDB).SaveTransitionState(header.Root, &overlay.TransitionState{Ended: true}) keys := statedb.AccessEvents().Keys() // Open the pre-tree to prove the pre-state against diff --git a/core/genesis.go b/core/genesis.go index da01a420c4ad..c08decc0568f 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/overlay" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/tracing" @@ -284,10 +285,10 @@ func (o *ChainOverrides) apply(cfg *params.ChainConfig) error { // representing a converted state, which is used in devnets that activate // verkle at genesis. func saveVerkleTransitionStatusAtVerlkeGenesis(db ethdb.Database) { - saveVerkleTransitionStatus(db, common.Hash{}, &state.TransitionState{Ended: true}) + saveVerkleTransitionStatus(db, common.Hash{}, &overlay.TransitionState{Ended: true}) } -func saveVerkleTransitionStatus(db ethdb.Database, root common.Hash, ts *state.TransitionState) { +func saveVerkleTransitionStatus(db ethdb.Database, root common.Hash, ts *overlay.TransitionState) { var buf bytes.Buffer enc := gob.NewEncoder(&buf) err := enc.Encode(ts) @@ -575,7 +576,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo return nil, err } if g.IsVerkle() { - saveVerkleTransitionStatus(db, block.Root(), &state.TransitionState{Ended: true}) + saveVerkleTransitionStatus(db, block.Root(), &overlay.TransitionState{Ended: true}) } batch := db.NewBatch() rawdb.WriteGenesisStateSpec(batch, block.Hash(), blob) diff --git a/core/overlay/state_transition.go b/core/overlay/state_transition.go new file mode 100644 index 000000000000..b5487634a938 --- /dev/null +++ b/core/overlay/state_transition.go @@ -0,0 +1,63 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package overlay + +import "github.com/ethereum/go-ethereum/common" + +// TransitionState is a structure that holds the progress markers of the +// translation process. +type TransitionState struct { + CurrentAccountAddress *common.Address // addresss of the last translated account + CurrentSlotHash common.Hash // hash of the last translated storage slot + CurrentPreimageOffset int64 // next byte to read from the preimage file + Started, Ended bool + + // Mark whether the storage for an account has been processed. This is useful if the + // maximum number of leaves of the conversion is reached before the whole storage is + // processed. + StorageProcessed bool + + BaseRoot common.Hash // hash of the last read-only MPT base tree +} + +// InTransition returns true if the translation process is in progress. +func (ts *TransitionState) InTransition() bool { + return ts != nil && ts.Started && !ts.Ended +} + +// Transitioned returns true if the translation process has been completed. +func (ts *TransitionState) Transitioned() bool { + return ts != nil && ts.Ended +} + +// Copy returns a deep copy of the TransitionState object. +func (ts *TransitionState) Copy() *TransitionState { + ret := &TransitionState{ + Started: ts.Started, + Ended: ts.Ended, + CurrentSlotHash: ts.CurrentSlotHash, + CurrentPreimageOffset: ts.CurrentPreimageOffset, + StorageProcessed: ts.StorageProcessed, + } + + if ts.CurrentAccountAddress != nil { + ret.CurrentAccountAddress = &common.Address{} + copy(ret.CurrentAccountAddress[:], ts.CurrentAccountAddress[:]) + } + + return ret +} diff --git a/core/state/database.go b/core/state/database.go index b5f92df79c34..60ba46b27c42 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/overlay" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" @@ -145,50 +146,6 @@ type Trie interface { IsVerkle() bool } -// TransitionState is a structure that holds the progress markers of the -// translation process. -type TransitionState struct { - CurrentAccountAddress *common.Address // addresss of the last translated account - CurrentSlotHash common.Hash // hash of the last translated storage slot - CurrentPreimageOffset int64 // next byte to read from the preimage file - Started, Ended bool - - // Mark whether the storage for an account has been processed. This is useful if the - // maximum number of leaves of the conversion is reached before the whole storage is - // processed. - StorageProcessed bool - - BaseRoot common.Hash // hash of the last read-only MPT base tree -} - -// InTransition returns true if the translation process is in progress. -func (ts *TransitionState) InTransition() bool { - return ts != nil && ts.Started && !ts.Ended -} - -// Transitioned returns true if the translation process has been completed. -func (ts *TransitionState) Transitioned() bool { - return ts != nil && ts.Ended -} - -// Copy returns a deep copy of the TransitionState object. -func (ts *TransitionState) Copy() *TransitionState { - ret := &TransitionState{ - Started: ts.Started, - Ended: ts.Ended, - CurrentSlotHash: ts.CurrentSlotHash, - CurrentPreimageOffset: ts.CurrentPreimageOffset, - StorageProcessed: ts.StorageProcessed, - } - - if ts.CurrentAccountAddress != nil { - ret.CurrentAccountAddress = &common.Address{} - copy(ret.CurrentAccountAddress[:], ts.CurrentAccountAddress[:]) - } - - return ret -} - // CachingDB is an implementation of Database interface. It leverages both trie and // state snapshot to provide functionalities for state access. It's meant to be a // long-live object and has a few caches inside for sharing between blocks. @@ -201,7 +158,7 @@ type CachingDB struct { pointCache *utils.PointCache // Transition-specific fields - TransitionStatePerRoot lru.BasicLRU[common.Hash, *TransitionState] + TransitionStatePerRoot lru.BasicLRU[common.Hash, *overlay.TransitionState] } // NewDatabase creates a state database with the provided data sources. @@ -213,7 +170,7 @@ func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), pointCache: utils.NewPointCache(pointCacheSize), - TransitionStatePerRoot: lru.NewBasicLRU[common.Hash, *TransitionState](1000), + TransitionStatePerRoot: lru.NewBasicLRU[common.Hash, *overlay.TransitionState](1000), } } @@ -338,7 +295,7 @@ func mustCopyTrie(t Trie) Trie { // SaveTransitionState saves the transition state to the cache and commits // it to the database if it's not already in the cache. -func (db *CachingDB) SaveTransitionState(root common.Hash, ts *TransitionState) { +func (db *CachingDB) SaveTransitionState(root common.Hash, ts *overlay.TransitionState) { if ts == nil { panic("nil transition state") } @@ -370,22 +327,18 @@ func (db *CachingDB) SaveTransitionState(root common.Hash, ts *TransitionState) log.Debug("saving transition state", "storage processed", ts.StorageProcessed, "addr", ts.CurrentAccountAddress, "slot hash", ts.CurrentSlotHash, "root", root, "ended", ts.Ended, "started", ts.Started) } -func (db *CachingDB) LoadTransitionState(root common.Hash) *TransitionState { +func (db *CachingDB) LoadTransitionState(root common.Hash) *overlay.TransitionState { // Try to get the transition state from the cache and // the DB if it's not there. ts, ok := db.TransitionStatePerRoot.Get(root) if !ok { // Not in the cache, try getting it from the DB data, _ := rawdb.ReadVerkleTransitionState(db.TrieDB().Disk(), root) - // if err != nil && errors.Is(err, triedb.ErrNotFound) { - // log.Error("failed to read transition state", "err", err) - // return nil - // } // if a state could be read from the db, attempt to decode it if len(data) > 0 { var ( - newts TransitionState + newts overlay.TransitionState buf = bytes.NewBuffer(data[:]) dec = gob.NewDecoder(buf) ) @@ -405,7 +358,7 @@ func (db *CachingDB) LoadTransitionState(root common.Hash) *TransitionState { // as a verkle database. log.Debug("no transition state found, starting fresh", "is verkle", db.triedb.IsVerkle()) // Start with a fresh state - ts = &TransitionState{Ended: db.triedb.IsVerkle()} + ts = &overlay.TransitionState{Ended: db.triedb.IsVerkle()} } db.TransitionStatePerRoot.Add(root, ts) diff --git a/core/state/reader.go b/core/state/reader.go index ae2bf0945d12..fd19464d915c 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/overlay" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -207,7 +208,7 @@ type trieReader struct { // trieReader constructs a trie reader of the specific state. An error will be // returned if the associated trie specified by root is not existent. -func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache, ts *TransitionState) (*trieReader, error) { +func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache, ts *overlay.TransitionState) (*trieReader, error) { var ( tr Trie err error From 5adf3da2afc329d50489d45eb4cdabe0fdc324b0 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 15 May 2025 12:16:36 +0200 Subject: [PATCH 4/5] review feedback: use an uint64 to prepare for RLP --- core/overlay/state_transition.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/overlay/state_transition.go b/core/overlay/state_transition.go index b5487634a938..1156c8092910 100644 --- a/core/overlay/state_transition.go +++ b/core/overlay/state_transition.go @@ -23,7 +23,7 @@ import "github.com/ethereum/go-ethereum/common" type TransitionState struct { CurrentAccountAddress *common.Address // addresss of the last translated account CurrentSlotHash common.Hash // hash of the last translated storage slot - CurrentPreimageOffset int64 // next byte to read from the preimage file + CurrentPreimageOffset uint64 // next byte to read from the preimage file Started, Ended bool // Mark whether the storage for an account has been processed. This is useful if the From c38f2729d9ee0b6022cab0140946d25f5173be08 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 15 May 2025 14:15:09 +0200 Subject: [PATCH 5/5] replace gob with rlp --- core/genesis.go | 7 ++----- core/overlay/state_transition.go | 2 +- core/state/database.go | 18 ++++++------------ 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/core/genesis.go b/core/genesis.go index c08decc0568f..e0359bdb136c 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -18,7 +18,6 @@ package core import ( "bytes" - "encoding/gob" "encoding/json" "errors" "fmt" @@ -289,14 +288,12 @@ func saveVerkleTransitionStatusAtVerlkeGenesis(db ethdb.Database) { } func saveVerkleTransitionStatus(db ethdb.Database, root common.Hash, ts *overlay.TransitionState) { - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - err := enc.Encode(ts) + enc, err := rlp.EncodeToBytes(ts) if err != nil { log.Error("failed to encode transition state", "err", err) return } - rawdb.WriteVerkleTransitionState(db, root, buf.Bytes()) + rawdb.WriteVerkleTransitionState(db, root, enc) } // SetupGenesisBlock writes or updates the genesis block in db. diff --git a/core/overlay/state_transition.go b/core/overlay/state_transition.go index 1156c8092910..415904bce40e 100644 --- a/core/overlay/state_transition.go +++ b/core/overlay/state_transition.go @@ -21,7 +21,7 @@ import "github.com/ethereum/go-ethereum/common" // TransitionState is a structure that holds the progress markers of the // translation process. type TransitionState struct { - CurrentAccountAddress *common.Address // addresss of the last translated account + CurrentAccountAddress *common.Address `rlp:"nil"` // addresss of the last translated account CurrentSlotHash common.Hash // hash of the last translated storage slot CurrentPreimageOffset uint64 // next byte to read from the preimage file Started, Ended bool diff --git a/core/state/database.go b/core/state/database.go index 60ba46b27c42..bb9b40077b61 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -17,8 +17,6 @@ package state import ( - "bytes" - "encoding/gob" "fmt" "reflect" @@ -31,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/utils" @@ -300,9 +299,7 @@ func (db *CachingDB) SaveTransitionState(root common.Hash, ts *overlay.Transitio panic("nil transition state") } - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - err := enc.Encode(ts) + enc, err := rlp.EncodeToBytes(ts) if err != nil { log.Error("failed to encode transition state", "err", err) return @@ -312,7 +309,7 @@ func (db *CachingDB) SaveTransitionState(root common.Hash, ts *overlay.Transitio // Copy so that the address pointer isn't updated after // it has been saved. db.TransitionStatePerRoot.Add(root, ts.Copy()) - rawdb.WriteVerkleTransitionState(db.TrieDB().Disk(), root, buf.Bytes()) + rawdb.WriteVerkleTransitionState(db.TrieDB().Disk(), root, enc) } else { // Check that the state is consistent with what is in the cache, // which is not strictly necessary but a good sanity check. Can @@ -337,13 +334,10 @@ func (db *CachingDB) LoadTransitionState(root common.Hash) *overlay.TransitionSt // if a state could be read from the db, attempt to decode it if len(data) > 0 { - var ( - newts overlay.TransitionState - buf = bytes.NewBuffer(data[:]) - dec = gob.NewDecoder(buf) - ) + var newts overlay.TransitionState + // Decode transition state - err := dec.Decode(&newts) + err := rlp.DecodeBytes(data, &newts) if err != nil { log.Error("failed to decode transition state", "err", err) return nil