Roll CoinStore, BlockStore and BlockHeightMap into ConsensusStore

benchmarks/block_ref.py

@@ -19,6 +19,7 @@
 from chia.consensus.get_block_generator import get_block_generator
 from chia.full_node.block_store import BlockStore
 from chia.full_node.coin_store import CoinStore
+from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3
 from chia.types.blockchain_format.serialized_program import SerializedProgram
 from chia.util.db_version import lookup_db_version
 from chia.util.db_wrapper import DBWrapper2
@@ -70,7 +71,8 @@ async def main(db_path: Path) -> None:
         # make configurable
         reserved_cores = 4
         height_map = await BlockHeightMap.create(db_path.parent, db_wrapper)
-        blockchain = await Blockchain.create(coin_store, block_store, height_map, DEFAULT_CONSTANTS, reserved_cores)
+        consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map)
+        blockchain = await Blockchain.create(consensus_store, DEFAULT_CONSTANTS, reserved_cores)
 
         peak = blockchain.get_peak()
         assert peak is not None

chia/_tests/blockchain/blockchain_test_utils.py

@@ -15,31 +15,20 @@
 
 
 async def check_block_store_invariant(bc: Blockchain):
-    db_wrapper = bc.block_store.db_wrapper
-
-    if db_wrapper.db_version == 1:
-        return
-
     in_chain = set()
     max_height = -1
-    async with bc.block_store.transaction() as conn:
-        async with conn.execute("SELECT height, in_main_chain FROM full_blocks") as cursor:
-            rows = await cursor.fetchall()
-            for row in rows:
-                height = row[0]
-
-                # if this block is in-chain, ensure we haven't found another block
-                # at this height that's also in chain. That would be an invariant
-                # violation
-                if row[1]:
-                    # make sure we don't have any duplicate heights. Each block
-                    # height can only have a single block with in_main_chain set
-                    assert height not in in_chain
-                    in_chain.add(height)
-                    max_height = max(max_height, height)
-
-            # make sure every height is represented in the set
-            assert len(in_chain) == max_height + 1
+    async for height in bc.consensus_store.get_block_heights_in_main_chain():
+        # if this block is in-chain, ensure we haven't found another block
+        # at this height that's also in chain. That would be an invariant
+        # violation
+        # make sure we don't have any duplicate heights. Each block
+        # height can only have a single block with in_main_chain set
+        assert height not in in_chain
+        in_chain.add(height)
+        max_height = max(max_height, height)
+
+    # make sure every height is represented in the set
+    assert len(in_chain) == max_height + 1
 
 
 async def _validate_and_add_block(

chia/_tests/blockchain/test_blockchain.py

@@ -2076,7 +2076,8 @@ async def test_timelock_conditions(
 
             if expected == AddBlockResult.NEW_PEAK:
                 # ensure coin was in fact spent
-                c = await b.coin_store.get_coin_record(coin.name())
+                recs = await b.consensus_store.get_coin_records([coin.name()])
+                c = recs[0] if len(recs) > 0 else None
                 assert c is not None and c.spent
 
     @pytest.mark.anyio
@@ -2286,10 +2287,12 @@ async def test_ephemeral_timelock(
 
             if expected == AddBlockResult.NEW_PEAK:
                 # ensure coin1 was in fact spent
-                c = await b.coin_store.get_coin_record(coin1.name())
+                recs1 = await b.consensus_store.get_coin_records([coin1.name()])
+                c = recs1[0] if len(recs1) > 0 else None
                 assert c is not None and c.spent
                 # ensure coin2 was NOT spent
-                c = await b.coin_store.get_coin_record(coin2.name())
+                recs2 = await b.consensus_store.get_coin_records([coin2.name()])
+                c = recs2[0] if len(recs2) > 0 else None
                 assert c is not None and not c.spent
 
     @pytest.mark.anyio
@@ -3103,9 +3106,11 @@ async def test_double_spent_in_reorg(self, empty_blockchain: Blockchain, bt: Blo
             )
 
         # ephemeral coin is spent
-        first_coin = await b.coin_store.get_coin_record(new_coin.name())
+        recs_first = await b.consensus_store.get_coin_records([new_coin.name()])
+        first_coin = recs_first[0] if len(recs_first) > 0 else None
         assert first_coin is not None and first_coin.spent
-        second_coin = await b.coin_store.get_coin_record(tx_2.additions()[0].name())
+        recs_second = await b.consensus_store.get_coin_records([tx_2.additions()[0].name()])
+        second_coin = recs_second[0] if len(recs_second) > 0 else None
         assert second_coin is not None and not second_coin.spent
 
         farmer_coin = create_farmer_coin(
@@ -3121,7 +3126,8 @@ async def test_double_spent_in_reorg(self, empty_blockchain: Blockchain, bt: Blo
         )
         await _validate_and_add_block(b, blocks_reorg[-1])
 
-        farmer_coin_record = await b.coin_store.get_coin_record(farmer_coin.name())
+        recs_farmer = await b.consensus_store.get_coin_records([farmer_coin.name()])
+        farmer_coin_record = recs_farmer[0] if len(recs_farmer) > 0 else None
         assert farmer_coin_record is not None and farmer_coin_record.spent
 
     @pytest.mark.anyio
@@ -3876,11 +3882,14 @@ async def test_chain_failed_rollback(empty_blockchain: Blockchain, bt: BlockTool
         await _validate_and_add_block(b, block, expected_result=AddBlockResult.ADDED_AS_ORPHAN, fork_info=fork_info)
 
     # Incorrectly set the height as spent in DB to trigger an error
-    print(f"{await b.coin_store.get_coin_record(spend_bundle.coin_spends[0].coin.name())}")
+    recs_dbg1 = await b.consensus_store.get_coin_records([spend_bundle.coin_spends[0].coin.name()])
+    print(f"{recs_dbg1[0] if len(recs_dbg1) > 0 else None}")
     print(spend_bundle.coin_spends[0].coin.name())
-    # await b.coin_store._set_spent([spend_bundle.coin_spends[0].coin.name()], 8)
-    await b.coin_store.rollback_to_block(2)
-    print(f"{await b.coin_store.get_coin_record(spend_bundle.coin_spends[0].coin.name())}")
+    # await b.consensus_store._set_spent([spend_bundle.coin_spends[0].coin.name()], 8)
+    async with b.consensus_store as writer:
+        await writer.rollback_to_block(2)
+    recs_dbg2 = await b.consensus_store.get_coin_records([spend_bundle.coin_spends[0].coin.name()])
+    print(f"{recs_dbg2[0] if len(recs_dbg2) > 0 else None}")
 
     fork_block = blocks_reorg_chain[10 - 1]
     # fork_info = ForkInfo(fork_block.height, fork_block.height, fork_block.header_hash)
@@ -4209,7 +4218,7 @@ async def get_fork_info(blockchain: Blockchain, block: FullBlock, peak: BlockRec
     counter = 0
     start = time.monotonic()
     for height in range(fork_info.fork_height + 1, block.height):
-        fork_block: Optional[FullBlock] = await blockchain.block_store.get_full_block(fork_chain[uint32(height)])
+        fork_block: Optional[FullBlock] = await blockchain.consensus_store.get_full_block(fork_chain[uint32(height)])
         assert fork_block is not None
         assert fork_block.height - 1 == fork_info.peak_height
         assert fork_block.height == 0 or fork_block.prev_header_hash == fork_info.peak_hash

chia/_tests/core/full_node/ram_db.py

@@ -11,6 +11,7 @@
 from chia.consensus.blockchain import Blockchain
 from chia.full_node.block_store import BlockStore
 from chia.full_node.coin_store import CoinStore
+from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3
 from chia.util.db_wrapper import DBWrapper2
 
 
@@ -23,7 +24,8 @@ async def create_ram_blockchain(
         block_store = await BlockStore.create(db_wrapper)
         coin_store = await CoinStore.create(db_wrapper)
         height_map = await BlockHeightMap.create(Path("."), db_wrapper)
-        blockchain = await Blockchain.create(coin_store, block_store, height_map, consensus_constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map)
+        blockchain = await Blockchain.create(consensus_store, consensus_constants, 2)
         try:
             yield db_wrapper, blockchain
         finally:

chia/_tests/core/full_node/stores/test_block_store.py

@@ -24,6 +24,7 @@
 from chia.consensus.full_block_to_block_record import header_block_to_sub_block_record
 from chia.full_node.block_store import BlockStore
 from chia.full_node.coin_store import CoinStore
+from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3
 from chia.full_node.full_block_utils import GeneratorBlockInfo
 from chia.simulator.block_tools import BlockTools
 from chia.simulator.wallet_tools import WalletTool
@@ -74,7 +75,8 @@ async def test_block_store(tmp_dir: Path, db_version: int, bt: BlockTools, use_c
         coin_store_2 = await CoinStore.create(db_wrapper_2)
         store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2)
-        bc = await Blockchain.create(coin_store_2, store_2, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
 
         store = await BlockStore.create(db_wrapper, use_cache=use_cache)
         await BlockStore.create(db_wrapper_2)
@@ -150,7 +152,8 @@ async def test_get_full_blocks_at(
         coin_store = await CoinStore.create(db_wrapper)
         block_store = await BlockStore.create(db_wrapper, use_cache=use_cache)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
-        bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
 
         count = 0
         fork_info = ForkInfo(-1, -1, bt.constants.GENESIS_CHALLENGE)
@@ -178,7 +181,8 @@ async def test_get_block_records_in_range(
         coin_store = await CoinStore.create(db_wrapper)
         block_store = await BlockStore.create(db_wrapper, use_cache=use_cache)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
-        bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
 
         count = 0
         fork_info = ForkInfo(-1, -1, bt.constants.GENESIS_CHALLENGE)
@@ -208,7 +212,8 @@ async def test_get_block_bytes_in_range_in_main_chain(
         coin_store = await CoinStore.create(db_wrapper)
         block_store = await BlockStore.create(db_wrapper, use_cache=use_cache)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
-        bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
         count = 0
         fork_info = ForkInfo(-1, -1, bt.constants.GENESIS_CHALLENGE)
         for b1, b2 in zip(blocks, alt_blocks):
@@ -237,7 +242,8 @@ async def test_deadlock(tmp_dir: Path, db_version: int, bt: BlockTools, use_cach
         coin_store_2 = await CoinStore.create(wrapper_2)
         store_2 = await BlockStore.create(wrapper_2)
         height_map = await BlockHeightMap.create(tmp_dir, wrapper_2)
-        bc = await Blockchain.create(coin_store_2, store_2, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
         block_records = []
         for block in blocks:
             await _validate_and_add_block(bc, block)
@@ -268,7 +274,8 @@ async def test_rollback(bt: BlockTools, tmp_dir: Path, use_cache: bool, default_
         coin_store = await CoinStore.create(db_wrapper)
         block_store = await BlockStore.create(db_wrapper, use_cache=use_cache)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
-        bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
 
         # insert all blocks
         count = 0
@@ -331,7 +338,8 @@ async def test_count_compactified_blocks(bt: BlockTools, tmp_dir: Path, db_versi
         coin_store = await CoinStore.create(db_wrapper)
         block_store = await BlockStore.create(db_wrapper, use_cache=use_cache)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
-        bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
 
         count = await block_store.count_compactified_blocks()
         assert count == 0
@@ -352,7 +360,8 @@ async def test_count_uncompactified_blocks(bt: BlockTools, tmp_dir: Path, db_ver
         coin_store = await CoinStore.create(db_wrapper)
         block_store = await BlockStore.create(db_wrapper, use_cache=use_cache)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
-        bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
 
         count = await block_store.count_uncompactified_blocks()
         assert count == 0
@@ -380,7 +389,8 @@ def rand_vdf_proof() -> VDFProof:
         coin_store = await CoinStore.create(db_wrapper)
         block_store = await BlockStore.create(db_wrapper, use_cache=use_cache)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
-        bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
         for block in blocks:
             await _validate_and_add_block(bc, block)
 
@@ -400,7 +410,7 @@ def rand_vdf_proof() -> VDFProof:
 
             # make sure we get the same result when we hit the database
             # itself (and not just the block cache)
-            block_store.rollback_cache_block(block.header_hash)
+            consensus_store.rollback_cache_block(block.header_hash)
             b = await block_store.get_full_block(block.header_hash)
             assert b is not None
             assert b.challenge_chain_ip_proof == proof
@@ -461,7 +471,8 @@ async def test_get_blocks_by_hash(tmp_dir: Path, bt: BlockTools, db_version: int
         coin_store_2 = await CoinStore.create(db_wrapper_2)
         store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2)
-        bc = await Blockchain.create(coin_store_2, store_2, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
 
         store = await BlockStore.create(db_wrapper, use_cache=use_cache)
         await BlockStore.create(db_wrapper_2)
@@ -501,7 +512,8 @@ async def test_get_block_bytes_in_range(tmp_dir: Path, bt: BlockTools, db_versio
         coin_store_2 = await CoinStore.create(db_wrapper_2)
         store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2)
-        bc = await Blockchain.create(coin_store_2, store_2, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
 
         await BlockStore.create(db_wrapper_2)
 
@@ -574,7 +586,8 @@ async def test_get_prev_hash(tmp_dir: Path, bt: BlockTools, db_version: int, use
         coin_store_2 = await CoinStore.create(db_wrapper_2)
         store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2)
-        bc = await Blockchain.create(coin_store_2, store_2, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map)
+        bc = await Blockchain.create(consensus_store, bt.constants, 2)
 
         store = await BlockStore.create(db_wrapper, use_cache=use_cache)
         await BlockStore.create(db_wrapper_2)

chia/_tests/core/full_node/stores/test_coin_store.py

@@ -22,6 +22,7 @@
 from chia.consensus.coinbase import create_farmer_coin, create_pool_coin
 from chia.full_node.block_store import BlockStore
 from chia.full_node.coin_store import CoinStore
+from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3
 from chia.full_node.hint_store import HintStore
 from chia.simulator.block_tools import BlockTools, test_constants
 from chia.simulator.wallet_tools import WalletTool
@@ -318,7 +319,8 @@ async def test_basic_reorg(tmp_dir: Path, db_version: int, bt: BlockTools) -> No
         coin_store = await CoinStore.create(db_wrapper)
         store = await BlockStore.create(db_wrapper)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
-        b: Blockchain = await Blockchain.create(coin_store, store, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(store, coin_store, height_map)
+        b: Blockchain = await Blockchain.create(consensus_store, bt.constants, 2)
         try:
             records: list[Optional[CoinRecord]] = []
 
@@ -385,7 +387,8 @@ async def test_get_puzzle_hash(tmp_dir: Path, db_version: int, bt: BlockTools) -
         coin_store = await CoinStore.create(db_wrapper)
         store = await BlockStore.create(db_wrapper)
         height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
-        b: Blockchain = await Blockchain.create(coin_store, store, height_map, bt.constants, 2)
+        consensus_store = await ConsensusStoreSQLite3.create(store, coin_store, height_map)
+        b: Blockchain = await Blockchain.create(consensus_store, bt.constants, 2)
         for block in blocks:
             await _validate_and_add_block(b, block)
         peak = b.get_peak()

chia/_tests/core/full_node/test_conditions.py

@@ -83,8 +83,8 @@ async def check_spend_bundle_validity(
 
         if expected_err is None:
             await _validate_and_add_block(blockchain, newest_block)
-            coins_added = await blockchain.coin_store.get_coins_added_at_height(uint32(len(blocks)))
-            coins_removed = await blockchain.coin_store.get_coins_removed_at_height(uint32(len(blocks)))
+            coins_added = await blockchain.consensus_store.get_coins_added_at_height(uint32(len(blocks)))
+            coins_removed = await blockchain.consensus_store.get_coins_removed_at_height(uint32(len(blocks)))
         else:
             await _validate_and_add_block(blockchain, newest_block, expected_error=expected_err)
             coins_added = []