Blobstoredump

concordium-consensus/package.yaml

@@ -283,6 +283,34 @@ executables:
     - optparse-applicative >= 0.15
     - filepath >= 1.4
 
+  block-state-dump:
+    main:         Main.hs
+    source-dirs:  tools/block-state-dump
+    ghc-options:
+    - -threaded
+    - -rtsopts
+    - -with-rtsopts=-N
+    - -Wall
+    - -Wcompat
+
+    when:
+      - condition: os(windows)
+        then:
+          ghc-options: -static
+        else:
+          when:
+            - condition: flag(dynamic)
+              then:
+                ghc-options: -dynamic
+              else:
+                ghc-options: -static
+    dependencies:
+    - concordium-consensus
+    - optparse-applicative >= 0.15
+    - filepath >= 1.4  
+    - directory >= 1.3
+    - pretty-simple >= 4.1
+
 tests:
   consensus:
     main:                Spec.hs

concordium-consensus/src/Concordium/GlobalState.hs

@@ -20,7 +20,13 @@ import Concordium.GlobalState.BlockState
 import Concordium.GlobalState.Parameters
 import Concordium.GlobalState.Persistent.Account (newAccountCache)
 import Concordium.GlobalState.Persistent.BlobStore (BlobStoreT (runBlobStoreT), closeBlobStore, createBlobStore, destroyBlobStore, loadBlobStore)
-import Concordium.GlobalState.Persistent.BlockState
+import Concordium.GlobalState.Persistent.BlockState (
+    HashedPersistentBlockState (hpbsPointers),
+    PersistentBlockStateContext (..),
+    PersistentBlockStateMonad (runPersistentBlockStateMonad),
+    hashBlockState,
+    migratePersistentBlockState,
+ )
 import qualified Concordium.GlobalState.Persistent.BlockState.Modules as Modules
 import Concordium.GlobalState.Persistent.Genesis
 import Concordium.GlobalState.Persistent.TreeState

concordium-consensus/src/Concordium/GlobalState/ContractStateV1.hs

@@ -29,6 +29,7 @@ module Concordium.GlobalState.ContractStateV1 (
     -- * Testing
     lookupKey,
     generatePersistentTree,
+    dumpPersistentState,
 )
 where
 
@@ -49,6 +50,7 @@ import qualified Data.FixedByteString as FBS
 
 import Concordium.GlobalState.ContractStateFFIHelpers (LoadCallback, StoreCallback, errorLoadCallback)
 import Concordium.GlobalState.Persistent.BlobStore
+import Foreign.C.String
 
 -- | Opaque pointer to the mutable state. This state exists only for the duration
 --  of a transaction and is then deallocated by running a finalizer.
@@ -453,3 +455,52 @@ toByteString ps = do
         bytePtr <- serializePersistentState loadCallback psPtr sizePtr
         len <- peek sizePtr
         BSU.unsafePackCStringFinalizer (castPtr bytePtr) (fromIntegral len) (rs_free_array_len bytePtr (fromIntegral len))
+
+-- | Dump PLT block state. Use for debugging purposes only.
+dumpPersistentState ::
+    (MonadBlobStore m) =>
+    -- | Current block state
+    PersistentState ->
+    -- | Parent node
+    Word64 ->
+    -- | Path to graph file
+    String ->
+    -- | Path to state file
+    String ->
+    m ()
+dumpPersistentState persistentState parentNode graphFilePath dataFilePath =
+    do
+        loadCallbackPtr <- fst <$> getCallbacks
+        liftIO $ do
+            withPersistentState persistentState $ \persistentStatePtr ->
+                withCStringLen graphFilePath $ \(graphFilePathPtr, graphFilePathLen) ->
+                    withCStringLen dataFilePath $ \(dataFilePathPtr, dataFilePathLen) ->
+                        ffiDumpPersistentState
+                            loadCallbackPtr
+                            persistentStatePtr
+                            parentNode
+                            (castPtr graphFilePathPtr)
+                            (fromIntegral graphFilePathLen)
+                            (castPtr dataFilePathPtr)
+                            (fromIntegral dataFilePathLen)
+
+-- | Dump PLT block state. Use for debugging purposes only.
+--
+-- See the exported function in the Rust code for documentation of safety.
+foreign import ccall "ffi_dump_persistent_state"
+    ffiDumpPersistentState ::
+        -- | Called to read data from blob store.
+        LoadCallback ->
+        -- | Pointer to the block state to dump.
+        Ptr PersistentState ->
+        -- | Parent node
+        Word64 ->
+        -- | Path to graph file
+        Ptr Word8 ->
+        -- | Length of path to graph file
+        CSize ->
+        -- | Path to state file
+        Ptr Word8 ->
+        -- | Length of path to state file
+        CSize ->
+        IO ()

concordium-consensus/src/Concordium/GlobalState/Persistent/BlobStore.hs

@@ -75,7 +75,7 @@ module Concordium.GlobalState.Persistent.BlobStore (
     migrateReference,
 
     -- ** 'BufferedRef'
-    BufferedRef,
+    BufferedRef (..),
     makeBufferedRef,
     blobRefToBufferedRef,
     loadBufferedRef,
@@ -84,7 +84,7 @@ module Concordium.GlobalState.Persistent.BlobStore (
     uncacheBufferedRef,
 
     -- ** 'EagerBufferedRef'
-    EagerBufferedRef,
+    EagerBufferedRef (..),
     eagerBufferedDeref,
     eagerBufferedRefFromBufferedRef,
     migrateEagerBufferedRef,
@@ -93,7 +93,7 @@ module Concordium.GlobalState.Persistent.BlobStore (
     LazyBufferedRef,
 
     -- ** 'HashedBufferedRef'
-    HashedBufferedRef',
+    HashedBufferedRef' (..),
     HashedBufferedRef,
     bufferHashed,
     makeHashedBufferedRef,
@@ -102,12 +102,12 @@ module Concordium.GlobalState.Persistent.BlobStore (
     HashedBufferedRefO,
 
     -- ** 'EagerlyHashedBufferedRef'
-    EagerlyHashedBufferedRef',
+    EagerlyHashedBufferedRef' (..),
     EagerlyHashedBufferedRef,
     migrateEagerlyHashedBufferedRefKeepHash,
 
     -- ** 'UnbufferedRef'
-    UnbufferedRef,
+    UnbufferedRef (..),
     makeUnbufferedRef,
     makeFlushedUnbufferedRef,
     blobRefToUnbufferedRef,
@@ -1686,7 +1686,7 @@ makeHashedBufferedRef val = do
     return $ HashedBufferedRef br hashRef
 
 instance (DirectBlobStorable m a, MHashableTo m h a) => MHashableTo m h (HashedBufferedRef' h a) where
-    getHashM HashedBufferedRef{..} =
+    getHashM HashedBufferedRef{..} = do
         liftIO (readIORef bufferedHash) >>= \case
             Null -> do
                 !h <- getHashM bufferedReference

concordium-consensus/src/Concordium/GlobalState/Persistent/BlockState/ProtocolLevelTokens/RustPLTBlockState.hs

@@ -11,6 +11,7 @@ module Concordium.GlobalState.Persistent.BlockState.ProtocolLevelTokens.RustPLTB
     withPLTBlockState,
     migrate,
     ProtocolLevelTokensHash (..),
+    dumpPLTBlockState,
 ) where
 
 import qualified Data.Serialize as S
@@ -22,6 +23,7 @@ import qualified Concordium.GlobalState.Persistent.BlobStore as BlobStore
 import qualified Concordium.Types.HashableTo as Hashable
 import Control.Monad.Trans (lift, liftIO)
 import qualified Data.FixedByteString as FixedByteString
+import qualified Foreign.C as FFI
 
 -- | Opaque type representing a Rust maintained PLT state.
 -- The value is allocated in Rust and must be deallocated in Rust.
@@ -167,3 +169,52 @@ foreign import ccall "ffi_migrate_plt_block_state"
         FFI.Ptr RustPLTBlockState ->
         -- | Pointer to the new block state.
         IO (FFI.Ptr RustPLTBlockState)
+
+-- | Dump PLT block state. Use for debugging purposes only.
+dumpPLTBlockState ::
+    (BlobStore.MonadBlobStore m) =>
+    -- | Current block state
+    ForeignPLTBlockStatePtr ->
+    -- | Parent node
+    FFI.Word64 ->
+    -- | Path to graph file
+    String ->
+    -- | Path to state file
+    String ->
+    m ()
+dumpPLTBlockState blockState parentNode graphFilePath dataFilePath =
+    do
+        loadCallbackPtr <- fst <$> BlobStore.getCallbacks
+        liftIO $ do
+            withPLTBlockState blockState $ \blockStatePtr ->
+                FFI.withCStringLen graphFilePath $ \(graphFilePathPtr, graphFilePathLen) ->
+                    FFI.withCStringLen dataFilePath $ \(dataFilePathPtr, dataFilePathLen) ->
+                        ffiDumpPLTBlockState
+                            loadCallbackPtr
+                            blockStatePtr
+                            parentNode
+                            (FFI.castPtr graphFilePathPtr)
+                            (fromIntegral graphFilePathLen)
+                            (FFI.castPtr dataFilePathPtr)
+                            (fromIntegral dataFilePathLen)
+
+-- | Dump PLT block state. Use for debugging purposes only.
+--
+-- See the exported function in the Rust code for documentation of safety.
+foreign import ccall "ffi_dump_plt_block_state"
+    ffiDumpPLTBlockState ::
+        -- | Called to read data from blob store.
+        FFI.LoadCallback ->
+        -- | Pointer to the block state to dump.
+        FFI.Ptr RustPLTBlockState ->
+        -- | Parent node
+        FFI.Word64 ->
+        -- | Path to graph file
+        FFI.Ptr FFI.Word8 ->
+        -- | Length of path to graph file
+        FFI.CSize ->
+        -- | Path to state file
+        FFI.Ptr FFI.Word8 ->
+        -- | Length of path to state file
+        FFI.CSize ->
+        IO ()

concordium-consensus/src/Concordium/GlobalState/Persistent/LFMBTree.hs

@@ -21,8 +21,9 @@ module Concordium.GlobalState.Persistent.LFMBTree (
 
     -- * Tree type
     LFMBTree,
-    LFMBTree',
+    LFMBTree' (..),
     size,
+    T (..),
 
     -- * Construction
     empty,

concordium-consensus/src/Concordium/GlobalState/Persistent/Trie.hs

@@ -166,7 +166,7 @@ pairBranch (k1, v1) (k2, v2)
 -- | Trie with keys all of same fixed length treated as lists of bytes.
 --  The first parameter of 'TrieF' is the type of keys, which should
 --  by an instance of 'FixedTrieKey'.
---  The second parameter is the type of values.
+--  The second paraDmeter is the type of values.
 --  The third parameter is the recursive type argument: a type for Tries
 --  is obtained by applying a fixed-point combinator to @TrieF k v@.
 data TrieF k v r

concordium-consensus/src/Concordium/KonsensusV1/TreeState/LowLevel.hs

@@ -30,6 +30,7 @@ data StoredBlock (pv :: ProtocolVersion) = StoredBlock
       -- | Pointer to the state in the block state storage.
       stbStatePointer :: !(BlockStateRef pv)
     }
+    deriving (Show)
 
 type instance BlockProtocolVersion (StoredBlock pv) = pv
 

concordium-consensus/tools/block-state-dump/BlockStateDump/Config.hs

@@ -0,0 +1,64 @@
+module BlockStateDump.Config where
+
+import Data.Functor.Identity
+import Options.Applicative
+
+import Concordium.Types
+
+data Config
+    = DumpState
+    { cTreeStateDbPath :: FilePath,
+      cAccountMapDbPath :: FilePath,
+      cBlockStatePath :: FilePath,
+      cOutDir :: FilePath,
+      cProtocolVersion :: ProtocolVersion,
+      cBlockHeights :: [BlockHeight]
+    }
+
+config :: Parser (Identity Config)
+config =
+    Identity
+        <$> ( hsubparser
+                ( metavar "command"
+                    <> ( command
+                            "state"
+                            ( info
+                                ( DumpState
+                                    <$> strOption
+                                        ( long "tree-state-db-path"
+                                            <> metavar "TREESTATEDBPATH"
+                                            <> help "Path to tree state LMDB database directory (e.g. xyz/database-v4/treestate-0)"
+                                        )
+                                    <*> strOption
+                                        ( long "account-map-db-path"
+                                            <> metavar "ACCOUNTMAPDBPATH"
+                                            <> help "Path to account map LMDB database directory (e.g. xyz/database-v4/accountmap)"
+                                        )
+                                    <*> strOption
+                                        ( long "block-state-path"
+                                            <> metavar "BLOCKSTATEPATH"
+                                            <> help "Path to block state file (e.g. xyz/database-v4/blockstate-0.dat)"
+                                        )
+                                    <*> strOption
+                                        ( long "out-dir"
+                                            <> metavar "OUTDIR"
+                                            <> help "Path to directory to output dump to"
+                                        )
+                                    <*> option
+                                        auto
+                                        ( long "protocol-version"
+                                            <> metavar "PROTOCOLVERSION"
+                                            <> help "Protocol version for the tree state and block state, e.g. P9"
+                                        )
+                                    <*> option
+                                        auto
+                                        ( long "block-heights"
+                                            <> metavar "BLOCKHEIGHTS"
+                                            <> help "Relative heights of block to dump state for"
+                                        )
+                                )
+                                (progDesc "Dump block state")
+                            )
+                       )
+                )
+            )