kernel: Add chainstate manager object to C header

This is the main driver class for anything validation related, so expose
it here.

Creating the chainstate manager options will currently also trigger the
creation of their respectively configured directories.

The chainstate manager and block manager options are consolidated into a
single object. The kernel might eventually introduce a separate block
manager object for the purposes of being a light-weight block store
reader.

The chainstate manager will associate with the context with which it was
created for the duration of its lifetime. It is only valid if that
context remains in memory too.

The tests now also create dedicated temporary directories. This is
similar to the behaviour in the existing unit test framework.

Co-authored-by: stickies-v <stickies-v@protonmail.com>

src/kernel/bitcoinkernel.cpp

@@ -7,6 +7,7 @@
 #include <kernel/bitcoinkernel.h>
 
 #include <consensus/amount.h>
+#include <kernel/caches.h>
 #include <kernel/chainparams.h>
 #include <kernel/checks.h>
 #include <kernel/context.h>
@@ -14,6 +15,7 @@
 #include <kernel/notifications_interface.h>
 #include <kernel/warning.h>
 #include <logging.h>
+#include <node/blockstorage.h>
 #include <primitives/transaction.h>
 #include <script/interpreter.h>
 #include <script/script.h>
@@ -21,6 +23,7 @@
 #include <streams.h>
 #include <sync.h>
 #include <tinyformat.h>
+#include <util/fs.h>
 #include <util/result.h>
 #include <util/signalinterrupt.h>
 #include <util/translation.h>
@@ -363,6 +366,39 @@ public:
     }
 };
 
+//! Helper struct to wrap the ChainstateManager-related Options
+struct ChainstateManagerOptions {
+    mutable Mutex m_mutex;
+    ChainstateManager::Options m_chainman_options GUARDED_BY(m_mutex);
+    node::BlockManager::Options m_blockman_options GUARDED_BY(m_mutex);
+    std::shared_ptr<const Context> m_context;
+
+    ChainstateManagerOptions(const std::shared_ptr<const Context>& context, const fs::path& data_dir, const fs::path& blocks_dir)
+        : m_chainman_options{ChainstateManager::Options{
+              .chainparams = *context->m_chainparams,
+              .datadir = data_dir,
+              .notifications = *context->m_notifications}},
+          m_blockman_options{node::BlockManager::Options{
+              .chainparams = *context->m_chainparams,
+              .blocks_dir = blocks_dir,
+              .notifications = *context->m_notifications,
+              .block_tree_db_params = DBParams{
+                  .path = data_dir / "blocks" / "index",
+                  .cache_bytes = kernel::CacheSizes{DEFAULT_KERNEL_CACHE}.block_tree_db,
+              }}},
+          m_context{context}
+    {
+    }
+};
+
+struct ChainMan {
+    std::unique_ptr<ChainstateManager> m_chainman;
+    std::shared_ptr<const Context> m_context;
+
+    ChainMan(std::unique_ptr<ChainstateManager> chainman, std::shared_ptr<const Context> context)
+        : m_chainman(std::move(chainman)), m_context(std::move(context)) {}
+};
+
 } // namespace
 
 struct btck_Transaction : Handle<btck_Transaction, std::shared_ptr<const CTransaction>> {};
@@ -372,6 +408,8 @@ struct btck_LoggingConnection : Handle<btck_LoggingConnection, LoggingConnection
 struct btck_ContextOptions : Handle<btck_ContextOptions, ContextOptions> {};
 struct btck_Context : Handle<btck_Context, std::shared_ptr<const Context>> {};
 struct btck_ChainParameters : Handle<btck_ChainParameters, std::unique_ptr<const CChainParams>> {};
+struct btck_ChainstateManagerOptions : Handle<btck_ChainstateManagerOptions, ChainstateManagerOptions> {};
+struct btck_ChainstateManager : Handle<btck_ChainstateManager, ChainMan> {};
 
 btck_Transaction* btck_transaction_create(const void* raw_transaction, size_t raw_transaction_len)
 {
@@ -632,3 +670,52 @@ void btck_context_destroy(btck_Context* context)
 {
     delete context;
 }
+
+btck_ChainstateManagerOptions* btck_chainstate_manager_options_create(const btck_Context* context, const char* data_dir, size_t data_dir_len, const char* blocks_dir, size_t blocks_dir_len)
+{
+    try {
+        fs::path abs_data_dir{fs::absolute(fs::PathFromString({data_dir, data_dir_len}))};
+        fs::create_directories(abs_data_dir);
+        fs::path abs_blocks_dir{fs::absolute(fs::PathFromString({blocks_dir, blocks_dir_len}))};
+        fs::create_directories(abs_blocks_dir);
+        return btck_ChainstateManagerOptions::create(btck_Context::get(context), abs_data_dir, abs_blocks_dir);
+    } catch (const std::exception& e) {
+        LogError("Failed to create chainstate manager options: %s", e.what());
+        return nullptr;
+    }
+}
+
+void btck_chainstate_manager_options_destroy(btck_ChainstateManagerOptions* options)
+{
+    delete options;
+}
+
+btck_ChainstateManager* btck_chainstate_manager_create(
+    const btck_ChainstateManagerOptions* chainman_opts)
+{
+    try {
+        auto& opts{btck_ChainstateManagerOptions::get(chainman_opts)};
+        LOCK(opts.m_mutex);
+        auto& context{opts.m_context};
+        auto chainman{std::make_unique<ChainstateManager>(*context->m_interrupt, opts.m_chainman_options, opts.m_blockman_options)};
+        return btck_ChainstateManager::create(std::move(chainman), context);
+    } catch (const std::exception& e) {
+        LogError("Failed to create chainstate manager: %s", e.what());
+        return nullptr;
+    }
+}
+
+void btck_chainstate_manager_destroy(btck_ChainstateManager* chainman)
+{
+    {
+        LOCK(btck_ChainstateManager::get(chainman).m_chainman->GetMutex());
+        for (Chainstate* chainstate : btck_ChainstateManager::get(chainman).m_chainman->GetAll()) {
+            if (chainstate->CanFlushToDisk()) {
+                chainstate->ForceFlushStateToDisk();
+                chainstate->ResetCoinsViews();
+            }
+        }
+    }
+
+    delete chainman;
+}

src/kernel/bitcoinkernel.h

@@ -165,6 +165,26 @@ typedef struct btck_Context btck_Context;
  */
 typedef struct btck_BlockTreeEntry btck_BlockTreeEntry;
 
+/**
+ * Opaque data structure for holding options for creating a new chainstate
+ * manager.
+ *
+ * The chainstate manager options are used to set some parameters for the
+ * chainstate manager. For now it just holds default options.
+ */
+typedef struct btck_ChainstateManagerOptions btck_ChainstateManagerOptions;
+
+/**
+ * Opaque data structure for holding a chainstate manager.
+ *
+ * The chainstate manager is the central object for doing validation tasks as
+ * well as retrieving data from the chain. Internally it is a complex data
+ * structure with diverse functionality.
+ *
+ * Its functionality will be more and more exposed in the future.
+ */
+typedef struct btck_ChainstateManager btck_ChainstateManager;
+
 /** Current sync state passed to tip changed callbacks. */
 typedef uint8_t btck_SynchronizationState;
 #define btck_SynchronizationState_INIT_REINDEX ((btck_SynchronizationState)(0))
@@ -671,6 +691,58 @@ BITCOINKERNEL_API void btck_context_destroy(btck_Context* context);
 
 ///@}
 
+/** @name ChainstateManagerOptions
+ * Functions for working with chainstate manager options.
+ */
+///@{
+
+/**
+ * @brief Create options for the chainstate manager.
+ *
+ * @param[in] context          Non-null, the created options and through it the chainstate manager will
+                               associate with this kernel context for the duration of their lifetimes.
+ * @param[in] data_directory   Non-null, path string of the directory containing the chainstate data.
+ *                             If the directory does not exist yet, it will be created.
+ * @param[in] blocks_directory Non-null, path string of the directory containing the block data. If
+ *                             the directory does not exist yet, it will be created.
+ * @return                     The allocated chainstate manager options, or null on error.
+ */
+BITCOINKERNEL_API btck_ChainstateManagerOptions* BITCOINKERNEL_WARN_UNUSED_RESULT btck_chainstate_manager_options_create(
+    const btck_Context* context,
+    const char* data_directory,
+    size_t data_directory_len,
+    const char* blocks_directory,
+    size_t blocks_directory_len) BITCOINKERNEL_ARG_NONNULL(1, 2);
+
+/**
+ * Destroy the chainstate manager options.
+ */
+BITCOINKERNEL_API void btck_chainstate_manager_options_destroy(btck_ChainstateManagerOptions* chainstate_manager_options);
+
+///@}
+
+/** @name ChainstateManager
+ * Functions for chainstate management.
+ */
+///@{
+
+/**
+ * @brief Create a chainstate manager. This is the main object for many
+ * validation tasks as well as for retrieving data from the chain. *
+ *
+ * @param[in] chainstate_manager_options Non-null, created by @ref btck_chainstate_manager_options_create.
+ * @return                               The allocated chainstate manager, or null on error.
+ */
+BITCOINKERNEL_API btck_ChainstateManager* BITCOINKERNEL_WARN_UNUSED_RESULT btck_chainstate_manager_create(
+    const btck_ChainstateManagerOptions* chainstate_manager_options) BITCOINKERNEL_ARG_NONNULL(1);
+
+/**
+ * Destroy the chainstate manager.
+ */
+BITCOINKERNEL_API void btck_chainstate_manager_destroy(btck_ChainstateManager* chainstate_manager);
+
+///@}
+
 #ifdef __cplusplus
 } // extern "C"
 #endif // __cplusplus

src/kernel/bitcoinkernel_wrapper.h

@@ -11,6 +11,7 @@
 #include <memory>
 #include <span>
 #include <stdexcept>
+#include <string>
 #include <string_view>
 #include <type_traits>
 #include <utility>
@@ -618,6 +619,28 @@ public:
 
     Context()
         : Handle{btck_context_create(ContextOptions{}.get())} {}
+
+    friend class ChainstateManagerOptions;
+};
+
+class ChainstateManagerOptions : UniqueHandle<btck_ChainstateManagerOptions, btck_chainstate_manager_options_destroy>
+{
+public:
+    ChainstateManagerOptions(const Context& context, const std::string& data_dir, const std::string& blocks_dir)
+        : UniqueHandle{btck_chainstate_manager_options_create(context.get(), data_dir.c_str(), data_dir.length(), blocks_dir.c_str(), blocks_dir.length())}
+    {
+    }
+
+    friend class ChainMan;
+};
+
+class ChainMan : UniqueHandle<btck_ChainstateManager, btck_chainstate_manager_destroy>
+{
+public:
+    ChainMan(const Context& context, const ChainstateManagerOptions& chainman_opts)
+        : UniqueHandle{btck_chainstate_manager_create(chainman_opts.get())}
+    {
+    }
 };
 
 } // namespace btck

src/test/kernel/test_kernel.cpp

@@ -11,15 +11,36 @@
 #include <charconv>
 #include <cstdint>
 #include <cstdlib>
+#include <filesystem>
 #include <iostream>
 #include <memory>
+#include <random>
 #include <ranges>
 #include <span>
+#include <string>
 #include <string_view>
 #include <vector>
 
 using namespace btck;
 
+std::string random_string(uint32_t length)
+{
+    const std::string chars = "0123456789"
+                              "abcdefghijklmnopqrstuvwxyz"
+                              "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+    static std::random_device rd;
+    static std::default_random_engine dre{rd()};
+    static std::uniform_int_distribution<> distribution(0, chars.size() - 1);
+
+    std::string random;
+    random.reserve(length);
+    for (uint32_t i = 0; i < length; i++) {
+        random += chars[distribution(dre)];
+    }
+    return random;
+}
+
 std::vector<std::byte> hex_string_to_byte_vec(std::string_view hex)
 {
     std::vector<std::byte> bytes;
@@ -60,6 +81,20 @@ public:
     }
 };
 
+struct TestDirectory {
+    std::filesystem::path m_directory;
+    TestDirectory(std::string directory_name)
+        : m_directory{std::filesystem::temp_directory_path() / (directory_name + random_string(16))}
+    {
+        std::filesystem::create_directories(m_directory);
+    }
+
+    ~TestDirectory()
+    {
+        std::filesystem::remove_all(m_directory);
+    }
+};
+
 class TestKernelNotifications : public KernelNotifications<TestKernelNotifications>
 {
 public:
@@ -437,3 +472,45 @@ BOOST_AUTO_TEST_CASE(btck_context_tests)
         Context context{options};
     }
 }
+
+Context create_context(std::shared_ptr<TestKernelNotifications> notifications, ChainType chain_type)
+{
+    ContextOptions options{};
+    ChainParams params{chain_type};
+    options.SetChainParams(params);
+    options.SetNotifications(notifications);
+    auto context{Context{options}};
+    return context;
+}
+
+BOOST_AUTO_TEST_CASE(btck_chainman_tests)
+{
+    btck_LoggingOptions logging_options = {
+        .log_timestamps = true,
+        .log_time_micros = true,
+        .log_threadnames = false,
+        .log_sourcelocations = false,
+        .always_print_category_levels = true,
+    };
+    Logger logger{std::make_unique<TestLog>(TestLog{}), logging_options};
+    auto test_directory{TestDirectory{"chainman_test_bitcoin_kernel"}};
+
+    { // test with default context
+        Context context{};
+        ChainstateManagerOptions chainman_opts{context, test_directory.m_directory.string(), (test_directory.m_directory / "blocks").string()};
+        ChainMan chainman{context, chainman_opts};
+    }
+
+    { // test with default context options
+        ContextOptions options{};
+        Context context{options};
+        ChainstateManagerOptions chainman_opts{context, test_directory.m_directory.string(), (test_directory.m_directory / "blocks").string()};
+        ChainMan chainman{context, chainman_opts};
+    }
+
+    auto notifications{std::make_shared<TestKernelNotifications>()};
+    auto context{create_context(notifications, ChainType::MAINNET)};
+
+    ChainstateManagerOptions chainman_opts{context, test_directory.m_directory.string(), (test_directory.m_directory / "blocks").string()};
+    ChainMan chainman{context, chainman_opts};
+}

test/lint/test_runner/src/main.rs

@@ -373,6 +373,7 @@ fn lint_std_filesystem() -> LintResult {
             "./src/",
             ":(exclude)src/ipc/libmultiprocess/",
             ":(exclude)src/util/fs.h",
+            ":(exclude)src/test/kernel/test_kernel.cpp",
         ])
         .status()
         .expect("command error")