coins: warn on oversized -dbcache

Oversized allocations can cause out-of-memory errors or [heavy swapping](https://github.com/getumbrel/umbrel-os/issues/64#issuecomment-663637321), [grinding the system to a halt](https://x.com/murchandamus/status/1964432335849607224).

`LogOversizedDbCache()` now emits a startup warning if the configured `-dbcache` exceeds a cap derived from system RAM, using the same parsing/clamping as cache sizing via CalculateDbCacheBytes(). This isn't meant as a recommended setting, rather a likely upper limit.

Note that we're not modifying the set value, just issuing a warning.
Also note that the 75% calculation is rounded for the last two numbers since we have to divide first before multiplying, otherwise we wouldn't stay inside size_t on 32-bit systems - and this was simpler than casting back and forth.

We could have chosen the remaining free memory for the warning (e.g. warn if free memory is less than 1 GiB), but this is just a heuristic, we assumed that on systems with a lot of memory, other processes are also running, while memory constrained ones run only Core.

If total RAM < 2 GiB, cap is `DEFAULT_DB_CACHE` (`450 MiB`), otherwise it's 75% of total RAM.
The threshold is chosen to be close to values commonly used in [raspiblitz](https://github.com/raspiblitz/raspiblitz/blob/dev/home.admin/_provision.setup.sh#L98-L115) for common setups:

| Total RAM | `dbcache` (MiB) | raspiblitz % | proposed cap (MiB) |
|----------:|----------------:|-------------:|-------------------:|
|     1 GiB |             512 |        50.0% |               450* |
|     2 GiB |            1536 |        75.0% |               1536 |
|     4 GiB |            2560 |        62.5% |               3072 |
|     8 GiB |            4096 |        50.0% |               6144 |
|    16 GiB |            4096 |        25.0% |              12288 |
|    32 GiB |            4096 |        12.5% |              24576 |

[Umbrel issues](https://github.com/getumbrel/umbrel-os/issues/64#issuecomment-663816367) also mention 75% being the upper limit.

Starting `bitcoind` on an 8 GiB rpi4b with a dbcache of 7 GiB:
> ./build/bin/bitcoind -dbcache=7000

warns now as follows:
```
2025-09-07T17:24:29Z [warning] A 7000 MiB dbcache may be too large for a system memory of only 7800 MiB.
2025-09-07T17:24:29Z Cache configuration:
2025-09-07T17:24:29Z * Using 2.0 MiB for block index database
2025-09-07T17:24:29Z * Using 8.0 MiB for chain state database
2025-09-07T17:24:29Z * Using 6990.0 MiB for in-memory UTXO set (plus up to 286.1 MiB of unused mempool space)
```

Besides the [godbolt](https://godbolt.org/z/EPsaE3xTj) reproducers for the new total memory method, we also tested the warnings manually on:
- [x] Apple M4 Max, macOS 15.6.1
- [x] Intel Core i9-9900K, Ubuntu 24.04.2 LTS
- [x] Raspberry Pi 4 Model B, Armbian Linux 6.12.22-current-bcm2711
- [x] Intel Xeon x64, Windows 11 Home Version 24H2, OS Build 26100.4351

Co-authored-by: stickies-v <stickies-v@protonmail.com>
Co-authored-by: Hodlinator <172445034+hodlinator@users.noreply.github.com>
Co-authored-by: w0xlt <woltx@protonmail.com>

Github-Pull: #33333
Rebased-From: 168360f4ae
This commit is contained in:
Lőrinc 2025-09-06 21:25:13 -07:00 committed by fanquake
parent 49d4ebcbfe
commit 5226a92f28
No known key found for this signature in database
GPG Key ID: 2EEB9F5CC09526C1
5 changed files with 75 additions and 8 deletions

View File

@ -1733,6 +1733,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
// ********************************************************* Step 7: load block chain
// cache size calculations
node::LogOversizedDbCache(args);
const auto [index_cache_sizes, kernel_cache_sizes] = CalculateCacheSizes(args, g_enabled_filter_types.size());
LogInfo("Cache configuration:");

View File

@ -5,9 +5,12 @@
#include <node/caches.h>
#include <common/args.h>
#include <common/system.h>
#include <index/txindex.h>
#include <kernel/caches.h>
#include <logging.h>
#include <node/interface_ui.h>
#include <tinyformat.h>
#include <util/byte_units.h>
#include <algorithm>
@ -23,16 +26,20 @@ static constexpr size_t MAX_FILTER_INDEX_CACHE{1024_MiB};
static constexpr size_t MAX_32BIT_DBCACHE{1024_MiB};
namespace node {
size_t CalculateDbCacheBytes(const ArgsManager& args)
{
if (auto db_cache{args.GetIntArg("-dbcache")}) {
if (*db_cache < 0) db_cache = 0;
const uint64_t db_cache_bytes{SaturatingLeftShift<uint64_t>(*db_cache, 20)};
constexpr auto max_db_cache{sizeof(void*) == 4 ? MAX_32BIT_DBCACHE : std::numeric_limits<size_t>::max()};
return std::max<size_t>(MIN_DB_CACHE, std::min<uint64_t>(db_cache_bytes, max_db_cache));
}
return DEFAULT_DB_CACHE;
}
CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes)
{
// Convert -dbcache from MiB units to bytes. The total cache is floored by MIN_DB_CACHE and capped by max size_t value.
size_t total_cache{DEFAULT_DB_CACHE};
if (std::optional<int64_t> db_cache = args.GetIntArg("-dbcache")) {
if (*db_cache < 0) db_cache = 0;
uint64_t db_cache_bytes = SaturatingLeftShift<uint64_t>(*db_cache, 20);
constexpr auto max_db_cache{sizeof(void*) == 4 ? MAX_32BIT_DBCACHE : std::numeric_limits<size_t>::max()};
total_cache = std::max<size_t>(MIN_DB_CACHE, std::min<uint64_t>(db_cache_bytes, max_db_cache));
}
size_t total_cache{CalculateDbCacheBytes(args)};
IndexCacheSizes index_sizes;
index_sizes.tx_index = std::min(total_cache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? MAX_TX_INDEX_CACHE : 0);
@ -44,4 +51,15 @@ CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes)
}
return {index_sizes, kernel::CacheSizes{total_cache}};
}
void LogOversizedDbCache(const ArgsManager& args) noexcept
{
if (const auto total_ram{GetTotalRAM()}) {
const size_t db_cache{CalculateDbCacheBytes(args)};
if (ShouldWarnOversizedDbCache(db_cache, *total_ram)) {
InitWarning(bilingual_str{tfm::format(_("A %zu MiB dbcache may be too large for a system memory of only %zu MiB."),
db_cache >> 20, *total_ram >> 20)});
}
}
}
} // namespace node

View File

@ -27,6 +27,13 @@ struct CacheSizes {
kernel::CacheSizes kernel;
};
CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes = 0);
constexpr bool ShouldWarnOversizedDbCache(size_t dbcache, size_t total_ram) noexcept
{
const size_t cap{(total_ram < 2048_MiB) ? DEFAULT_DB_CACHE : (total_ram / 100) * 75};
return dbcache > cap;
}
void LogOversizedDbCache(const ArgsManager& args) noexcept;
} // namespace node
#endif // BITCOIN_NODE_CACHES_H

View File

@ -25,6 +25,7 @@ add_executable(test_bitcoin
blockmanager_tests.cpp
bloom_tests.cpp
bswap_tests.cpp
caches_tests.cpp
chainstate_write_tests.cpp
checkqueue_tests.cpp
cluster_linearize_tests.cpp

40
src/test/caches_tests.cpp Normal file
View File

@ -0,0 +1,40 @@
#include <node/caches.h>
#include <util/byte_units.h>
#include <boost/test/unit_test.hpp>
using namespace node;
BOOST_AUTO_TEST_SUITE(caches_tests)
BOOST_AUTO_TEST_CASE(oversized_dbcache_warning)
{
// memory restricted setup - cap is DEFAULT_DB_CACHE (450 MiB)
BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/4_MiB, /*total_ram=*/1024_MiB)); // Under cap
BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/512_MiB, /*total_ram=*/1024_MiB)); // At cap
BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/1500_MiB, /*total_ram=*/1024_MiB)); // Over cap
// 2 GiB RAM - cap is 75%
BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/1500_MiB, /*total_ram=*/2048_MiB)); // Under cap
BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/1600_MiB, /*total_ram=*/2048_MiB)); // Over cap
if constexpr (SIZE_MAX == UINT64_MAX) {
// 4 GiB RAM - cap is 75%
BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/2500_MiB, /*total_ram=*/4096_MiB)); // Under cap
BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/3500_MiB, /*total_ram=*/4096_MiB)); // Over cap
// 8 GiB RAM - cap is 75%
BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/6000_MiB, /*total_ram=*/8192_MiB)); // Under cap
BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/7000_MiB, /*total_ram=*/8192_MiB)); // Over cap
// 16 GiB RAM - cap is 75%
BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/10'000_MiB, /*total_ram=*/16384_MiB)); // Under cap
BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/15'000_MiB, /*total_ram=*/16384_MiB)); // Over cap
// 32 GiB RAM - cap is 75%
BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/20'000_MiB, /*total_ram=*/32768_MiB)); // Under cap
BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/30'000_MiB, /*total_ram=*/32768_MiB)); // Over cap
}
}
BOOST_AUTO_TEST_SUITE_END()