laantungir
/
bitcoin
mirror of https://github.com/bitcoin/bitcoin.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 70991e683d into b510893d00

This commit is contained in:
l0rinc 2025-10-08 02:01:23 +02:00 committed by GitHub
parent b510893d00 70991e683d
commit 51d4ef4ea1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 82 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/coins.cpp
View File

@ -76,9 +76,6 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi
bool inserted;
std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>());
bool fresh = false;
if (!inserted) {
cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage();
}
if (!possible_overwrite) {
if (!it->second.coin.IsSpent()) {
throw std::logic_error("Attempted to overwrite an unspent coin (when possible_overwrite is false)");
@ -98,8 +95,13 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi
// DIRTY, then it can be marked FRESH.
fresh = !it->second.IsDirty();
}
if (!inserted) {
m_dirty_count -= it->second.IsDirty();
cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage();
}
it->second.coin = std::move(coin);
CCoinsCacheEntry::SetDirty(*it, m_sentinel);
++m_dirty_count;
if (fresh) CCoinsCacheEntry::SetFresh(*it, m_sentinel);
cachedCoinsUsage += it->second.coin.DynamicMemoryUsage();
TRACEPOINT(utxocache, add,
@ -111,9 +113,13 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi
}
void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin) {
cachedCoinsUsage += coin.DynamicMemoryUsage();
const auto mem_usage{coin.DynamicMemoryUsage()};
auto [it, inserted] = cacheCoins.try_emplace(std::move(outpoint), std::move(coin));
if (inserted) CCoinsCacheEntry::SetDirty(*it, m_sentinel);
if (inserted) {
CCoinsCacheEntry::SetDirty(*it, m_sentinel);
++m_dirty_count;
cachedCoinsUsage += mem_usage;
}
}
void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check_for_overwrite) {
@ -130,6 +136,7 @@ void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool
bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) {
CCoinsMap::iterator it = FetchCoin(outpoint);
if (it == cacheCoins.end()) return false;
m_dirty_count -= it->second.IsDirty();
cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage();
TRACEPOINT(utxocache, spent,
outpoint.hash.data(),
@ -144,6 +151,7 @@ bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) {
cacheCoins.erase(it);
} else {
CCoinsCacheEntry::SetDirty(*it, m_sentinel);
++m_dirty_count;
it->second.coin.Clear();
}
return true;
@ -202,8 +210,9 @@ bool CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &ha
} else {
entry.coin = it->second.coin;
}
cachedCoinsUsage += entry.coin.DynamicMemoryUsage();
CCoinsCacheEntry::SetDirty(*itUs, m_sentinel);
++m_dirty_count;
cachedCoinsUsage += entry.coin.DynamicMemoryUsage();
// We can mark it FRESH in the parent if it was FRESH in the child
// Otherwise it might have just been flushed from the parent's cache
// and already exist in the grandparent
@ -222,6 +231,7 @@ bool CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &ha
if (itUs->second.IsFresh() && it->second.coin.IsSpent()) {
// The grandparent cache does not have an entry, and the coin
// has been spent. We can just delete it from the parent cache.
m_dirty_count -= itUs->second.IsDirty();
cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage();
cacheCoins.erase(itUs);
} else {
@ -235,7 +245,10 @@ bool CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &ha
itUs->second.coin = it->second.coin;
}
cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage();
CCoinsCacheEntry::SetDirty(*itUs, m_sentinel);
if (!itUs->second.IsDirty()) {
CCoinsCacheEntry::SetDirty(*itUs, m_sentinel);
++m_dirty_count;
}
// NOTE: It isn't safe to mark the coin as FRESH in the parent
// cache. If it already existed and was spent in the parent
// cache then marking it FRESH would prevent that spentness
@ -248,21 +261,23 @@ bool CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &ha
}
bool CCoinsViewCache::Flush() {
auto cursor{CoinsViewCacheCursor(cachedCoinsUsage, m_sentinel, cacheCoins, /*will_erase=*/true)};
auto cursor{CoinsViewCacheCursor(cachedCoinsUsage, m_dirty_count, m_sentinel, cacheCoins, /*will_erase=*/true)};
bool fOk = base->BatchWrite(cursor, hashBlock);
if (fOk) {
cacheCoins.clear();
ReallocateCache();
}
cachedCoinsUsage = 0;
m_dirty_count = 0;
return fOk;
}
bool CCoinsViewCache::Sync()
{
auto cursor{CoinsViewCacheCursor(cachedCoinsUsage, m_sentinel, cacheCoins, /*will_erase=*/false)};
auto cursor{CoinsViewCacheCursor(cachedCoinsUsage, m_dirty_count, m_sentinel, cacheCoins, /*will_erase=*/false)};
bool fOk = base->BatchWrite(cursor, hashBlock);
if (fOk) {
Assume(m_dirty_count == 0);
if (m_sentinel.second.Next() != &m_sentinel) {
/* BatchWrite must clear flags of all entries */
throw std::logic_error("Not all unspent flagged entries were cleared");
@ -306,6 +321,7 @@ void CCoinsViewCache::ReallocateCache()
{
// Cache should be empty when we're calling this.
assert(cacheCoins.size() == 0);
Assume(m_dirty_count == 0);
cacheCoins.~CCoinsMap();
m_cache_coins_memory_resource.~CCoinsMapMemoryResource();
::new (&m_cache_coins_memory_resource) CCoinsMapMemoryResource{};
@ -316,6 +332,7 @@ void CCoinsViewCache::SanityCheck() const
{
size_t recomputed_usage = 0;
size_t count_flagged = 0;
size_t dirty_count = 0;
for (const auto& [_, entry] : cacheCoins) {
unsigned attr = 0;
if (entry.IsDirty()) attr |= 1;
@ -327,6 +344,9 @@ void CCoinsViewCache::SanityCheck() const
// Recompute cachedCoinsUsage.
recomputed_usage += entry.coin.DynamicMemoryUsage();
// Recompute dirty_count.
dirty_count += entry.IsDirty();
// Count the number of entries we expect in the linked list.
if (entry.IsDirty() || entry.IsFresh()) ++count_flagged;
}
@ -343,6 +363,7 @@ void CCoinsViewCache::SanityCheck() const
}
assert(count_linked == count_flagged);
assert(recomputed_usage == cachedCoinsUsage);
assert(dirty_count == m_dirty_count);
}
static const size_t MIN_TRANSACTION_OUTPUT_WEIGHT = WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxOut());

18
src/coins.h
View File

@ -272,10 +272,11 @@ struct CoinsViewCacheCursor
//! Calling CCoinsMap::clear() afterwards is faster because a CoinsCachePair cannot be coerced back into a
//! CCoinsMap::iterator to be erased, and must therefore be looked up again by key in the CCoinsMap before being erased.
CoinsViewCacheCursor(size_t& usage LIFETIMEBOUND,
CoinsCachePair& sentinel LIFETIMEBOUND,
CCoinsMap& map LIFETIMEBOUND,
bool will_erase) noexcept
: m_usage(usage), m_sentinel(sentinel), m_map(map), m_will_erase(will_erase) {}
size_t& dirty_count LIFETIMEBOUND,
CoinsCachePair& sentinel LIFETIMEBOUND,
CCoinsMap& map LIFETIMEBOUND,
bool will_erase) noexcept
: m_usage(usage), m_dirty_count(dirty_count), m_sentinel(sentinel), m_map(map), m_will_erase(will_erase) {}
inline CoinsCachePair* Begin() const noexcept { return m_sentinel.second.Next(); }
inline CoinsCachePair* End() const noexcept { return &m_sentinel; }
@ -284,6 +285,7 @@ struct CoinsViewCacheCursor
inline CoinsCachePair* NextAndMaybeErase(CoinsCachePair& current) noexcept
{
const auto next_entry{current.second.Next()};
m_dirty_count -= current.second.IsDirty();
// If we are not going to erase the cache, we must still erase spent entries.
// Otherwise, clear the state of the entry.
if (!m_will_erase) {
@ -298,8 +300,11 @@ struct CoinsViewCacheCursor
}
inline bool WillErase(CoinsCachePair& current) const noexcept { return m_will_erase || current.second.coin.IsSpent(); }
size_t GetDirtyCount() const noexcept { return m_dirty_count; }
size_t GetTotalCount() const noexcept { return m_map.size(); }
private:
size_t& m_usage;
size_t& m_dirty_count;
CoinsCachePair& m_sentinel;
CCoinsMap& m_map;
bool m_will_erase;
@ -377,6 +382,8 @@ protected:
/* Cached dynamic memory usage for the inner Coin objects. */
mutable size_t cachedCoinsUsage{0};
/* Running count of dirty Coin cache entries. */
mutable size_t m_dirty_count{0};
public:
CCoinsViewCache(CCoinsView *baseIn, bool deterministic = false);
@ -463,6 +470,9 @@ public:
//! Calculate the size of the cache (in number of transaction outputs)
unsigned int GetCacheSize() const;
//! Calculate the number of dirty cache entries (transaction outputs)
size_t GetDirtyCount() const noexcept { return m_dirty_count; }
//! Calculate the size of the cache (in bytes)
size_t DynamicMemoryUsage() const;

16
src/test/coins_tests.cpp
View File

@ -100,6 +100,7 @@ public:
CCoinsMap& map() const { return cacheCoins; }
CoinsCachePair& sentinel() const { return m_sentinel; }
size_t& usage() const { return cachedCoinsUsage; }
size_t& dirty() const { return m_dirty_count; }
};
} // namespace
@ -194,8 +195,11 @@ void SimulationTest(CCoinsView* base, bool fake_best_block)
(coin.IsSpent() ? added_an_entry : updated_an_entry) = true;
coin = newcoin;
}
bool is_overwrite = !coin.IsSpent() || m_rng.rand32() & 1;
stack.back()->AddCoin(COutPoint(txid, 0), std::move(newcoin), is_overwrite);
if (COutPoint op(txid, 0); !coin.IsSpent() && m_rng.randbool()) {
stack.back()->EmplaceCoinInternalDANGER(std::move(op), std::move(newcoin));
} else {
stack.back()->AddCoin(std::move(op), std::move(newcoin), /*possible_overwrite=*/!coin.IsSpent() || m_rng.randbool());
}
} else {
// Spend the coin.
removed_an_entry = true;
@ -663,7 +667,8 @@ static void WriteCoinsViewEntry(CCoinsView& view, const MaybeCoin& cache_coin)
CCoinsMapMemoryResource resource;
CCoinsMap map{0, CCoinsMap::hasher{}, CCoinsMap::key_equal{}, &resource};
auto usage{cache_coin ? InsertCoinsMapEntry(map, sentinel, *cache_coin) : 0};
auto cursor{CoinsViewCacheCursor(usage, sentinel, map, /*will_erase=*/true)};
size_t dirty_count{cache_coin && cache_coin->IsDirty() ? 1U : 0U};
auto cursor{CoinsViewCacheCursor(usage, dirty_count, sentinel, map, /*will_erase=*/true)};
BOOST_CHECK(view.BatchWrite(cursor, {}));
}
@ -674,7 +679,10 @@ public:
{
auto base_cache_coin{base_value == ABSENT ? MISSING : CoinEntry{base_value, CoinEntry::State::DIRTY}};
WriteCoinsViewEntry(base, base_cache_coin);
if (cache_coin) cache.usage() += InsertCoinsMapEntry(cache.map(), cache.sentinel(), *cache_coin);
if (cache_coin) {
cache.usage() += InsertCoinsMapEntry(cache.map(), cache.sentinel(), *cache_coin);
cache.dirty() += cache_coin->IsDirty();
}
}
CCoinsView root;

30
src/test/fuzz/coins_view.cpp
View File

@ -59,25 +59,19 @@ void TestCoinsView(FuzzedDataProvider& fuzzed_data_provider, CCoinsView& backend
if (random_coin.IsSpent()) {
return;
}
Coin coin = random_coin;
bool expected_code_path = false;
const bool possible_overwrite = fuzzed_data_provider.ConsumeBool();
try {
coins_view_cache.AddCoin(random_out_point, std::move(coin), possible_overwrite);
expected_code_path = true;
} catch (const std::logic_error& e) {
if (e.what() == std::string{"Attempted to overwrite an unspent coin (when possible_overwrite is false)"}) {
COutPoint outpoint{random_out_point};
Coin coin{random_coin};
if (fuzzed_data_provider.ConsumeBool()) {
const bool possible_overwrite{fuzzed_data_provider.ConsumeBool()};
try {
coins_view_cache.AddCoin(outpoint, std::move(coin), possible_overwrite);
} catch (const std::logic_error& e) {
assert(e.what() == std::string{"Attempted to overwrite an unspent coin (when possible_overwrite is false)"});
assert(!possible_overwrite);
expected_code_path = true;
// AddCoin() decreases cachedCoinsUsage by the memory usage of the old coin at the beginning and
// increases it by the value of the new coin at the end. If it throws in the process, the value
// of cachedCoinsUsage would have been incorrectly decreased, leading to an underflow later on.
// To avoid this, use Flush() to reset the value of cachedCoinsUsage in sync with the cacheCoins
// mapping.
(void)coins_view_cache.Flush();
}
} else {
coins_view_cache.EmplaceCoinInternalDANGER(std::move(outpoint), std::move(coin));
}
assert(expected_code_path);
},
[&] {
(void)coins_view_cache.Flush();
@ -132,6 +126,7 @@ void TestCoinsView(FuzzedDataProvider& fuzzed_data_provider, CCoinsView& backend
CoinsCachePair sentinel{};
sentinel.second.SelfRef(sentinel);
size_t usage{0};
size_t dirty_count{0};
CCoinsMapMemoryResource resource;
CCoinsMap coins_map{0, SaltedOutpointHasher{/*deterministic=*/true}, CCoinsMap::key_equal{}, &resource};
LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000)
@ -153,10 +148,11 @@ void TestCoinsView(FuzzedDataProvider& fuzzed_data_provider, CCoinsView& backend
if (dirty) CCoinsCacheEntry::SetDirty(*it, sentinel);
if (fresh) CCoinsCacheEntry::SetFresh(*it, sentinel);
usage += it->second.coin.DynamicMemoryUsage();
dirty_count += dirty;
}
bool expected_code_path = false;
try {
auto cursor{CoinsViewCacheCursor(usage, sentinel, coins_map, /*will_erase=*/true)};
auto cursor{CoinsViewCacheCursor(usage, dirty_count, sentinel, coins_map, /*will_erase=*/true)};
uint256 best_block{coins_view_cache.GetBestBlock()};
if (fuzzed_data_provider.ConsumeBool()) best_block = ConsumeUInt256(fuzzed_data_provider);
// Set best block hash to non-null to satisfy the assertion in CCoinsViewDB::BatchWrite().

14
src/txdb.cpp
View File

@ -8,6 +8,7 @@
#include <coins.h>
#include <dbwrapper.h>
#include <logging.h>
#include <logging/timer.h>
#include <primitives/transaction.h>
#include <random.h>
#include <serialize.h>
@ -25,6 +26,9 @@ static constexpr uint8_t DB_HEAD_BLOCKS{'H'};
// Keys used in previous version that might still be found in the DB:
static constexpr uint8_t DB_COINS{'c'};
// Threshold for warning when writing this many dirty cache entries to disk.
static constexpr size_t WARN_FLUSH_COINS_COUNT{10'000'000};
bool CCoinsViewDB::NeedsUpgrade()
{
std::unique_ptr<CDBIterator> cursor{m_db->NewIterator()};
@ -93,7 +97,7 @@ std::vector<uint256> CCoinsViewDB::GetHeadBlocks() const {
bool CCoinsViewDB::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashBlock) {
CDBBatch batch(*m_db);
size_t count = 0;
size_t changed = 0;
const size_t dirty_count{cursor.GetDirtyCount()};
assert(!hashBlock.IsNull());
uint256 old_tip = GetBestBlock();
@ -109,6 +113,10 @@ bool CCoinsViewDB::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashB
}
}
if (dirty_count > WARN_FLUSH_COINS_COUNT) LogWarning("Flushing large (%d entries) UTXO set to disk, it may take several minutes", dirty_count);
LOG_TIME_MILLIS_WITH_CATEGORY(strprintf("write coins cache to disk (%d out of %d cached coins)",
dirty_count, cursor.GetTotalCount()), BCLog::BENCH);
// In the first batch, mark the database as being in the middle of a
// transition from old_tip to hashBlock.
// A vector is used for future extensibility, as we may want to support
@ -124,8 +132,6 @@ bool CCoinsViewDB::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashB
} else {
batch.Write(entry, it->second.coin);
}
changed++;
}
count++;
it = cursor.NextAndMaybeErase(*it);
@ -150,7 +156,7 @@ bool CCoinsViewDB::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashB
LogDebug(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.ApproximateSize() * (1.0 / 1048576.0));
bool ret = m_db->WriteBatch(batch);
LogDebug(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
LogDebug(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...", (unsigned int)dirty_count, (unsigned int)count);
return ret;
}

12
src/validation.cpp
View File

@ -88,8 +88,6 @@ using node::CBlockIndexHeightOnlyComparator;
using node::CBlockIndexWorkComparator;
using node::SnapshotMetadata;
/** Size threshold for warning about slow UTXO set flush to disk. */
static constexpr size_t WARN_FLUSH_COINS_SIZE = 1 << 30; // 1 GiB
/** Time window to wait between writing blocks/block index and chainstate to disk.
* Randomize writing time inside the window to prevent a situation where the
* network over time settles into a few cohorts of synchronized writers.
@ -2772,8 +2770,8 @@ bool Chainstate::FlushStateToDisk(
std::set<int> setFilesToPrune;
bool full_flush_completed = false;
const size_t coins_count = CoinsTip().GetCacheSize();
const size_t coins_mem_usage = CoinsTip().DynamicMemoryUsage();
[[maybe_unused]] const size_t coins_count{CoinsTip().GetCacheSize()};
[[maybe_unused]] const size_t coins_mem_usage{CoinsTip().DynamicMemoryUsage()};
try {
{
@ -2867,16 +2865,12 @@ bool Chainstate::FlushStateToDisk(
}
if (!CoinsTip().GetBestBlock().IsNull()) {
if (coins_mem_usage >= WARN_FLUSH_COINS_SIZE) LogWarning("Flushing large (%d GiB) UTXO set to disk, it may take several minutes", coins_mem_usage >> 30);
LOG_TIME_MILLIS_WITH_CATEGORY(strprintf("write coins cache to disk (%d coins, %.2fKiB)",
coins_count, coins_mem_usage >> 10), BCLog::BENCH);
// Typical Coin structures on disk are around 48 bytes in size.
// Pushing a new one to the database can cause it to be written
// twice (once in the log, and once in the tables). This is already
// an overestimation, as most will delete an existing entry or
// overwrite one. Still, use a conservative safety factor of 2.
if (!CheckDiskSpace(m_chainman.m_options.datadir, 48 * 2 * 2 * CoinsTip().GetCacheSize())) {
if (!CheckDiskSpace(m_chainman.m_options.datadir, 48 * 2 * 2 * CoinsTip().GetDirtyCount())) {
return FatalError(m_chainman.GetNotifications(), state, _("Disk space is too low!"));
}
// Flush the chainstate (which may refer to block index entries).