Where validation becomes chain state

5001                MessageStartChars buf;
5002                blkdat.FindByte(std::byte(params.MessageStart()[0]));
5003                nRewind = blkdat.GetPos() + 1;
5004                blkdat >> buf;
5005                if (buf != params.MessageStart()) {
5006                    continue;
5007                }
5008                // read size
5009                blkdat >> nSize;
5010                if (nSize < 80 || nSize > MAX_BLOCK_SERIALIZED_SIZE)
5011                    continue;
5012            } catch (const std::exception&) {
5013                // no valid block header found; don't complain
5014                // (this happens at the end of every blk.dat file)
5015                break;
5016            }
5017            try {
5018                // read block header
5019                const uint64_t nBlockPos{blkdat.GetPos()};
5020                if (dbp)
5021                    dbp->nPos = nBlockPos;
5022                blkdat.SetLimit(nBlockPos + nSize);
5023                CBlockHeader header;
5024                blkdat >> header;
5025                const uint256 hash{header.GetHash()};
5026                // Skip the rest of this block (this may read from disk into memory); position to the marker before the
5027                // next block, but it's still possible to rewind to the start of the current block (without a disk read).
5028                nRewind = nBlockPos + nSize;
5029                blkdat.SkipTo(nRewind);
5030
5031                std::shared_ptr<CBlock> pblock{}; // needs to remain available after the cs_main lock is released to avoid duplicate reads from disk
5032
5033                {
5034                    LOCK(cs_main);
5035                    // detect out of order blocks, and store them for later
5036                    if (hash != params.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(header.hashPrevBlock)) {
5037                        LogDebug(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(),
5038                                 header.hashPrevBlock.ToString());
5039                        if (dbp && blocks_with_unknown_parent) {
5040                            blocks_with_unknown_parent->emplace(header.hashPrevBlock, *dbp);
5041                        }
5042                        continue;
5043                    }
5044
5045                    // process in case the block isn't known yet
5046                    const CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash);
5047                    if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) {
5048                        // This block can be processed immediately; rewind to its start, read and deserialize it.
5049                        blkdat.SetPos(nBlockPos);
5050                        pblock = std::make_shared<CBlock>();
5051                        blkdat >> TX_WITH_WITNESS(*pblock);
5052                        nRewind = blkdat.GetPos();
5053
5054                        BlockValidationState state;
5055                        if (AcceptBlock(pblock, state, nullptr, true, dbp, nullptr, true)) {
5056                            nLoaded++;
5057                        }
5058                        if (state.IsError()) {
5059                            break;
5060                        }
5061                    } else if (hash != params.GetConsensus().hashGenesisBlock && pindex->nHeight % 1000 == 0) {
5062                        LogDebug(BCLog::REINDEX, "Block Import: already had block %s at height %d\n", hash.ToString(), pindex->nHeight);
5063                    }
5064                }
5065
5066                // Activate the genesis block so normal node progress can continue
5067                // During first -reindex, this will only connect Genesis since
5068                // ActivateBestChain only connects blocks which are in the block tree db,
5069                // which only contains blocks whose parents are in it.
5070                // But do this only if genesis isn't activated yet, to avoid connecting many blocks
5071                // without assumevalid in the case of a continuation of a reindex that
5072                // was interrupted by the user.
5073                if (hash == params.GetConsensus().hashGenesisBlock && WITH_LOCK(::cs_main, return ActiveHeight()) == -1) {
5074                    BlockValidationState state;
5075                    if (!ActiveChainstate().ActivateBestChain(state, nullptr)) {
5076                        break;
5077                    }
5078                }
5079
5080                if (m_blockman.IsPruneMode() && m_blockman.m_blockfiles_indexed && pblock) {
5081                    // must update the tip for pruning to work while importing with -loadblock.
5082                    // this is a tradeoff to conserve disk space at the expense of time
5083                    // spent updating the tip to be able to prune.
5084                    // otherwise, ActivateBestChain won't be called by the import process
5085                    // until after all of the block files are loaded. ActivateBestChain can be
5086                    // called by concurrent network message processing. but, that is not
5087                    // reliable for the purpose of pruning while importing.
5088                    if (auto result{ActivateBestChains()}; !result) {
5089                        LogDebug(BCLog::REINDEX, "%s\n", util::ErrorString(result).original);
5090                        break;
5091                    }
5092                }
5093
5094                NotifyHeaderTip();
5095
5096                if (!blocks_with_unknown_parent) continue;
5097
5098                // Recursively process earlier encountered successors of this block
5099                std::deque<uint256> queue;
5100                queue.push_back(hash);
5101                while (!queue.empty()) {
5102                    uint256 head = queue.front();
5103                    queue.pop_front();
5104                    auto range = blocks_with_unknown_parent->equal_range(head);
5105                    while (range.first != range.second) {
5106                        std::multimap<uint256, FlatFilePos>::iterator it = range.first;
5107                        std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>();
5108                        if (m_blockman.ReadBlock(*pblockrecursive, it->second, {})) {
5109                            const auto& block_hash{pblockrecursive->GetHash()};
5110                            LogDebug(BCLog::REINDEX, "%s: Processing out of order child %s of %s", __func__, block_hash.ToString(), head.ToString());
5111                            LOCK(cs_main);
5112                            BlockValidationState dummy;
5113                            if (AcceptBlock(pblockrecursive, dummy, nullptr, true, &it->second, nullptr, true)) {
5114                                nLoaded++;
5115                                queue.push_back(block_hash);
5116                            }
5117                        }
5118                        range.first++;
5119                        blocks_with_unknown_parent->erase(it);
5120                        NotifyHeaderTip();
5121                    }
5122                }
5123            } catch (const std::exception& e) {
5124                // historical bugs added extra data to the block files that does not deserialize cleanly.
5125                // commonly this data is between readable blocks, but it does not really matter. such data is not fatal to the import process.
5126                // the code that reads the block files deals with invalid data by simply ignoring it.
5127                // it continues to search for the next {4 byte magic message start bytes + 4 byte length + block} that does deserialize cleanly
5128                // and passes all of the other block validation checks dealing with POW and the merkle root, etc...
5129                // we merely note with this informational log message when unexpected data is encountered.
5130                // we could also be experiencing a storage system read error, or a read of a previous bad write. these are possible, but
5131                // less likely scenarios. we don't have enough information to tell a difference here.
5132                // the reindex process is not the place to attempt to clean and/or compact the block files. if so desired, a studious node operator
5133                // may use knowledge of the fact that the block files are not entirely pristine in order to prepare a set of pristine, and
5134                // perhaps ordered, block files for later reindexing.
5135                LogDebug(BCLog::REINDEX, "%s: unexpected data at file offset 0x%x - %s. continuing\n", __func__, (nRewind - 1), e.what());
5136            }
5137        }
5138    } catch (const std::runtime_error& e) {
5139        GetNotifications().fatalError(strprintf(_("System error while loading external block file: %s"), e.what()));
5140    }
5141    LogInfo("Loaded %i blocks from external file in %dms", nLoaded, Ticks<std::chrono::milliseconds>(SteadyClock::now() - start));
5142}
5143
5144bool ChainstateManager::ShouldCheckBlockIndex() const
5145{
5146    // Assert to verify Flatten() has been called.
5147    if (!*Assert(m_options.check_block_index)) return false;
5148    if (FastRandomContext().randrange(*m_options.check_block_index) >= 1) return false;
5149    return true;
5150}
5151
5152void ChainstateManager::CheckBlockIndex() const
5153{
5154    if (!ShouldCheckBlockIndex()) {
5155        return;
5156    }
5157
5158    LOCK(cs_main);
5159
5160    // During a reindex, we read the genesis block and call CheckBlockIndex before ActivateBestChain,
5161    // so we have the genesis block in m_blockman.m_block_index but no active chain. (A few of the
5162    // tests when iterating the block tree require that m_chain has been initialized.)
5163    if (ActiveChain().Height() < 0) {
5164        assert(m_blockman.m_block_index.size() <= 1);
5165        return;
5166    }
5167
5168    // Build forward-pointing data structure for the entire block tree.
5169    // For performance reasons, indexes of the best header chain are stored in a vector (within CChain).
5170    // All remaining blocks are stored in a multimap.
5171    // The best header chain can differ from the active chain: E.g. its entries may belong to blocks that
5172    // are not yet validated.
5173    CChain best_hdr_chain;
5174    assert(m_best_header);
5175    assert(!(m_best_header->nStatus & BLOCK_FAILED_VALID));
5176    best_hdr_chain.SetTip(*m_best_header);
5177
5178    std::multimap<const CBlockIndex*, const CBlockIndex*> forward;
5179    for (auto& [_, block_index] : m_blockman.m_block_index) {
5180        // Only save indexes in forward that are not part of the best header chain.
5181        if (!best_hdr_chain.Contains(block_index)) {
5182            // Only genesis, which must be part of the best header chain, can have a nullptr parent.
5183            assert(block_index.pprev);
5184            forward.emplace(block_index.pprev, &block_index);
5185        }
5186    }
5187    assert(forward.size() + best_hdr_chain.Height() + 1 == m_blockman.m_block_index.size());
5188
5189    const CBlockIndex* pindex = best_hdr_chain[0];
5190    assert(pindex);
5191    // Iterate over the entire block tree, using depth-first search.
5192    // Along the way, remember whether there are blocks on the path from genesis
5193    // block being explored which are the first to have certain properties.
5194    size_t nNodes = 0;
5195    int nHeight = 0;
5196    const CBlockIndex* pindexFirstInvalid = nullptr;              // Oldest ancestor of pindex which is invalid.
5197    const CBlockIndex* pindexFirstMissing = nullptr;              // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA, since assumeutxo snapshot if used.
5198    const CBlockIndex* pindexFirstNeverProcessed = nullptr;       // Oldest ancestor of pindex for which nTx == 0, since assumeutxo snapshot if used.
5199    const CBlockIndex* pindexFirstNotTreeValid = nullptr;         // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not).
5200    const CBlockIndex* pindexFirstNotTransactionsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not), since assumeutxo snapshot if used.
5201    const CBlockIndex* pindexFirstNotChainValid = nullptr;        // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not), since assumeutxo snapshot if used.
5202    const CBlockIndex* pindexFirstNotScriptsValid = nullptr;      // Oldest ancestor of pindex which does not have BLOCK_VALID_SCRIPTS (regardless of being valid or not), since assumeutxo snapshot if used.
5203
5204    // After checking an assumeutxo snapshot block, reset pindexFirst pointers
5205    // to earlier blocks that have not been downloaded or validated yet, so
5206    // checks for later blocks can assume the earlier blocks were validated and
5207    // be stricter, testing for more requirements.
5208    const CBlockIndex* snap_base{CurrentChainstate().SnapshotBase()};
5209    const CBlockIndex *snap_first_missing{}, *snap_first_notx{}, *snap_first_notv{}, *snap_first_nocv{}, *snap_first_nosv{};
5210    auto snap_update_firsts = [&] {
5211        if (pindex == snap_base) {
5212            std::swap(snap_first_missing, pindexFirstMissing);
5213            std::swap(snap_first_notx, pindexFirstNeverProcessed);
5214            std::swap(snap_first_notv, pindexFirstNotTransactionsValid);
5215            std::swap(snap_first_nocv, pindexFirstNotChainValid);
5216            std::swap(snap_first_nosv, pindexFirstNotScriptsValid);
5217        }
5218    };
5219
5220    while (pindex != nullptr) {
5221        nNodes++;
5222        if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex;
5223        if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) {
5224            pindexFirstMissing = pindex;
5225        }
5226        if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) pindexFirstNeverProcessed = pindex;
5227        if (pindex->pprev != nullptr && pindexFirstNotTreeValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex;
5228
5229        if (pindex->pprev != nullptr) {
5230            if (pindexFirstNotTransactionsValid == nullptr &&
5231                    (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) {
5232                pindexFirstNotTransactionsValid = pindex;
5233            }
5234
5235            if (pindexFirstNotChainValid == nullptr &&
5236                    (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) {
5237                pindexFirstNotChainValid = pindex;
5238            }
5239
5240            if (pindexFirstNotScriptsValid == nullptr &&
5241                    (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) {
5242                pindexFirstNotScriptsValid = pindex;
5243            }
5244        }
5245
5246        // Begin: actual consistency checks.
5247        if (pindex->pprev == nullptr) {
5248            // Genesis block checks.
5249            assert(pindex->GetBlockHash() == GetConsensus().hashGenesisBlock); // Genesis block's hash must match.
5250            for (const auto& c : m_chainstates) {