Reading block files, undo records and pruning metadata

501                    if (pindex->nStatus & BLOCK_HAVE_DATA) {
502                        AddUnlinkedBlock(pindex);
503                    }
504                }
505            } else {
506                pindex->m_chain_tx_count = pindex->nTx;
507            }
508        }
509
510        if (pindex->nStatus & BLOCK_FAILED_CHILD) {
511            // BLOCK_FAILED_CHILD is deprecated, but may still exist on disk. Replace it with BLOCK_FAILED_VALID.
512            pindex->nStatus = (pindex->nStatus & ~BLOCK_FAILED_CHILD) | BLOCK_FAILED_VALID;
513            m_dirty_blockindex.insert(pindex);
514        }
515        if (!(pindex->nStatus & BLOCK_FAILED_VALID) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_VALID)) {
516            // All descendants of invalid blocks are invalid too.
517            pindex->nStatus |= BLOCK_FAILED_VALID;
518            m_dirty_blockindex.insert(pindex);
519        }
520
521        if (pindex->pprev) {
522            pindex->BuildSkip();
523        }
524    }
525
526    return true;
527}
528
529void BlockManager::WriteBlockIndexDB()
530{
531    AssertLockHeld(::cs_main);
532    std::vector<std::pair<int, const CBlockFileInfo*>> vFiles;
533    vFiles.reserve(m_dirty_fileinfo.size());
534    for (std::set<int>::iterator it = m_dirty_fileinfo.begin(); it != m_dirty_fileinfo.end();) {
535        vFiles.emplace_back(*it, &m_blockfile_info[*it]);
536        m_dirty_fileinfo.erase(it++);
537    }
538    std::vector<const CBlockIndex*> vBlocks;
539    vBlocks.reserve(m_dirty_blockindex.size());
540    for (std::set<CBlockIndex*>::iterator it = m_dirty_blockindex.begin(); it != m_dirty_blockindex.end();) {
541        vBlocks.push_back(*it);
542        m_dirty_blockindex.erase(it++);
543    }
544    int max_blockfile{this->MaxBlockfileNum()};
545    m_block_tree_db->WriteBatchSync(vFiles, max_blockfile, vBlocks);
546}
547
548bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash)
549{
550    AssertLockHeld(::cs_main);
551    if (!LoadBlockIndex(snapshot_blockhash)) {
552        return false;
553    }
554    int max_blockfile_num{0};
555
556    // Load block file info
557    m_block_tree_db->ReadLastBlockFile(max_blockfile_num);
558    m_blockfile_info.resize(max_blockfile_num + 1);
559    LogInfo("Loading block index db: last block file = %i", max_blockfile_num);
560    for (int nFile = 0; nFile <= max_blockfile_num; nFile++) {
561        m_block_tree_db->ReadBlockFileInfo(nFile, m_blockfile_info[nFile]);
562    }
563    LogInfo("Loading block index db: last block file info: %s", m_blockfile_info[max_blockfile_num].ToString());
564    for (int nFile = max_blockfile_num + 1; true; nFile++) {
565        CBlockFileInfo info;
566        if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) {
567            m_blockfile_info.push_back(info);
568        } else {
569            break;
570        }
571    }
572
573    // Check presence of blk files
574    LogInfo("Checking all blk files are present...");
575    std::set<int> setBlkDataFiles;
576    for (const auto& [_, block_index] : m_block_index) {
577        if (block_index.nStatus & BLOCK_HAVE_DATA) {
578            setBlkDataFiles.insert(block_index.nFile);
579        }
580    }
581    for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) {
582        FlatFilePos pos(*it, 0);
583        if (OpenBlockFile(pos, /*fReadOnly=*/true).IsNull()) {
584            return false;
585        }
586    }
587
588    {
589        // Initialize the blockfile cursors.
590        for (size_t i = 0; i < m_blockfile_info.size(); ++i) {
591            const auto last_height_in_file = m_blockfile_info[i].nHeightLast;
592            m_blockfile_cursors[BlockfileTypeForHeight(last_height_in_file)] = {static_cast<int>(i), 0};
593        }
594    }
595
596    // Check whether we have ever pruned block & undo files
597    m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned);
598    if (m_have_pruned) {
599        LogInfo("Loading block index db: Block files have previously been pruned");
600    }
601
602    // Check whether we need to continue reindexing
603    bool fReindexing = false;
604    m_block_tree_db->ReadReindexing(fReindexing);
605    if (fReindexing) m_blockfiles_indexed = false;
606
607    return true;
608}
609
610void BlockManager::ScanAndUnlinkAlreadyPrunedFiles()
611{
612    AssertLockHeld(::cs_main);
613    int max_blockfile{this->MaxBlockfileNum()};
614    if (!m_have_pruned) {
615        return;
616    }
617
618    std::set<int> block_files_to_prune;
619    for (int file_number = 0; file_number < max_blockfile; file_number++) {
620        if (m_blockfile_info[file_number].nSize == 0) {
621            block_files_to_prune.insert(file_number);
622        }
623    }
624
625    UnlinkPrunedFiles(block_files_to_prune);
626}
627
628bool BlockManager::IsBlockPruned(const CBlockIndex& block) const
629{
630    AssertLockHeld(::cs_main);
631    return m_have_pruned && !(block.nStatus & BLOCK_HAVE_DATA) && (block.nTx > 0);
632}
633
634const CBlockIndex& BlockManager::GetFirstBlock(const CBlockIndex& upper_block, uint32_t status_mask, const CBlockIndex* lower_block) const
635{
636    AssertLockHeld(::cs_main);
637    const CBlockIndex* last_block = &upper_block;
638    assert((last_block->nStatus & status_mask) == status_mask); // 'upper_block' must satisfy the status mask
639    while (last_block->pprev && ((last_block->pprev->nStatus & status_mask) == status_mask)) {
640        if (lower_block) {
641            // Return if we reached the lower_block
642            if (last_block == lower_block) return *lower_block;
643            // if range was surpassed, means that 'lower_block' is not part of the 'upper_block' chain
644            // and so far this is not allowed.
645            assert(last_block->nHeight >= lower_block->nHeight);
646        }
647        last_block = last_block->pprev;
648    }
649    assert(last_block != nullptr);
650    return *last_block;
651}
652
653bool BlockManager::CheckBlockDataAvailability(const CBlockIndex& upper_block, const CBlockIndex& lower_block, BlockStatus block_status)
654{
655    if (!(upper_block.nStatus & block_status)) return false;
656    const auto& first_block = GetFirstBlock(upper_block, block_status, &lower_block);
657    // Special case: the genesis block has no undo data
658    if (block_status & BLOCK_HAVE_UNDO && lower_block.nHeight == 0 && first_block.nHeight == 1) {
659        // This might indicate missing data, or it could simply reflect the expected absence of undo data for the genesis block.
660        // To distinguish between the two, check if all required block data *except* undo is available up to the genesis block.
661        BlockStatus flags{block_status & ~BLOCK_HAVE_UNDO};
662        return first_block.pprev && first_block.pprev->nStatus & flags;
663    }
664    return &first_block == &lower_block;
665}
666
667// If we're using -prune with -reindex, then delete block files that will be ignored by the
668// reindex.  Since reindexing works by starting at block file 0 and looping until a blockfile
669// is missing, do the same here to delete any later block files after a gap.  Also delete all
670// rev files since they'll be rewritten by the reindex anyway.  This ensures that m_blockfile_info
671// is in sync with what's actually on disk by the time we start downloading, so that pruning
672// works correctly.
673void BlockManager::CleanupBlockRevFiles() const
674{
675    std::map<std::string, fs::path> mapBlockFiles;
676
677    // Glob all blk?????.dat and rev?????.dat files from the blocks directory.
678    // Remove the rev files immediately and insert the blk file paths into an
679    // ordered map keyed by block file index.
680    LogInfo("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune");
681    for (fs::directory_iterator it(m_opts.blocks_dir); it != fs::directory_iterator(); it++) {
682        const std::string path = fs::PathToString(it->path().filename());
683        if (fs::is_regular_file(*it) &&
684            path.length() == 12 &&
685            path.ends_with(".dat"))
686        {
687            if (path.starts_with("blk")) {
688                mapBlockFiles[path.substr(3, 5)] = it->path();
689            } else if (path.starts_with("rev")) {
690                remove(it->path());
691            }
692        }
693    }
694
695    // Remove all block files that aren't part of a contiguous set starting at
696    // zero by walking the ordered map (keys are block file indices) by
697    // keeping a separate counter.  Once we hit a gap (or if 0 doesn't exist)
698    // start removing block files.
699    int nContigCounter = 0;
700    for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
701        if (LocaleIndependentAtoi<int>(item.first) == nContigCounter) {
702            nContigCounter++;
703            continue;
704        }
705        remove(item.second);
706    }
707}
708
709CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n)
710{
711    AssertLockHeld(::cs_main);
712    return &m_blockfile_info.at(n);
713}
714
715bool BlockManager::ReadBlockUndo(CBlockUndo& blockundo, const CBlockIndex& index) const
716{
717    const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
718
719    // Open history file to read
720    AutoFile file{OpenUndoFile(pos, true)};
721    if (file.IsNull()) {
722        LogError("OpenUndoFile failed for %s while reading block undo", pos.ToString());
723        return false;
724    }
725    BufferedReader filein{std::move(file)};
726
727    try {
728        // Read block
729        HashVerifier verifier{filein}; // Use HashVerifier, as reserializing may lose data, c.f. commit d3424243
730
731        verifier << index.pprev->GetBlockHash();
732        verifier >> blockundo;
733
734        uint256 hashChecksum;
735        filein >> hashChecksum;
736
737        // Verify checksum
738        if (hashChecksum != verifier.GetHash()) {
739            LogError("Checksum mismatch at %s while reading block undo", pos.ToString());
740            return false;
741        }
742    } catch (const std::exception& e) {
743        LogError("Deserialize or I/O error - %s at %s while reading block undo", e.what(), pos.ToString());
744        return false;
745    }
746
747    return true;
748}
749
750bool BlockManager::FlushUndoFile(int block_file, bool finalize)