Reading block files, undo records and pruning metadata

1001        }
1002        {
1003            BufferedWriter fileout{file};
1004
1005            // Write index header
1006            fileout << GetParams().MessageStart() << blockundo_size;
1007            pos.nPos += STORAGE_HEADER_BYTES;
1008            {
1009                // Calculate checksum
1010                HashWriter hasher{};
1011                hasher << block.pprev->GetBlockHash() << blockundo;
1012                // Write undo data & checksum
1013                fileout << blockundo << hasher.GetHash();
1014            }
1015            // BufferedWriter will flush pending data to file when fileout goes out of scope.
1016        }
1017
1018        // Make sure that the file is closed before we call `FlushUndoFile`.
1019        if (file.fclose() != 0) {
1020            LogError("Failed to close block undo file %s: %s", pos.ToString(), SysErrorString(errno));
1021            return FatalError(m_opts.notifications, state, _("Failed to close block undo file."));
1022        }
1023
1024        // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
1025        // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
1026        // in the block file info as below; note that this does not catch the case where the undo writes are keeping up
1027        // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
1028        // the FindNextBlockPos function
1029        if (pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[pos.nFile].nHeightLast) {
1030            // Do not propagate the return code, a failed flush here should not
1031            // be an indication for a failed write. If it were propagated here,
1032            // the caller would assume the undo data not to be written, when in
1033            // fact it is. Note though, that a failed flush might leave the data
1034            // file untrimmed.
1035            if (!FlushUndoFile(pos.nFile, true)) {
1036                LogWarning("Failed to flush undo file %05i\n", pos.nFile);
1037            }
1038        } else if (pos.nFile == cursor.file_num && block.nHeight > cursor.undo_height) {
1039            cursor.undo_height = block.nHeight;
1040        }
1041        // update nUndoPos in block index
1042        block.nUndoPos = pos.nPos;
1043        block.nStatus |= BLOCK_HAVE_UNDO;
1044        m_dirty_blockindex.insert(&block);
1045    }
1046
1047    return true;
1048}
1049
1050bool BlockManager::ReadBlock(CBlock& block, const FlatFilePos& pos, const std::optional<uint256>& expected_hash) const
1051{
1052    block.SetNull();
1053
1054    // Open history file to read
1055    const auto block_data{ReadRawBlock(pos)};
1056    if (!block_data) {
1057        return false;
1058    }
1059
1060    try {
1061        // Read block
1062        SpanReader{*block_data} >> TX_WITH_WITNESS(block);
1063    } catch (const std::exception& e) {
1064        LogError("Deserialize or I/O error - %s at %s while reading block", e.what(), pos.ToString());
1065        return false;
1066    }
1067
1068    const auto block_hash{block.GetHash()};
1069
1070    // Check the header
1071    if (!CheckProofOfWork(block_hash, block.nBits, GetConsensus())) {
1072        LogError("Errors in block header at %s while reading block", pos.ToString());
1073        return false;
1074    }
1075
1076    // Signet only: check block solution
1077    if (GetConsensus().signet_blocks && !CheckSignetBlockSolution(block, GetConsensus())) {
1078        LogError("Errors in block solution at %s while reading block", pos.ToString());
1079        return false;
1080    }
1081
1082    if (expected_hash && block_hash != *expected_hash) {
1083        LogError("GetHash() doesn't match index at %s while reading block (%s != %s)",
1084                 pos.ToString(), block_hash.ToString(), expected_hash->ToString());
1085        return false;
1086    }
1087
1088    return true;
1089}
1090
1091bool BlockManager::ReadBlock(CBlock& block, const CBlockIndex& index) const
1092{
1093    const FlatFilePos block_pos{WITH_LOCK(cs_main, return index.GetBlockPos())};
1094    return ReadBlock(block, block_pos, index.GetBlockHash());
1095}
1096
1097BlockManager::ReadRawBlockResult BlockManager::ReadRawBlock(const FlatFilePos& pos, std::optional<std::pair<size_t, size_t>> block_part) const
1098{
1099    if (pos.nPos < STORAGE_HEADER_BYTES) {
1100        // If nPos is less than STORAGE_HEADER_BYTES, we can't read the header that precedes the block data
1101        // This would cause an unsigned integer underflow when trying to position the file cursor
1102        // This can happen after pruning or default constructed positions
1103        LogError("Failed for %s while reading raw block storage header", pos.ToString());
1104        return util::Unexpected{ReadRawError::IO};
1105    }
1106    AutoFile filein{OpenBlockFile({pos.nFile, pos.nPos - STORAGE_HEADER_BYTES}, /*fReadOnly=*/true)};
1107    if (filein.IsNull()) {
1108        LogError("OpenBlockFile failed for %s while reading raw block", pos.ToString());
1109        return util::Unexpected{ReadRawError::IO};
1110    }
1111
1112    try {
1113        MessageStartChars blk_start;
1114        unsigned int blk_size;
1115
1116        filein >> blk_start >> blk_size;
1117
1118        if (blk_start != GetParams().MessageStart()) {
1119            LogError("Block magic mismatch for %s: %s versus expected %s while reading raw block",
1120                pos.ToString(), HexStr(blk_start), HexStr(GetParams().MessageStart()));
1121            return util::Unexpected{ReadRawError::IO};
1122        }
1123
1124        if (blk_size > MAX_SIZE) {
1125            LogError("Block data is larger than maximum deserialization size for %s: %s versus %s while reading raw block",
1126                pos.ToString(), blk_size, MAX_SIZE);
1127            return util::Unexpected{ReadRawError::IO};
1128        }
1129
1130        if (block_part) {
1131            const auto [offset, size]{*block_part};
1132            if (size == 0 || SaturatingAdd(offset, size) > blk_size) {
1133                return util::Unexpected{ReadRawError::BadPartRange}; // Avoid logging - offset/size come from untrusted REST input
1134            }
1135            filein.seek(offset, SEEK_CUR);
1136            blk_size = size;
1137        }
1138
1139        std::vector<std::byte> data(blk_size); // Zeroing of memory is intentional here
1140        filein.read(data);
1141        return data;
1142    } catch (const std::exception& e) {
1143        LogError("Read from block file failed: %s for %s while reading raw block", e.what(), pos.ToString());
1144        return util::Unexpected{ReadRawError::IO};
1145    }
1146}
1147
1148FlatFilePos BlockManager::WriteBlock(const CBlock& block, int nHeight)
1149{
1150    AssertLockHeld(::cs_main);
1151    const unsigned int block_size{static_cast<unsigned int>(GetSerializeSize(TX_WITH_WITNESS(block)))};
1152    FlatFilePos pos{FindNextBlockPos(block_size + STORAGE_HEADER_BYTES, nHeight, block.GetBlockTime())};
1153    if (pos.IsNull()) {
1154        LogError("FindNextBlockPos failed for %s while writing block", pos.ToString());
1155        return FlatFilePos();
1156    }
1157    AutoFile file{OpenBlockFile(pos, /*fReadOnly=*/false)};
1158    if (file.IsNull()) {
1159        LogError("OpenBlockFile failed for %s while writing block", pos.ToString());
1160        m_opts.notifications.fatalError(_("Failed to write block."));
1161        return FlatFilePos();
1162    }
1163    {
1164        BufferedWriter fileout{file};
1165
1166        // Write index header
1167        fileout << GetParams().MessageStart() << block_size;
1168        pos.nPos += STORAGE_HEADER_BYTES;
1169        // Write block
1170        fileout << TX_WITH_WITNESS(block);
1171    }
1172
1173    if (file.fclose() != 0) {
1174        LogError("Failed to close block file %s: %s", pos.ToString(), SysErrorString(errno));
1175        m_opts.notifications.fatalError(_("Failed to close file when writing block."));
1176        return FlatFilePos();
1177    }
1178
1179    return pos;
1180}
1181
1182static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
1183{
1184    // Bytes are serialized without length indicator, so this is also the exact
1185    // size of the XOR-key file.
1186    std::array<std::byte, Obfuscation::KEY_SIZE> obfuscation{};
1187
1188    // Consider this to be the first run if the blocksdir contains only hidden
1189    // files (those which start with a .). Checking for a fully-empty dir would
1190    // be too aggressive as a .lock file may have already been written.
1191    bool first_run = true;
1192    for (const auto& entry : fs::directory_iterator(opts.blocks_dir)) {
1193        const std::string path = fs::PathToString(entry.path().filename());
1194        if (!entry.is_regular_file() || !path.starts_with('.')) {
1195            first_run = false;
1196            break;
1197        }
1198    }
1199
1200    if (opts.use_xor && first_run) {
1201        // Only use random fresh key when the boolean option is set and on the
1202        // very first start of the program.
1203        FastRandomContext{}.fillrand(obfuscation);
1204    }
1205
1206    const fs::path xor_key_path{opts.blocks_dir / "xor.dat"};
1207    if (fs::exists(xor_key_path)) {
1208        // A pre-existing xor key file has priority.
1209        AutoFile xor_key_file{fsbridge::fopen(xor_key_path, "rb")};
1210        xor_key_file >> obfuscation;
1211    } else {
1212        // Create initial or missing xor key file
1213        AutoFile xor_key_file{fsbridge::fopen(xor_key_path,
1214#ifdef __MINGW64__
1215            "wb" // Temporary workaround for https://github.com/bitcoin/bitcoin/issues/30210
1216#else
1217            "wbx"
1218#endif
1219        )};
1220        xor_key_file << obfuscation;
1221        if (xor_key_file.fclose() != 0) {
1222            throw std::runtime_error{strprintf("Error closing XOR key file %s: %s",
1223                                               fs::PathToString(xor_key_path),
1224                                               SysErrorString(errno))};
1225        }
1226    }
1227    // If the user disabled the key, it must be zero.
1228    if (!opts.use_xor && obfuscation != decltype(obfuscation){}) {
1229        throw std::runtime_error{
1230            strprintf("The blocksdir XOR-key can not be disabled when a random key was already stored! "
1231                      "Stored key: '%s', stored path: '%s'.",
1232                      HexStr(obfuscation), fs::PathToString(xor_key_path)),
1233        };
1234    }
1235    LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(obfuscation));
1236    return Obfuscation{obfuscation};
1237}
1238
1239BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)
1240    : m_prune_mode{opts.prune_target > 0},
1241      m_obfuscation{InitBlocksdirXorKey(opts)},
1242      m_opts{std::move(opts)},
1243      m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
1244      m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},
1245      m_interrupt{interrupt}
1246{
1247    m_block_tree_db = std::make_unique<BlockTreeDB>(m_opts.block_tree_db_params);
1248
1249    if (m_opts.block_tree_db_params.wipe_data) {
1250        m_block_tree_db->WriteReindexing(true);