1// Copyright (c) 2011-present The Bitcoin Core developers
2// Distributed under the MIT software license, see the accompanying
3// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5#ifndef BITCOIN_NODE_BLOCKSTORAGE_H
6#define BITCOIN_NODE_BLOCKSTORAGE_H
7
8#include <attributes.h>
9#include <chain.h>
10#include <dbwrapper.h>
11#include <flatfile.h>
12#include <kernel/blockmanager_opts.h>
13#include <kernel/chainparams.h>
14#include <kernel/cs_main.h>
15#include <kernel/messagestartchars.h>
16#include <primitives/block.h>
17#include <serialize.h>
18#include <streams.h>
19#include <sync.h>
20#include <uint256.h>
21#include <util/byte_units.h>
22#include <util/expected.h>
23#include <util/fs.h>
24#include <util/hasher.h>
25#include <util/obfuscation.h>
26
27#include <algorithm>
28#include <array>
29#include <atomic>
30#include <cstddef>
31#include <cstdint>
32#include <functional>
33#include <iosfwd>
34#include <limits>
35#include <map>
36#include <memory>
37#include <optional>
38#include <set>
39#include <span>
40#include <string>
41#include <unordered_map>
42#include <utility>
43#include <vector>
44
45class BlockValidationState;
46class CBlockUndo;
47class Chainstate;
48class ChainstateManager;
49namespace Consensus {
50struct Params;
51}
52namespace util {
53class SignalInterrupt;
54} // namespace util
55
56namespace kernel {
57class CBlockFileInfo
58{
59public:
60    uint32_t nBlocks{};      //!< number of blocks stored in file
61    uint64_t nSize{};        //!< number of used bytes of block file
62    uint64_t nUndoSize{};    //!< number of used bytes in the undo file
63    uint32_t nHeightFirst{}; //!< lowest height of block in file
64    uint32_t nHeightLast{};  //!< highest height of block in file
65    uint64_t nTimeFirst{};   //!< earliest time of block in file
66    uint64_t nTimeLast{};    //!< latest time of block in file
67
68    SERIALIZE_METHODS(CBlockFileInfo, obj)
69    {
70        READWRITE(VARINT(obj.nBlocks));
71        READWRITE(VARINT(obj.nSize));
72        READWRITE(VARINT(obj.nUndoSize));
73        READWRITE(VARINT(obj.nHeightFirst));
74        READWRITE(VARINT(obj.nHeightLast));
75        READWRITE(VARINT(obj.nTimeFirst));
76        READWRITE(VARINT(obj.nTimeLast));
77    }
78
79    CBlockFileInfo() = default;
80
81    std::string ToString() const;
82
83    /** update statistics (does not update nSize) */
84    void AddBlock(unsigned int nHeightIn, uint64_t nTimeIn)
85    {
86        if (nBlocks == 0 || nHeightFirst > nHeightIn)
87            nHeightFirst = nHeightIn;
88        if (nBlocks == 0 || nTimeFirst > nTimeIn)
89            nTimeFirst = nTimeIn;
90        nBlocks++;
91        if (nHeightIn > nHeightLast)
92            nHeightLast = nHeightIn;
93        if (nTimeIn > nTimeLast)
94            nTimeLast = nTimeIn;
95    }
96};
97
98/** Access to the block database (blocks/index/) */
99class BlockTreeDB : public CDBWrapper
100{
101public:
102    using CDBWrapper::CDBWrapper;
103    void WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*>>& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo);
104    bool ReadBlockFileInfo(int nFile, CBlockFileInfo& info);
105    bool ReadLastBlockFile(int& nFile);
106    void WriteReindexing(bool fReindexing);
107    void ReadReindexing(bool& fReindexing);
108    void WriteFlag(const std::string& name, bool fValue);
109    bool ReadFlag(const std::string& name, bool& fValue);
110    bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, const util::SignalInterrupt& interrupt)
111        EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
112};
113} // namespace kernel
114
115namespace node {
116using kernel::CBlockFileInfo;
117using kernel::BlockTreeDB;
118
119/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
120inline constexpr unsigned int BLOCKFILE_CHUNK_SIZE{16_MiB};
121/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */
122inline constexpr unsigned int UNDOFILE_CHUNK_SIZE{1_MiB};
123/** The maximum size of a blk?????.dat file (since 0.8) */
124inline constexpr unsigned int MAX_BLOCKFILE_SIZE{128_MiB};
125
126/** Size of header written by WriteBlock before a serialized CBlock (8 bytes) */
127inline constexpr uint32_t STORAGE_HEADER_BYTES{std::tuple_size_v<MessageStartChars> + sizeof(unsigned int)};
128
129/** Total overhead when writing undo data: header (8 bytes) plus checksum (32 bytes) */
130inline constexpr uint32_t UNDO_DATA_DISK_OVERHEAD{STORAGE_HEADER_BYTES + uint256::size()};
131
132// Because validation code takes pointers to the map's CBlockIndex objects, if
133// we ever switch to another associative container, we need to either use a
134// container that has stable addressing (true of all std associative
135// containers), or make the key a `std::unique_ptr<CBlockIndex>`
136using BlockMap = std::unordered_map<uint256, CBlockIndex, BlockHasher>;
137
138struct CBlockIndexWorkComparator {
139    bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
140    using is_transparent = void;
141};
142
143struct CBlockIndexHeightOnlyComparator {
144    /* Only compares the height of two block indices, doesn't try to tie-break */
145    bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
146};
147
148struct PruneLockInfo {
149    /// Height of earliest block that should be kept and not pruned
150    int height_first{std::numeric_limits<int>::max()};
151};
152
153enum BlockfileType {
154    // Values used as array indexes - do not change carelessly.
155    NORMAL = 0,
156    ASSUMED = 1,
157    NUM_TYPES = 2,
158};
159
160std::ostream& operator<<(std::ostream& os, const BlockfileType& type);
161
162struct BlockfileCursor {
163    // The latest blockfile number.
164    int file_num{0};
165
166    // Track the height of the highest block in file_num whose undo
167    // data has been written. Block data is written to block files in download
168    // order, but is written to undo files in validation order, which is
169    // usually in order by height. To avoid wasting disk space, undo files will
170    // be trimmed whenever the corresponding block file is finalized and
171    // the height of the highest block written to the block file equals the
172    // height of the highest block written to the undo file. This is a
173    // heuristic and can sometimes preemptively trim undo files that will write
174    // more data later, and sometimes fail to trim undo files that can't have
175    // more data written later.
176    int undo_height{0};
177};
178
179std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor);
180
181enum class ReadRawError {
182    IO,
183    BadPartRange,
184};
185
186/**
187 * Maintains a tree of blocks (stored in `m_block_index`) which is consulted
188 * to determine where the most-work tip is.
189 *
190 * This data is used mostly in `Chainstate` - information about, e.g.,
191 * candidate tips is not maintained here.
192 */
193class BlockManager
194{
195    friend Chainstate;
196    friend ChainstateManager;
197
198private:
199    const CChainParams& GetParams() const { return m_opts.chainparams; }
200    const Consensus::Params& GetConsensus() const { return m_opts.chainparams.GetConsensus(); }
201    /**
202     * Load the blocktree off disk and into memory. Populate certain metadata
203     * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral
204     * collections like m_dirty_blockindex.
205     */
206    bool LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash)
207        EXCLUSIVE_LOCKS_REQUIRED(cs_main);
208
209    /** Return false if block file or undo file flushing fails. */
210    [[nodiscard]] bool FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
211
212    /** Return false if undo file flushing fails. */
213    [[nodiscard]] bool FlushUndoFile(int block_file, bool finalize = false);
214
215    /**
216     * Helper function performing various preparations before a block can be saved to disk:
217     * Returns the correct position for the block to be saved, which may be in the current or a new
218     * block file depending on nAddSize. May flush the previous blockfile to disk if full, updates
219     * blockfile info, and checks if there is enough disk space to save the block.
220     *
221     * The nAddSize argument passed to this function should include not just the size of the serialized CBlock, but also the size of
222     * separator fields (STORAGE_HEADER_BYTES).
223     */
224    [[nodiscard]] FlatFilePos FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
225    [[nodiscard]] bool FlushChainstateBlockFile(int tip_height) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
226    [[nodiscard]] bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
227
228    AutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const;
229
230    /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
231    void FindFilesToPruneManual(
232        std::set<int>& setFilesToPrune,
233        int nManualPruneHeight,
234        const Chainstate& chain);
235
236    /**
237     * Prune block and undo files (blk???.dat and rev???.dat) so that the disk space used is less than a user-defined target.
238     * The user sets the target (in MB) on the command line or in config file.  This will be run on startup and whenever new
239     * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex
240     * (which in this case means the blockchain must be re-downloaded.)
241     *
242     * Pruning functions are called from FlushStateToDisk when the m_check_for_pruning flag has been set.
243     * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.)
244     * Pruning cannot take place until the longest chain is at least a certain length (CChainParams::nPruneAfterHeight).
245     * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip.
246     * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files.
247     * A db flag records the fact that at least some block files have been pruned.
248     *
249     * @param[out]   setFilesToPrune   The set of file indices that can be unlinked will be returned
250     * @param        last_prune        The last height we're able to prune, according to the prune locks