1// Copyright (c) 2018-present The Bitcoin Core developers
2// Distributed under the MIT software license, see the accompanying
3// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5#include <index/blockfilterindex.h>
6
7#include <blockfilter.h>
8#include <chain.h>
9#include <common/args.h>
10#include <dbwrapper.h>
11#include <flatfile.h>
12#include <hash.h>
13#include <index/base.h>
14#include <index/db_key.h>
15#include <interfaces/chain.h>
16#include <interfaces/types.h>
17#include <serialize.h>
18#include <streams.h>
19#include <sync.h>
20#include <uint256.h>
21#include <util/check.h>
22#include <util/byte_units.h>
23#include <util/fs.h>
24#include <util/hasher.h>
25#include <util/log.h>
26#include <util/syserror.h>
27
28#include <cerrno>
29#include <exception>
30#include <map>
31#include <optional>
32#include <stdexcept>
33#include <string>
34#include <tuple>
35#include <utility>
36#include <vector>
37
38/* The index database stores three items for each block: the disk location of the encoded filter,
39 * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by
40 * height, and those belonging to blocks that have been reorganized out of the active chain are
41 * indexed by block hash. This ensures that filter data for any block that becomes part of the
42 * active chain can always be retrieved, alleviating timing concerns.
43 *
44 * The filters themselves are stored in flat files and referenced by the LevelDB entries. This
45 * minimizes the amount of data written to LevelDB and keeps the database values constant size. The
46 * disk location of the next block filter to be written (represented as a FlatFilePos) is stored
47 * under the DB_FILTER_POS key.
48 *
49 * The logic for keys is shared with other indexes, see index/db_key.h.
50 */
51constexpr uint8_t DB_FILTER_POS{'P'};
52
53constexpr unsigned int MAX_FLTR_FILE_SIZE{16_MiB};
54/** The pre-allocation chunk size for fltr?????.dat files */
55constexpr unsigned int FLTR_FILE_CHUNK_SIZE{1_MiB};
56/** Maximum size of the cfheaders cache
57 *  We have a limit to prevent a bug in filling this cache
58 *  potentially turning into an OOM. At 2000 entries, this cache
59 *  is big enough for a 2,000,000 length block chain, which
60 *  we should be enough until ~2047. */
61constexpr size_t CF_HEADERS_CACHE_MAX_SZ{2000};
62
63namespace {
64
65std::string BlockFilterThreadName(BlockFilterType filter_type)
66{
67    switch (filter_type) {
68    case BlockFilterType::BASIC: return "blkfltbscidx";
69    case BlockFilterType::INVALID: return "";
70    } // no default case, so the compiler can warn about missing cases
71    assert(false);
72}
73
74struct DBVal {
75    uint256 hash;
76    uint256 header;
77    FlatFilePos pos;
78
79    SERIALIZE_METHODS(DBVal, obj) { READWRITE(obj.hash, obj.header, obj.pos); }
80};
81
82}; // namespace
83
84static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes;
85
86BlockFilterIndex::BlockFilterIndex(std::unique_ptr<interfaces::Chain> chain, BlockFilterType filter_type,
87                                   size_t n_cache_size, bool f_memory, bool f_wipe)
88    : BaseIndex(std::move(chain), BlockFilterTypeName(filter_type) + " block filter index", BlockFilterThreadName(filter_type))
89    , m_filter_type(filter_type)
90{
91    const std::string& filter_name = BlockFilterTypeName(filter_type);
92    if (filter_name.empty()) throw std::invalid_argument("unknown filter_type");
93
94    fs::path path = gArgs.GetDataDirNet() / "indexes" / "blockfilter" / fs::u8path(filter_name);
95    fs::create_directories(path);
96
97    m_db = std::make_unique<BaseIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe);
98    m_filter_fileseq = std::make_unique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE);
99}
100
101interfaces::Chain::NotifyOptions BlockFilterIndex::CustomOptions()
102{
103    interfaces::Chain::NotifyOptions options;
104    options.connect_undo_data = true;
105    return options;
106}
107
108bool BlockFilterIndex::CustomInit(const std::optional<interfaces::BlockRef>& block)
109{
110    if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) {
111        // Check that the cause of the read failure is that the key does not exist. Any other errors
112        // indicate database corruption or a disk failure, and starting the index would cause
113        // further corruption.
114        if (m_db->Exists(DB_FILTER_POS)) {
115            LogError("Cannot read current %s state; index may be corrupted",
116                      GetName());
117            return false;
118        }
119
120        // If the DB_FILTER_POS is not set, then initialize to the first location.
121        m_next_filter_pos.nFile = 0;
122        m_next_filter_pos.nPos = 0;
123    }
124
125    if (block) {
126        auto op_last_header = ReadFilterHeader(block->height, block->hash);
127        if (!op_last_header) {
128            LogError("Cannot read last block filter header; index may be corrupted");
129            return false;
130        }
131        m_last_header = *op_last_header;
132    }
133
134    return true;
135}
136
137bool BlockFilterIndex::CustomCommit(CDBBatch& batch)
138{
139    const FlatFilePos& pos = m_next_filter_pos;
140
141    // Flush current filter file to disk.
142    AutoFile file{m_filter_fileseq->Open(pos)};
143    if (file.IsNull()) {
144        LogError("Failed to open filter file %d", pos.nFile);
145        return false;
146    }
147    if (!file.Commit()) {
148        LogError("Failed to commit filter file %d", pos.nFile);
149        (void)file.fclose();
150        return false;
151    }
152    if (file.fclose() != 0) {
153        LogError("Failed to close filter file %d after commit: %s", pos.nFile, SysErrorString(errno));
154        return false;
155    }
156
157    batch.Write(DB_FILTER_POS, pos);
158    return true;
159}
160
161bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const
162{
163    AutoFile filein{m_filter_fileseq->Open(pos, true)};
164    if (filein.IsNull()) {
165        return false;
166    }
167
168    // Check that the hash of the encoded_filter matches the one stored in the db.
169    uint256 block_hash;
170    std::vector<uint8_t> encoded_filter;
171    try {
172        filein >> block_hash >> encoded_filter;
173        if (Hash(encoded_filter) != hash) {
174            LogError("Checksum mismatch in filter decode.");
175            return false;
176        }
177        filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter), /*skip_decode_check=*/true);
178    }
179    catch (const std::exception& e) {
180        LogError("Failed to deserialize block filter from disk: %s", e.what());
181        return false;
182    }
183
184    return true;
185}
186
187size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter)
188{
189    assert(filter.GetFilterType() == GetFilterType());
190
191    uint64_t data_size{
192        GetSerializeSize(filter.GetBlockHash()) +
193        GetSerializeSize(filter.GetEncodedFilter())};
194
195    // If writing the filter would overflow the file, flush and move to the next one.
196    if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) {
197        AutoFile last_file{m_filter_fileseq->Open(pos)};
198        if (last_file.IsNull()) {
199            LogError("Failed to open filter file %d", pos.nFile);
200            return 0;
201        }
202        if (!last_file.Truncate(pos.nPos)) {
203            LogError("Failed to truncate filter file %d", pos.nFile);
204            return 0;
205        }
206        if (!last_file.Commit()) {
207            LogError("Failed to commit filter file %d", pos.nFile);
208            (void)last_file.fclose();
209            return 0;
210        }
211        if (last_file.fclose() != 0) {
212            LogError("Failed to close filter file %d after commit: %s", pos.nFile, SysErrorString(errno));
213            return 0;
214        }
215
216        pos.nFile++;
217        pos.nPos = 0;
218    }
219
220    // Pre-allocate sufficient space for filter data.
221    bool out_of_space;
222    m_filter_fileseq->Allocate(pos, data_size, out_of_space);
223    if (out_of_space) {
224        LogError("out of disk space");
225        return 0;
226    }
227
228    AutoFile fileout{m_filter_fileseq->Open(pos)};
229    if (fileout.IsNull()) {
230        LogError("Failed to open filter file %d", pos.nFile);
231        return 0;
232    }
233
234    fileout << filter.GetBlockHash() << filter.GetEncodedFilter();
235
236    if (fileout.fclose() != 0) {
237        LogError("Failed to close filter file %d: %s", pos.nFile, SysErrorString(errno));
238        return 0;
239    }
240
241    return data_size;
242}
243
244std::optional<uint256> BlockFilterIndex::ReadFilterHeader(int height, const uint256& expected_block_hash)
245{
246    std::pair<uint256, DBVal> read_out;
247    if (!m_db->Read(index_util::DBHeightKey(height), read_out)) {
248        return std::nullopt;
249    }
250