Reading legacy Berkeley DB files for migration

251        data.resize(m_header.len);
252        s.read(std::as_writable_bytes(std::span(data.data(), data.size())));
253    }
254};
255
256/** Class for records representing internal nodes of the BTree. */
257class InternalRecord
258{
259public:
260    InternalRecord(const RecordHeader& header) : m_header(header) {}
261    InternalRecord() = delete;
262
263    RecordHeader m_header;
264
265    uint8_t unused;              // Padding, unused
266    uint32_t page_num;           // Page number of referenced page
267    uint32_t records;            // Subtree record count
268    std::vector<std::byte> data; // Variable length key item
269
270    static constexpr size_t FIXED_SIZE = 9; // Size of fixed data is 9 bytes
271
272    template <typename Stream>
273    void Unserialize(Stream& s)
274    {
275        s >> unused;
276        s >> page_num;
277        s >> records;
278
279        data.resize(m_header.len);
280        s.read(std::as_writable_bytes(std::span(data.data(), data.size())));
281
282        if (m_header.other_endian) {
283            page_num = internal_bswap_32(page_num);
284            records = internal_bswap_32(records);
285        }
286    }
287};
288
289/** Class for records representing overflow records of the BTree.
290 * Overflow records point to a page which contains the data in the record.
291 * Those pages may point to further pages with the rest of the data if it does not fit
292 * in one page */
293class OverflowRecord
294{
295public:
296    OverflowRecord(const RecordHeader& header) : m_header(header) {}
297    OverflowRecord() = delete;
298
299    RecordHeader m_header;
300
301    uint8_t unused2;      // Padding, unused
302    uint32_t page_number; // Page number where data begins
303    uint32_t item_len;    // Total length of item
304
305    static constexpr size_t SIZE = 9; // Overflow record is always 9 bytes
306
307    template <typename Stream>
308    void Unserialize(Stream& s)
309    {
310        s >> unused2;
311        s >> page_number;
312        s >> item_len;
313
314        if (m_header.other_endian) {
315            page_number = internal_bswap_32(page_number);
316            item_len = internal_bswap_32(item_len);
317        }
318    }
319};
320
321/** A generic data page in the database. Contains fields common to all data pages. */
322class PageHeader
323{
324public:
325    uint32_t lsn_file;   // Log Sequence Number file
326    uint32_t lsn_offset; // Log Sequence Number offset
327    uint32_t page_num;   // Current page number
328    uint32_t prev_page;  // Previous page number
329    uint32_t next_page;  // Next page number
330    uint16_t entries;    // Number of items on the page
331    uint16_t hf_offset;  // High free byte page offset
332    uint8_t level;       // Btree page level
333    PageType type;       // Page type
334
335    static constexpr int64_t SIZE = 26; // The header is 26 bytes
336
337    uint32_t expected_page_num;
338    bool other_endian;
339
340    PageHeader(uint32_t page_num, bool other_endian) : expected_page_num(page_num), other_endian(other_endian) {}
341    PageHeader() = delete;
342
343    template <typename Stream>
344    void Unserialize(Stream& s)
345    {
346        s >> lsn_file;
347        s >> lsn_offset;
348        s >> page_num;
349        s >> prev_page;
350        s >> next_page;
351        s >> entries;
352        s >> hf_offset;
353        s >> level;
354
355        uint8_t uint8_type;
356        s >> uint8_type;
357        type = static_cast<PageType>(uint8_type);
358
359        if (other_endian) {
360            lsn_file = internal_bswap_32(lsn_file);
361            lsn_offset = internal_bswap_32(lsn_offset);
362            page_num = internal_bswap_32(page_num);
363            prev_page = internal_bswap_32(prev_page);
364            next_page = internal_bswap_32(next_page);
365            entries = internal_bswap_16(entries);
366            hf_offset = internal_bswap_16(hf_offset);
367        }
368
369        if (expected_page_num != page_num) {
370            throw std::runtime_error("Page number mismatch");
371        }
372        if ((type != PageType::OVERFLOW_DATA && level < 1) || (type == PageType::OVERFLOW_DATA && level != 0)) {
373            throw std::runtime_error("Bad btree level");
374        }
375    }
376};
377
378/** A page of records in the database */
379class RecordsPage
380{
381public:
382    RecordsPage(const PageHeader& header) : m_header(header) {}
383    RecordsPage() = delete;
384
385    PageHeader m_header;
386
387    std::vector<uint16_t> indexes;
388    std::vector<std::variant<DataRecord, OverflowRecord>> records;
389
390    template <typename Stream>
391    void Unserialize(Stream& s)
392    {
393        // Current position within the page
394        int64_t pos = PageHeader::SIZE;
395
396        // Get the items
397        for (uint32_t i = 0; i < m_header.entries; ++i) {
398            // Get the index
399            uint16_t index;
400            s >> index;
401            if (m_header.other_endian) {
402                index = internal_bswap_16(index);
403            }
404            indexes.push_back(index);
405            pos += sizeof(uint16_t);
406
407            // Go to the offset from the index
408            int64_t to_jump = index - pos;
409            if (to_jump < 0) {
410                throw std::runtime_error("Data record position not in page");
411            }
412            s.ignore(to_jump);
413
414            // Read the record
415            RecordHeader rec_hdr(m_header.other_endian);
416            s >> rec_hdr;
417            to_jump += RecordHeader::SIZE;
418
419            switch (rec_hdr.type) {
420            case RecordType::KEYDATA: {
421                DataRecord record(rec_hdr);
422                s >> record;
423                records.emplace_back(record);
424                to_jump += rec_hdr.len;
425                break;
426            }
427            case RecordType::OVERFLOW_DATA: {
428                OverflowRecord record(rec_hdr);
429                s >> record;
430                records.emplace_back(record);
431                to_jump += OverflowRecord::SIZE;
432                break;
433            }
434            default:
435                throw std::runtime_error("Unknown record type in records page");
436            }
437
438            // Go back to the indexes
439            s.seek(-to_jump, SEEK_CUR);
440        }
441    }
442};
443
444/** A page containing overflow data */
445class OverflowPage
446{
447public:
448    OverflowPage(const PageHeader& header) : m_header(header) {}
449    OverflowPage() = delete;
450
451    PageHeader m_header;
452
453    // BDB overloads some page fields to store overflow page data
454    // hf_offset contains the length of the overflow data stored on this page
455    // entries contains a reference count for references to this item
456
457    // The overflow data itself. Begins immediately following header
458    std::vector<std::byte> data;
459
460    template <typename Stream>
461    void Unserialize(Stream& s)
462    {
463        data.resize(m_header.hf_offset);
464        s.read(std::as_writable_bytes(std::span(data.data(), data.size())));
465    }
466};
467
468/** A page of records in the database */
469class InternalPage
470{
471public:
472    InternalPage(const PageHeader& header) : m_header(header) {}
473    InternalPage() = delete;
474
475    PageHeader m_header;
476
477    std::vector<uint16_t> indexes;
478    std::vector<InternalRecord> records;
479
480    template <typename Stream>
481    void Unserialize(Stream& s)
482    {
483        // Current position within the page
484        int64_t pos = PageHeader::SIZE;
485
486        // Get the items
487        for (uint32_t i = 0; i < m_header.entries; ++i) {
488            // Get the index
489            uint16_t index;
490            s >> index;
491            if (m_header.other_endian) {
492                index = internal_bswap_16(index);
493            }
494            indexes.push_back(index);
495            pos += sizeof(uint16_t);
496
497            // Go to the offset from the index
498            int64_t to_jump = index - pos;
499            if (to_jump < 0) {
500                throw std::runtime_error("Internal record position not in page");