Reading legacy Berkeley DB files for migration

501            }
502            s.ignore(to_jump);
503
504            // Read the record
505            RecordHeader rec_hdr(m_header.other_endian);
506            s >> rec_hdr;
507            to_jump += RecordHeader::SIZE;
508
509            if (rec_hdr.type != RecordType::KEYDATA) {
510                throw std::runtime_error("Unknown record type in internal page");
511            }
512            InternalRecord record(rec_hdr);
513            s >> record;
514            records.emplace_back(record);
515            to_jump += InternalRecord::FIXED_SIZE + rec_hdr.len;
516
517            // Go back to the indexes
518            s.seek(-to_jump, SEEK_CUR);
519        }
520    }
521};
522
523static void SeekToPage(AutoFile& s, uint32_t page_num, uint32_t page_size)
524{
525    int64_t pos = int64_t{page_num} * page_size;
526    s.seek(pos, SEEK_SET);
527}
528
529void BerkeleyRODatabase::Open()
530{
531    // Open the file
532    FILE* file = fsbridge::fopen(m_filepath, "rb");
533    AutoFile db_file(file);
534    if (db_file.IsNull()) {
535        throw std::runtime_error("BerkeleyRODatabase: Failed to open database file");
536    }
537
538    uint32_t page_size = 4096; // Default page size
539
540    // Read the outer metapage
541    // Expected page number is 0
542    MetaPage outer_meta(0);
543    db_file >> outer_meta;
544    page_size = outer_meta.pagesize;
545
546    // Verify the size of the file is a multiple of the page size
547    const int64_t size{db_file.size()};
548
549    // Since BDB stores everything in a page, the file size should be a multiple of the page size;
550    // However, BDB doesn't actually check that this is the case, and enforcing this check results
551    // in us rejecting a database that BDB would not, so this check needs to be excluded.
552    // This is left commented out as a reminder to not accidentally implement this in the future.
553    // if (size % page_size != 0) {
554    //     throw std::runtime_error("File size is not a multiple of page size");
555    // }
556
557    // Check the last page number
558    uint32_t expected_last_page{uint32_t((size / page_size) - 1)};
559    if (outer_meta.last_page != expected_last_page) {
560        throw std::runtime_error("Last page number could not fit in file");
561    }
562
563    // Make sure encryption is disabled
564    if (outer_meta.encrypt_algo != 0) {
565        throw std::runtime_error("BDB builtin encryption is not supported");
566    }
567
568    // Check all Log Sequence Numbers (LSN) point to file 0 and offset 1 which indicates that the LSNs were
569    // reset and that the log files are not necessary to get all of the data in the database.
570    for (uint32_t i = 0; i <= outer_meta.last_page; ++i) {
571        // The LSN is composed of 2 32-bit ints, the first is a file id, the second an offset
572        // It will always be the first 8 bytes of a page, so we deserialize it directly for every page
573        uint32_t file;
574        uint32_t offset;
575        SeekToPage(db_file, i, page_size);
576        db_file >> file >> offset;
577        if (outer_meta.other_endian) {
578            file = internal_bswap_32(file);
579            offset = internal_bswap_32(offset);
580        }
581        if (file != 0 || offset != 1) {
582            throw std::runtime_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support");
583        }
584    }
585
586    // Read the root page
587    SeekToPage(db_file, outer_meta.root, page_size);
588    PageHeader header(outer_meta.root, outer_meta.other_endian);
589    db_file >> header;
590    if (header.type != PageType::BTREE_LEAF) {
591        throw std::runtime_error("Unexpected outer database root page type");
592    }
593    if (header.entries != 2) {
594        throw std::runtime_error("Unexpected number of entries in outer database root page");
595    }
596    RecordsPage page(header);
597    db_file >> page;
598
599    // First record should be the string "main"
600    if (!std::holds_alternative<DataRecord>(page.records.at(0)) || std::get<DataRecord>(page.records.at(0)).data != SUBDATABASE_NAME) {
601        throw std::runtime_error("Subdatabase has an unexpected name");
602    }
603    // Check length of page number for subdatabase location
604    if (!std::holds_alternative<DataRecord>(page.records.at(1)) || std::get<DataRecord>(page.records.at(1)).m_header.len != 4) {
605        throw std::runtime_error("Subdatabase page number has unexpected length");
606    }
607
608    // Read subdatabase page number
609    // It is written as a big endian 32 bit number
610    uint32_t main_db_page = ReadBE32(std::get<DataRecord>(page.records.at(1)).data.data());
611
612    // The main database is in a page that doesn't exist
613    if (main_db_page > outer_meta.last_page) {
614        throw std::runtime_error("Page number is greater than database last page");
615    }
616
617    // Read the inner metapage
618    SeekToPage(db_file, main_db_page, page_size);
619    MetaPage inner_meta(main_db_page);
620    db_file >> inner_meta;
621
622    if (inner_meta.pagesize != page_size) {
623        throw std::runtime_error("Unexpected page size");
624    }
625
626    if (inner_meta.last_page > outer_meta.last_page) {
627        throw std::runtime_error("Subdatabase last page is greater than database last page");
628    }
629    uint64_t max_data_size = static_cast<uint64_t>(outer_meta.last_page) * page_size;
630
631    // Make sure encryption is disabled
632    if (inner_meta.encrypt_algo != 0) {
633        throw std::runtime_error("BDB builtin encryption is not supported");
634    }
635
636    // Read the root's level from its header
637    // Note that we will read the root page twice in order to process it.
638    SeekToPage(db_file, inner_meta.root, page_size);
639    PageHeader root_header(inner_meta.root, inner_meta.other_endian);
640    db_file >> root_header;
641
642    // Do a DFS through the BTree, starting at root
643    // We track the expected level of each page in order to avoid loops
644    std::vector<std::pair<uint32_t, uint32_t>> pages{{inner_meta.root, root_header.level}};
645    while (pages.size() > 0) {
646        auto [curr_page, expected_level] = pages.back();
647        // It turns out BDB completely ignores this last_page field and doesn't actually update it to the correct
648        // last page. While we should be checking this, we can't.
649        // This is left commented out as a reminder to not accidentally implement this in the future.
650        // if (curr_page > inner_meta.last_page) {
651        //     throw std::runtime_error("Page number is greater than subdatabase last page");
652        // }
653        pages.pop_back();
654        SeekToPage(db_file, curr_page, page_size);
655        PageHeader header(curr_page, inner_meta.other_endian);
656        db_file >> header;
657        if (header.level != expected_level) {
658            throw std::runtime_error("BTree page has an unexpected level");
659        }
660        switch (header.type) {
661        case PageType::BTREE_INTERNAL: {
662            InternalPage int_page(header);
663            db_file >> int_page;
664            for (const InternalRecord& rec : int_page.records) {
665                if (rec.m_header.deleted) continue;
666                pages.emplace_back(rec.page_num, header.level - 1);
667            }
668            break;
669        }
670        case PageType::BTREE_LEAF: {
671            if (header.level != 1) {
672                throw std::runtime_error("BTree Leaf page is not at level 1");
673            }
674            RecordsPage rec_page(header);
675            db_file >> rec_page;
676            if (rec_page.records.size() % 2 != 0) {
677                // BDB stores key value pairs in consecutive records, thus an odd number of records is unexpected
678                throw std::runtime_error("Records page has odd number of records");
679            }
680            bool is_key = true;
681            std::vector<std::byte> key;
682            for (const std::variant<DataRecord, OverflowRecord>& rec : rec_page.records) {
683                std::vector<std::byte> data;
684                if (const DataRecord* drec = std::get_if<DataRecord>(&rec)) {
685                    if (drec->m_header.deleted) continue;
686                    data = drec->data;
687                } else if (const OverflowRecord* orec = std::get_if<OverflowRecord>(&rec)) {
688                    if (orec->m_header.deleted) continue;
689                    uint32_t next_page = orec->page_number;
690                    if (orec->item_len > max_data_size) {
691                        throw std::runtime_error("Overflow record has an impossible length");
692                    }
693                    while (next_page != 0) {
694                        SeekToPage(db_file, next_page, page_size);
695                        PageHeader opage_header(next_page, inner_meta.other_endian);
696                        db_file >> opage_header;
697                        if (opage_header.type != PageType::OVERFLOW_DATA) {
698                            throw std::runtime_error("Bad overflow record page type");
699                        }
700                        OverflowPage opage(opage_header);
701                        db_file >> opage;
702                        data.insert(data.end(), opage.data.begin(), opage.data.end());
703                        if (data.size() > orec->item_len) {
704                            throw std::runtime_error("Overflow record data is larger than stated size");
705                        }
706                        next_page = opage_header.next_page;
707                    }
708                }
709
710                if (is_key) {
711                    key = data;
712                } else {
713                    m_records.emplace(SerializeData{key.begin(), key.end()}, SerializeData{data.begin(), data.end()});
714                    key.clear();
715                }
716                is_key = !is_key;
717            }
718            break;
719        }
720        default:
721            throw std::runtime_error("Unexpected page type");
722        }
723    }
724}
725
726std::unique_ptr<DatabaseBatch> BerkeleyRODatabase::MakeBatch()
727{
728    return std::make_unique<BerkeleyROBatch>(*this);
729}
730
731bool BerkeleyRODatabase::Backup(const std::string& dest) const
732{
733    fs::path src(m_filepath);
734    fs::path dst(fs::PathFromString(dest));
735
736    if (fs::is_directory(dst)) {
737        dst = BDBDataFile(dst);
738    }
739    try {
740        if (fs::exists(dst) && fs::equivalent(src, dst)) {
741            LogWarning("cannot backup to wallet source file %s", fs::PathToString(dst));
742            return false;
743        }
744
745        fs::copy_file(src, dst, fs::copy_options::overwrite_existing);
746        LogInfo("copied %s to %s\n", fs::PathToString(m_filepath), fs::PathToString(dst));
747        return true;
748    } catch (const fs::filesystem_error& e) {
749        LogWarning("error copying %s to %s - %s\n", fs::PathToString(m_filepath), fs::PathToString(dst), e.code().message());
750        return false;