How objects become bytes

501struct VarIntFormatter
502{
503    template<typename Stream, typename I> void Ser(Stream &s, I v)
504    {
505        WriteVarInt<Stream,Mode, std::remove_cv_t<I>>(s, v);
506    }
507
508    template<typename Stream, typename I> void Unser(Stream& s, I& v)
509    {
510        v = ReadVarInt<Stream,Mode, std::remove_cv_t<I>>(s);
511    }
512};
513
514/** Serialization wrapper class for custom integers and enums.
515 *
516 * It permits specifying the serialized size (1 to 8 bytes) and endianness.
517 *
518 * Use the big endian mode for values that are stored in memory in native
519 * byte order, but serialized in big endian notation. This is only intended
520 * to implement serializers that are compatible with existing formats, and
521 * its use is not recommended for new data structures.
522 */
523template<int Bytes, bool BigEndian = false>
524struct CustomUintFormatter
525{
526    static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
527    static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
528
529    template <typename Stream, typename I> void Ser(Stream& s, I v)
530    {
531        if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
532        if (BigEndian) {
533            uint64_t raw = htobe64_internal(v);
534            s.write(std::as_bytes(std::span{&raw, 1}).last(Bytes));
535        } else {
536            uint64_t raw = htole64_internal(v);
537            s.write(std::as_bytes(std::span{&raw, 1}).first(Bytes));
538        }
539    }
540
541    template <typename Stream, typename I> void Unser(Stream& s, I& v)
542    {
543        using U = typename std::conditional_t<std::is_enum_v<I>, std::underlying_type<I>, std::common_type<I>>::type;
544        static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small");
545        uint64_t raw = 0;
546        if (BigEndian) {
547            s.read(std::as_writable_bytes(std::span{&raw, 1}).last(Bytes));
548            v = static_cast<I>(be64toh_internal(raw));
549        } else {
550            s.read(std::as_writable_bytes(std::span{&raw, 1}).first(Bytes));
551            v = static_cast<I>(le64toh_internal(raw));
552        }
553    }
554};
555
556template<int Bytes> using BigEndianFormatter = CustomUintFormatter<Bytes, true>;
557
558/** Formatter for integers in CompactSize format. */
559template<bool RangeCheck>
560struct CompactSizeFormatter
561{
562    template<typename Stream, typename I>
563    void Unser(Stream& s, I& v)
564    {
565        uint64_t n = ReadCompactSize<Stream>(s, RangeCheck);
566        if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
567            throw std::ios_base::failure("CompactSize exceeds limit of type");
568        }
569        v = n;
570    }
571
572    template<typename Stream, typename I>
573    void Ser(Stream& s, I v)
574    {
575        static_assert(std::is_unsigned_v<I>, "CompactSize only supported for unsigned integers");
576        static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
577
578        WriteCompactSize<Stream>(s, v);
579    }
580};
581
582template <typename U, bool LOSSY = false>
583struct ChronoFormatter {
584    template <typename Stream, typename Tp>
585    void Unser(Stream& s, Tp& tp)
586    {
587        U u;
588        s >> u;
589        // Lossy deserialization does not make sense, so force Wnarrowing
590        tp = Tp{typename Tp::duration{typename Tp::duration::rep{u}}};
591    }
592    template <typename Stream, typename Tp>
593    void Ser(Stream& s, Tp tp)
594    {
595        if constexpr (LOSSY) {
596            s << U(tp.time_since_epoch().count());
597        } else {
598            s << U{tp.time_since_epoch().count()};
599        }
600    }
601};
602template <typename U>
603using LossyChronoFormatter = ChronoFormatter<U, true>;
604
605class CompactSizeReader
606{
607protected:
608    uint64_t& n;
609public:
610    explicit CompactSizeReader(uint64_t& n_in) : n(n_in) {}
611
612    template<typename Stream>
613    void Unserialize(Stream &s) const {
614        n = ReadCompactSize<Stream>(s);
615    }
616};
617
618class CompactSizeWriter
619{
620protected:
621    uint64_t n;
622public:
623    explicit CompactSizeWriter(uint64_t n_in) : n(n_in) { }
624
625    template<typename Stream>
626    void Serialize(Stream &s) const {
627        WriteCompactSize<Stream>(s, n);
628    }
629};
630
631template<size_t Limit>
632struct LimitedStringFormatter
633{
634    template<typename Stream>
635    void Unser(Stream& s, std::string& v)
636    {
637        size_t size = ReadCompactSize(s);
638        if (size > Limit) {
639            throw std::ios_base::failure("String length limit exceeded");
640        }
641        v.resize(size);
642        if (size != 0) s.read(MakeWritableByteSpan(v));
643    }
644
645    template<typename Stream>
646    void Ser(Stream& s, const std::string& v)
647    {
648        s << v;
649    }
650};
651
652/** Formatter to serialize/deserialize vector elements using another formatter
653 *
654 * Example:
655 *   struct X {
656 *     std::vector<uint64_t> v;
657 *     SERIALIZE_METHODS(X, obj) { READWRITE(Using<VectorFormatter<VarInt>>(obj.v)); }
658 *   };
659 * will define a struct that contains a vector of uint64_t, which is serialized
660 * as a vector of VarInt-encoded integers.
661 *
662 * V is not required to be an std::vector type. It works for any class that
663 * exposes a value_type, size, reserve, emplace_back, back, and const iterators.
664 */
665template<class Formatter>
666struct VectorFormatter
667{
668    template<typename Stream, typename V>
669    void Ser(Stream& s, const V& v)
670    {
671        Formatter formatter;
672        WriteCompactSize(s, v.size());
673        for (const typename V::value_type& elem : v) {
674            formatter.Ser(s, elem);
675        }
676    }
677
678    template<typename Stream, typename V>
679    void Unser(Stream& s, V& v)
680    {
681        Formatter formatter;
682        v.clear();
683        size_t size = ReadCompactSize(s);
684        size_t allocated = 0;
685        while (allocated < size) {
686            // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
687            // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
688            // X MiB of data to make us allocate X+5 Mib.
689            static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
690            allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
691            v.reserve(allocated);
692            while (v.size() < allocated) {
693                v.emplace_back();
694                formatter.Unser(s, v.back());
695            }
696        }
697    };
698};
699
700/**
701 * Forward declarations
702 */
703
704/**
705 *  string
706 */
707template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str);
708template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str);
709
710/**
711 *  string_view
712 */
713template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string_view<C>& str);
714template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string_view<C>& str) = delete;
715
716/**
717 * prevector
718 */
719template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v);
720template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v);
721
722/**
723 * vector
724 */
725template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v);
726template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v);
727
728/**
729 * pair
730 */
731template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item);
732template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item);
733
734/**
735 * map
736 */
737template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m);
738template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m);
739
740/**
741 * set
742 */
743template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m);
744template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m);
745
746/**
747 * shared_ptr
748 */
749template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p);
750template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p);