How objects become bytes

251template <typename Stream> void Serialize(Stream& s, int16_t a)   { ser_writedata16(s, uint16_t(a)); }
252template <typename Stream> void Serialize(Stream& s, uint16_t a)  { ser_writedata16(s, a); }
253template <typename Stream> void Serialize(Stream& s, int32_t a)   { ser_writedata32(s, uint32_t(a)); }
254template <typename Stream> void Serialize(Stream& s, uint32_t a)  { ser_writedata32(s, a); }
255template <typename Stream> void Serialize(Stream& s, int64_t a)   { ser_writedata64(s, uint64_t(a)); }
256template <typename Stream> void Serialize(Stream& s, uint64_t a)  { ser_writedata64(s, a); }
257
258template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, const B (&a)[N])           { s.write(MakeByteSpan(a)); }
259template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, const std::array<B, N>& a) { s.write(MakeByteSpan(a)); }
260template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, std::span<B, N> span)      { s.write(std::as_bytes(span)); }
261template <typename Stream, BasicByte B>           void Serialize(Stream& s, std::span<B> span)         { s.write(std::as_bytes(span)); }
262
263template <typename Stream, CharNotInt8 V> void Unserialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t
264template <typename Stream> void Unserialize(Stream& s, std::byte& a) { a = std::byte(ser_readdata8(s)); }
265template <typename Stream> void Unserialize(Stream& s, int8_t& a)    { a = int8_t(ser_readdata8(s)); }
266template <typename Stream> void Unserialize(Stream& s, uint8_t& a)   { a = ser_readdata8(s); }
267template <typename Stream> void Unserialize(Stream& s, int16_t& a)   { a = int16_t(ser_readdata16(s)); }
268template <typename Stream> void Unserialize(Stream& s, uint16_t& a)  { a = ser_readdata16(s); }
269template <typename Stream> void Unserialize(Stream& s, int32_t& a)   { a = int32_t(ser_readdata32(s)); }
270template <typename Stream> void Unserialize(Stream& s, uint32_t& a)  { a = ser_readdata32(s); }
271template <typename Stream> void Unserialize(Stream& s, int64_t& a)   { a = int64_t(ser_readdata64(s)); }
272template <typename Stream> void Unserialize(Stream& s, uint64_t& a)  { a = ser_readdata64(s); }
273
274template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, B (&a)[N])            { s.read(MakeWritableByteSpan(a)); }
275template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, std::array<B, N>& a)  { s.read(MakeWritableByteSpan(a)); }
276template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, std::span<B, N> span) { s.read(std::as_writable_bytes(span)); }
277template <typename Stream, BasicByte B>           void Unserialize(Stream& s, std::span<B> span)    { s.read(std::as_writable_bytes(span)); }
278
279template <typename Stream> void Serialize(Stream& s, bool a)    { uint8_t f = a; ser_writedata8(s, f); }
280template <typename Stream> void Unserialize(Stream& s, bool& a) { uint8_t f = ser_readdata8(s); a = f; }
281// clang-format on
282
283
284/**
285 * Compact Size
286 * size <  253        -- 1 byte
287 * size <= USHRT_MAX  -- 3 bytes  (253 + 2 bytes)
288 * size <= UINT_MAX   -- 5 bytes  (254 + 4 bytes)
289 * size >  UINT_MAX   -- 9 bytes  (255 + 8 bytes)
290 */
291constexpr inline unsigned int GetSizeOfCompactSize(uint64_t nSize)
292{
293    if (nSize < 253)             return sizeof(unsigned char);
294    else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t);
295    else if (nSize <= std::numeric_limits<unsigned int>::max())  return sizeof(unsigned char) + sizeof(unsigned int);
296    else                         return sizeof(unsigned char) + sizeof(uint64_t);
297}
298
299inline void WriteCompactSize(SizeComputer& os, uint64_t nSize);
300
301template<typename Stream>
302void WriteCompactSize(Stream& os, uint64_t nSize)
303{
304    if (nSize < 253)
305    {
306        ser_writedata8(os, nSize);
307    }
308    else if (nSize <= std::numeric_limits<uint16_t>::max())
309    {
310        ser_writedata8(os, 253);
311        ser_writedata16(os, nSize);
312    }
313    else if (nSize <= std::numeric_limits<unsigned int>::max())
314    {
315        ser_writedata8(os, 254);
316        ser_writedata32(os, nSize);
317    }
318    else
319    {
320        ser_writedata8(os, 255);
321        ser_writedata64(os, nSize);
322    }
323    return;
324}
325
326/**
327 * Decode a CompactSize-encoded variable-length integer.
328 *
329 * As these are primarily used to encode the size of vector-like serializations, by default a range
330 * check is performed. When used as a generic number encoding, range_check should be set to false.
331 */
332template<typename Stream>
333uint64_t ReadCompactSize(Stream& is, bool range_check = true)
334{
335    uint8_t chSize = ser_readdata8(is);
336    uint64_t nSizeRet = 0;
337    if (chSize < 253)
338    {
339        nSizeRet = chSize;
340    }
341    else if (chSize == 253)
342    {
343        nSizeRet = ser_readdata16(is);
344        if (nSizeRet < 253)
345            throw std::ios_base::failure("non-canonical ReadCompactSize()");
346    }
347    else if (chSize == 254)
348    {
349        nSizeRet = ser_readdata32(is);
350        if (nSizeRet < 0x10000u)
351            throw std::ios_base::failure("non-canonical ReadCompactSize()");
352    }
353    else
354    {
355        nSizeRet = ser_readdata64(is);
356        if (nSizeRet < 0x100000000ULL)
357            throw std::ios_base::failure("non-canonical ReadCompactSize()");
358    }
359    if (range_check && nSizeRet > MAX_SIZE) {
360        throw std::ios_base::failure("ReadCompactSize(): size too large");
361    }
362    return nSizeRet;
363}
364
365/**
366 * Variable-length integers: bytes are a MSB base-128 encoding of the number.
367 * The high bit in each byte signifies whether another digit follows. To make
368 * sure the encoding is one-to-one, one is subtracted from all but the last digit.
369 * Thus, the byte sequence a[] with length len, where all but the last byte
370 * has bit 128 set, encodes the number:
371 *
372 *  (a[len-1] & 0x7F) + sum(i=1..len-1, 128^i*((a[len-i-1] & 0x7F)+1))
373 *
374 * Properties:
375 * * Very small (0-127: 1 byte, 128-16511: 2 bytes, 16512-2113663: 3 bytes)
376 * * Every integer has exactly one encoding
377 * * Encoding does not depend on size of original integer type
378 * * No redundancy: every (infinite) byte sequence corresponds to a list
379 *   of encoded integers.
380 *
381 * 0:         [0x00]  256:        [0x81 0x00]
382 * 1:         [0x01]  16383:      [0xFE 0x7F]
383 * 127:       [0x7F]  16384:      [0xFF 0x00]
384 * 128:  [0x80 0x00]  16511:      [0xFF 0x7F]
385 * 255:  [0x80 0x7F]  65535: [0x82 0xFE 0x7F]
386 * 2^32:           [0x8E 0xFE 0xFE 0xFF 0x00]
387 */
388
389/**
390 * Mode for encoding VarInts.
391 *
392 * Currently there is no support for signed encodings. The default mode will not
393 * compile with signed values, and the legacy "nonnegative signed" mode will
394 * accept signed values, but improperly encode and decode them if they are
395 * negative. In the future, the DEFAULT mode could be extended to support
396 * negative numbers in a backwards compatible way, and additional modes could be
397 * added to support different varint formats (e.g. zigzag encoding).
398 */
399enum class VarIntMode { DEFAULT, NONNEGATIVE_SIGNED };
400
401template <VarIntMode Mode, typename I>
402struct CheckVarIntMode {
403    constexpr CheckVarIntMode()
404    {
405        static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned_v<I>, "Unsigned type required with mode DEFAULT.");
406        static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed_v<I>, "Signed type required with mode NONNEGATIVE_SIGNED.");
407    }
408};
409
410template<VarIntMode Mode, typename I>
411inline unsigned int GetSizeOfVarInt(I n)
412{
413    CheckVarIntMode<Mode, I>();
414    int nRet = 0;
415    while(true) {
416        nRet++;
417        if (n <= 0x7F)
418            break;
419        n = (n >> 7) - 1;
420    }
421    return nRet;
422}
423
424template<typename I>
425inline void WriteVarInt(SizeComputer& os, I n);
426
427template<typename Stream, VarIntMode Mode, typename I>
428void WriteVarInt(Stream& os, I n)
429{
430    CheckVarIntMode<Mode, I>();
431    unsigned char tmp[CeilDiv(sizeof(n) * 8, 7u)];
432    int len=0;
433    while(true) {
434        tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
435        if (n <= 0x7F)
436            break;
437        n = (n >> 7) - 1;
438        len++;
439    }
440    do {
441        ser_writedata8(os, tmp[len]);
442    } while(len--);
443}
444
445template<typename Stream, VarIntMode Mode, typename I>
446I ReadVarInt(Stream& is)
447{
448    CheckVarIntMode<Mode, I>();
449    I n = 0;
450    while(true) {
451        unsigned char chData = ser_readdata8(is);
452        if (n > (std::numeric_limits<I>::max() >> 7)) {
453           throw std::ios_base::failure("ReadVarInt(): size too large");
454        }
455        n = (n << 7) | (chData & 0x7F);
456        if (chData & 0x80) {
457            if (n == std::numeric_limits<I>::max()) {
458                throw std::ios_base::failure("ReadVarInt(): size too large");
459            }
460            n++;
461        } else {
462            return n;
463        }
464    }
465}
466
467/** Simple wrapper class to serialize objects using a formatter; used by Using(). */
468template<typename Formatter, typename T>
469class Wrapper
470{
471    static_assert(std::is_lvalue_reference_v<T>, "Wrapper needs an lvalue reference type T");
472protected:
473    T m_object;
474public:
475    explicit Wrapper(T obj) : m_object(obj) {}
476    template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); }
477    template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); }
478};
479
480/** Cause serialization/deserialization of an object to be done using a specified formatter class.
481 *
482 * To use this, you need a class Formatter that has public functions Ser(stream, const object&) for
483 * serialization, and Unser(stream, object&) for deserialization. Serialization routines (inside
484 * READWRITE, or directly with << and >> operators), can then use Using<Formatter>(object).
485 *
486 * This works by constructing a Wrapper<Formatter, T>-wrapped version of object, where T is
487 * const during serialization, and non-const during deserialization, which maintains const
488 * correctness.
489 */
490template<typename Formatter, typename T>
491static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); }
492
493#define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
494#define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
495#define COMPACTSIZE(obj) Using<CompactSizeFormatter<true>>(obj)
496#define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj)
497#define LIMITED_VECTOR(obj,n) Using<LimitedVectorFormatter<n>>(obj)
498
499/** Serialization wrapper class for integers in VarInt format. */
500template<VarIntMode Mode>