How objects become bytes

1// Copyright (c) 2009-2010 Satoshi Nakamoto
2// Copyright (c) 2009-present The Bitcoin Core developers
3// Distributed under the MIT software license, see the accompanying
4// file COPYING or http://www.opensource.org/licenses/mit-license.php.
5
6#ifndef BITCOIN_SERIALIZE_H
7#define BITCOIN_SERIALIZE_H
8
9#include <attributes.h>
10#include <compat/assumptions.h> // IWYU pragma: keep
11#include <compat/endian.h>
12#include <prevector.h>
13#include <span.h>
14#include <util/overflow.h>
15
16#include <algorithm>
17#include <concepts>
18#include <cstdint>
19#include <cstring>
20#include <ios>
21#include <limits>
22#include <map>
23#include <memory>
24#include <set>
25#include <span>
26#include <string>
27#include <string_view>
28#include <utility>
29#include <vector>
30
31/**
32 * The maximum size of a serialized object in bytes or number of elements
33 * (for eg vectors) when the size is encoded as CompactSize.
34 */
35inline constexpr uint64_t MAX_SIZE = 0x02000000;
36
37/** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */
38inline constexpr unsigned int MAX_VECTOR_ALLOCATE{5'000'000};
39
40/**
41 * Dummy data type to identify deserializing constructors.
42 *
43 * By convention, a constructor of a type T with signature
44 *
45 *   template <typename Stream> T::T(deserialize_type, Stream& s)
46 *
47 * is a deserializing constructor, which builds the type by
48 * deserializing it from s. If T contains const fields, this
49 * is likely the only way to do so.
50 */
51struct deserialize_type {};
52inline constexpr deserialize_type deserialize {};
53
54/*
55 * Lowest-level serialization and conversion.
56 */
57template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
58{
59    s.write(std::as_bytes(std::span{&obj, 1}));
60}
61template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
62{
63    obj = htole16_internal(obj);
64    s.write(std::as_bytes(std::span{&obj, 1}));
65}
66template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
67{
68    obj = htole32_internal(obj);
69    s.write(std::as_bytes(std::span{&obj, 1}));
70}
71template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj)
72{
73    obj = htobe32_internal(obj);
74    s.write(std::as_bytes(std::span{&obj, 1}));
75}
76template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
77{
78    obj = htole64_internal(obj);
79    s.write(std::as_bytes(std::span{&obj, 1}));
80}
81template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
82{
83    uint8_t obj;
84    s.read(std::as_writable_bytes(std::span{&obj, 1}));
85    return obj;
86}
87template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
88{
89    uint16_t obj;
90    s.read(std::as_writable_bytes(std::span{&obj, 1}));
91    return le16toh_internal(obj);
92}
93template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
94{
95    uint32_t obj;
96    s.read(std::as_writable_bytes(std::span{&obj, 1}));
97    return le32toh_internal(obj);
98}
99template<typename Stream> inline uint32_t ser_readdata32be(Stream &s)
100{
101    uint32_t obj;
102    s.read(std::as_writable_bytes(std::span{&obj, 1}));
103    return be32toh_internal(obj);
104}
105template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
106{
107    uint64_t obj;
108    s.read(std::as_writable_bytes(std::span{&obj, 1}));
109    return le64toh_internal(obj);
110}
111
112
113class SizeComputer;
114
115/**
116 * Convert any argument to a reference to X, maintaining constness.
117 *
118 * This can be used in serialization code to invoke a base class's
119 * serialization routines.
120 *
121 * Example use:
122 *   class Base { ... };
123 *   class Child : public Base {
124 *     int m_data;
125 *   public:
126 *     SERIALIZE_METHODS(Child, obj) {
127 *       READWRITE(AsBase<Base>(obj), obj.m_data);
128 *     }
129 *   };
130 *
131 * static_cast cannot easily be used here, as the type of Obj will be const Child&
132 * during serialization and Child& during deserialization. AsBase will convert to
133 * const Base& and Base& appropriately.
134 */
135template <class Out, class In>
136Out& AsBase(In& x)
137{
138    static_assert(std::is_base_of_v<Out, In>);
139    return x;
140}
141template <class Out, class In>
142const Out& AsBase(const In& x)
143{
144    static_assert(std::is_base_of_v<Out, In>);
145    return x;
146}
147
148#define READWRITE(...) (ser_action.SerReadWriteMany(s, __VA_ARGS__))
149#define SER_READ(obj, code) ser_action.SerRead(s, obj, [&](Stream& s, std::remove_const_t<Type>& obj) { code; })
150#define SER_WRITE(obj, code) ser_action.SerWrite(s, obj, [&](Stream& s, const Type& obj) { code; })
151
152/**
153 * Implement the Ser and Unser methods needed for implementing a formatter (see Using below).
154 *
155 * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic
156 * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when
157 * deserializing).
158 *
159 * Example use:
160 *   struct FooFormatter {
161 *     FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); }
162 *   }
163 *   would define a class FooFormatter that defines a serialization of Class objects consisting
164 *   of serializing its val1 member using the default serialization, and its val2 member using
165 *   VARINT serialization. That FooFormatter can then be used in statements like
166 *   READWRITE(Using<FooFormatter>(obj.bla)).
167 */
168#define FORMATTER_METHODS(cls, obj) \
169    template<typename Stream> \
170    static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, ActionSerialize{}); } \
171    template<typename Stream> \
172    static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, ActionUnserialize{}); } \
173    template<typename Stream, typename Type, typename Operation> \
174    static void SerializationOps(Type& obj, Stream& s, Operation ser_action)
175
176/**
177 * Formatter methods can retrieve parameters attached to a stream using the
178 * SER_PARAMS(type) macro as long as the stream is created directly or
179 * indirectly with a parameter of that type. This permits making serialization
180 * depend on run-time context in a type-safe way.
181 *
182 * Example use:
183 *   struct BarParameter { bool fancy; ... };
184 *   struct Bar { ... };
185 *   struct FooFormatter {
186 *     FORMATTER_METHODS(Bar, obj) {
187 *       auto& param = SER_PARAMS(BarParameter);
188 *       if (param.fancy) {
189 *         READWRITE(VARINT(obj.value));
190 *       } else {
191 *         READWRITE(obj.value);
192 *       }
193 *     }
194 *   };
195 * which would then be invoked as
196 *   READWRITE(BarParameter{...}(Using<FooFormatter>(obj.foo)))
197 *
198 * parameter(obj) can be invoked anywhere in the call stack; it is
199 * passed down recursively into all serialization code, until another
200 * serialization parameter overrides it.
201 *
202 * Parameters will be implicitly converted where appropriate. This means that
203 * "parent" serialization code can use a parameter that derives from, or is
204 * convertible to, a "child" formatter's parameter type.
205 *
206 * Compilation will fail in any context where serialization is invoked but
207 * no parameter of a type convertible to BarParameter is provided.
208 */
209#define SER_PARAMS(type) (s.template GetParams<type>())
210
211#define BASE_SERIALIZE_METHODS(cls)                                                                 \
212    template <typename Stream>                                                                      \
213    void Serialize(Stream& s) const                                                                 \
214    {                                                                                               \
215        static_assert(std::is_same_v<const cls&, decltype(*this)>, "Serialize type mismatch");      \
216        Ser(s, *this);                                                                              \
217    }                                                                                               \
218    template <typename Stream>                                                                      \
219    void Unserialize(Stream& s)                                                                     \
220    {                                                                                               \
221        static_assert(std::is_same_v<cls&, decltype(*this)>, "Unserialize type mismatch");          \
222        Unser(s, *this);                                                                            \
223    }
224
225/**
226 * Implement the Serialize and Unserialize methods by delegating to a single templated
227 * static method that takes the to-be-(de)serialized object as a parameter. This approach
228 * has the advantage that the constness of the object becomes a template parameter, and
229 * thus allows a single implementation that sees the object as const for serializing
230 * and non-const for deserializing, without casts.
231 */
232#define SERIALIZE_METHODS(cls, obj) \
233    BASE_SERIALIZE_METHODS(cls)     \
234    FORMATTER_METHODS(cls, obj)
235
236// Templates for serializing to anything that looks like a stream,
237// i.e. anything that supports .read(std::span<std::byte>) and .write(std::span<const std::byte>)
238//
239
240// Typically int8_t and char are distinct types, but some systems may define int8_t
241// in terms of char. Forbid serialization of char in the typical case, but allow it if
242// it's the only way to describe an int8_t.
243template<class T>
244concept CharNotInt8 = std::same_as<T, char> && !std::same_as<T, int8_t>;
245
246// clang-format off
247template <typename Stream, CharNotInt8 V> void Serialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t
248template <typename Stream> void Serialize(Stream& s, std::byte a) { ser_writedata8(s, uint8_t(a)); }
249template <typename Stream> void Serialize(Stream& s, int8_t a)    { ser_writedata8(s, uint8_t(a)); }
250template <typename Stream> void Serialize(Stream& s, uint8_t a)   { ser_writedata8(s, a); }