251    DataStream& operator<<(const T& obj)
252    {
253        ::Serialize(*this, obj);
254        return (*this);
255    }
256
257    template <typename T>
258    DataStream& operator>>(T&& obj)
259    {
260        ::Unserialize(*this, obj);
261        return (*this);
262    }
263
264    /** Compute total memory usage of this object (own memory + any dynamic memory). */
265    size_t GetMemoryUsage() const noexcept;
266};
267
268// Require empty scratch streams on entry and reset them on exit.
269class ScopedDataStreamUsage
270{
271    DataStream& m_stream;
272
273public:
274    explicit ScopedDataStreamUsage(DataStream& stream) : m_stream{stream} { assert(m_stream.empty()); }
275
276    ScopedDataStreamUsage(const ScopedDataStreamUsage&) = delete;
277    ScopedDataStreamUsage& operator=(const ScopedDataStreamUsage&) = delete;
278
279    ~ScopedDataStreamUsage() { m_stream.clear(); }
280};
281
282template <typename IStream>
283class BitStreamReader
284{
285private:
286    IStream& m_istream;
287
288    /// Buffered byte read in from the input stream. A new byte is read into the
289    /// buffer when m_offset reaches 8.
290    uint8_t m_buffer{0};
291
292    /// Number of high order bits in m_buffer already returned by previous
293    /// Read() calls. The next bit to be returned is at this offset from the
294    /// most significant bit position.
295    int m_offset{8};
296
297public:
298    explicit BitStreamReader(IStream& istream) : m_istream(istream) {}
299
300    /** Read the specified number of bits from the stream. The data is returned
301     * in the nbits least significant bits of a 64-bit uint.
302     */
303    uint64_t Read(int nbits) {
304        if (nbits < 0 || nbits > 64) {
305            throw std::out_of_range("nbits must be between 0 and 64");
306        }
307
308        uint64_t data = 0;
309        while (nbits > 0) {
310            if (m_offset == 8) {
311                m_istream >> m_buffer;
312                m_offset = 0;
313            }
314
315            int bits = std::min(8 - m_offset, nbits);
316            data <<= bits;
317            data |= static_cast<uint8_t>(m_buffer << m_offset) >> (8 - bits);
318            m_offset += bits;
319            nbits -= bits;
320        }
321        return data;
322    }
323};
324
325template <typename OStream>
326class BitStreamWriter
327{
328private:
329    OStream& m_ostream;
330
331    /// Buffered byte waiting to be written to the output stream. The byte is
332    /// written buffer when m_offset reaches 8 or Flush() is called.
333    uint8_t m_buffer{0};
334
335    /// Number of high order bits in m_buffer already written by previous
336    /// Write() calls and not yet flushed to the stream. The next bit to be
337    /// written to is at this offset from the most significant bit position.
338    int m_offset{0};
339
340public:
341    explicit BitStreamWriter(OStream& ostream) : m_ostream(ostream) {}
342
343    ~BitStreamWriter()
344    {
345        Flush();
346    }
347
348    /** Write the nbits least significant bits of a 64-bit int to the output
349     * stream. Data is buffered until it completes an octet.
350     */
351    void Write(uint64_t data, int nbits) {
352        if (nbits < 0 || nbits > 64) {
353            throw std::out_of_range("nbits must be between 0 and 64");
354        }
355
356        while (nbits > 0) {
357            int bits = std::min(8 - m_offset, nbits);
358            m_buffer |= (data << (64 - nbits)) >> (64 - 8 + m_offset);
359            m_offset += bits;
360            nbits -= bits;
361
362            if (m_offset == 8) {
363                Flush();
364            }
365        }
366    }
367
368    /** Flush any unwritten bits to the output stream, padding with 0's to the
369     * next byte boundary.
370     */
371    void Flush() {
372        if (m_offset == 0) {
373            return;
374        }
375
376        m_ostream << m_buffer;
377        m_buffer = 0;
378        m_offset = 0;
379    }
380};
381
382/** Non-refcounted RAII wrapper for FILE*
383 *
384 * Will automatically close the file when it goes out of scope if not null.
385 * If you're returning the file pointer, return file.release().
386 * If you need to close the file early, use autofile.fclose() instead of fclose(underlying_FILE).
387 *
388 * @note If the file has been written to, then the caller must close it
389 * explicitly with the `fclose()` method, check if it returns an error and treat
390 * such an error as if the `write()` method failed. The OS's `fclose(3)` may
391 * fail to flush to disk data that has been previously written, rendering the
392 * file corrupt.
393 */
394class AutoFile
395{
396protected:
397    std::FILE* m_file;
398    Obfuscation m_obfuscation;
399    std::optional<int64_t> m_position;
400    bool m_was_written{false};
401
402public:
403    explicit AutoFile(std::FILE* file, const Obfuscation& obfuscation = {});
404
405    ~AutoFile()
406    {
407        if (m_was_written) {
408            // Callers that wrote to the file must have closed it explicitly
409            // with the fclose() method and checked that the close succeeded.
410            // This is because here in the destructor we have no way to signal
411            // errors from fclose() which, after write, could mean the file is
412            // corrupted and must be handled properly at the call site.
413            // Destructors in C++ cannot signal an error to the callers because
414            // they do not return a value and are not allowed to throw exceptions.
415            Assume(IsNull());
416        }
417
418        if (fclose() != 0) {
419            LogError("Failed to close file: %s", SysErrorString(errno));
420        }
421    }
422
423    // Disallow copies
424    AutoFile(const AutoFile&) = delete;
425    AutoFile& operator=(const AutoFile&) = delete;
426
427    bool feof() const { return std::feof(m_file); }
428
429    [[nodiscard]] int fclose()
430    {
431        if (auto rel{release()}) return std::fclose(rel);
432        return 0;
433    }
434
435    /** Get wrapped FILE* with transfer of ownership.
436     * @note This will invalidate the AutoFile object, and makes it the responsibility of the caller
437     * of this function to clean up the returned FILE*.
438     */
439    std::FILE* release()
440    {
441        std::FILE* ret{m_file};
442        m_file = nullptr;
443        return ret;
444    }
445
446    /** Return true if the wrapped FILE* is nullptr, false otherwise.
447     */
448    bool IsNull() const { return m_file == nullptr; }
449
450    /** Continue with a different XOR key */
451    void SetObfuscation(const Obfuscation& obfuscation) { m_obfuscation = obfuscation; }
452
453    /** Implementation detail, only used internally. */
454    std::size_t detail_fread(std::span<std::byte> dst);
455
456    /** Wrapper around fseek(). Will throw if seeking is not possible. */
457    void seek(int64_t offset, int origin);
458
459    /** Find position within the file. Will throw if unknown. */
460    int64_t tell();
461
462    /** Return the size of the file. Will throw if unknown. */
463    int64_t size();
464
465    /** Wrapper around FileCommit(). */
466    bool Commit();
467
468    /** Wrapper around TruncateFile(). */
469    bool Truncate(uint64_t size);
470
471    //! Write a mutable buffer more efficiently than write(), obfuscating the buffer in-place.
472    void write_buffer(std::span<std::byte> src);
473
474    //
475    // Stream subset
476    //
477    void read(std::span<std::byte> dst);
478    void ignore(size_t nSize);
479    void write(std::span<const std::byte> src);
480
481    template <typename T>
482    AutoFile& operator<<(const T& obj)
483    {
484        ::Serialize(*this, obj);
485        return *this;
486    }
487
488    template <typename T>
489    AutoFile& operator>>(T&& obj)
490    {
491        ::Unserialize(*this, obj);
492        return *this;
493    }
494};
495
496using DataBuffer = std::vector<std::byte>;
497
498/** Wrapper around an AutoFile& that implements a ring buffer to
499 *  deserialize from. It guarantees the ability to rewind a given number of bytes.
500 *