Use of bitcoin serialization Library

Time:2020-11-19

Bitcoin serialization function is mainly implemented inserialize.hFile, the whole code is mainly aroundstreamAnd types that participate in serialization deserializationTopen.  

Stream this template parameter expression hasread(char**, size_t)and write(char**, size_t) Method, similar to golang’s io.reader , io.writer 。

Simple use example:

#include <serialize.h>
#include <streams.h>
#include <hash.h>
#include <test/test_bitcoin.h>

#include <stdint.h>
#include <memory>

#include <boost/test/unit_test.hpp>

BOOST_FIXTURE_TEST_SUITE(serialize_tests, BasicTestingSetup)


struct  student
{
    std::string name;
    double midterm, final;
    std::vector<double> homework;

    ADD_SERIALIZE_METHODS;

    template <typename Stream, typename Operation>
    inline void SerializationOp(Stream& s, Operation ser_action) {
        READWRITE(name);
        READWRITE(midterm);
        READWRITE(final);
        READWRITE(homework);
    }
        
};

bool operator==(student const& lhs,  student const& rhs){
        return lhs.name == rhs.name &&  \
               lhs.midterm ==  rhs.midterm && \
               lhs.final  ==  rhs.final && \
               lhs.homework == rhs.homework;
}

std::ostream& operator<<(std::ostream& os, student const& st){
        os << "name: " << st.name << '\n' 
           << "midterm: " << st.midterm << '\n'
           << "final: "   << st.final  << '\n'
           << "homework: " ;
        for (auto e : st.homework) {
            os << e <<  ' ';
        }
        return os;
}


BOOST_AUTO_TEST_CASE(normal)
{
    student  s, t;
    s.name = "john";
    s.midterm = 77;
    s.final = 82;
    auto  v = std::vector<double> {83, 50, 10, 88, 65};
    s.homework = v;

    CDataStream ss(SER_DISK, 0);
    ss <<  s;
    ss >>  t; 

    BOOST_CHECK(t.name  == "john");
    BOOST_CHECK(t.midterm  == 77);
    BOOST_CHECK(t.final  == 82);
    BOOST_TEST(t.homework == v,  boost::test_tools::per_element()); 
    

    CDataStream sd(SER_DISK, 0);
    CDataStream sn(SER_NETWORK, PROTOCOL_VERSION);
    sd << s;
    sn << s;
    BOOST_CHECK(Hash(sd.begin(), sd.end()) == Hash(sn.begin(), sn.end()));
}

BOOST_AUTO_TEST_CASE(vector)
{
    auto vs = std::vector<student>(3);
    vs[0].name = "bob";
    vs[0].midterm = 90;
    vs[0].final = 76;
    vs[0].homework = std::vector<double> {85, 53, 12, 75, 55};

    vs[1].name = "jim";
    vs[1].midterm = 96;
    vs[1].final = 72;
    vs[1].homework = std::vector<double> {91, 46, 19, 70, 59};

    vs[2].name = "tom";
    vs[2].midterm = 85;
    vs[2].final = 57;
    vs[2].homework = std::vector<double> {91, 77, 45, 50, 35};


    CDataStream ss(SER_DISK, 0);
    auto vt = std::vector<student>(3);
    ss <<  vs;
    ss >>  vt; 

    BOOST_TEST(vs == vt,  boost::test_tools::per_element()); 
}

BOOST_AUTO_TEST_CASE(unique_ptr){
    auto hex = "0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba26000000000490047304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000";
    CDataStream stream(ParseHex(hex), SER_NETWORK, PROTOCOL_VERSION);
        //CTransaction tx(deserialize, stream);
    auto utx = std::unique_ptr<const CTransaction>(nullptr);
    ::Unserialize(stream, utx);
    BOOST_TEST(utx->vin.size() == std::size_t(1));
    BOOST_TEST(utx->vout[0].nValue == 1000000);
}

BOOST_AUTO_TEST_SUITE_END()

It needs to be added inside the user‘s custom typeADD_SERIALIZE_METHODSCall, after macro open:

template<typename Stream>                                         \
    void Serialize(Stream& s) const {                                 \
        NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize());  \
    }                                                                 \
    template<typename Stream>                                         \
    void Unserialize(Stream& s) {                                     \
        SerializationOp(s, CSerActionUnserialize());                  \
    }

This macro adds two member functions to the user-defined type:SerializeandUnserializeThey call the user-defined template member functions internallySerializationOp, inSerializationOpFunction internal, mainly usingREADWRITEandREADWRITEMANYMacro to complete the serialization and deserialization of each data member of the custom type.

#define READWRITE(obj)      (::SerReadWrite(s, (obj), ser_action))
#define READWRITEMANY(...)      (::SerReadWriteMany(s, ser_action, __VA_ARGS__))

struct CSerActionSerialize
{
    constexpr bool ForRead() const { return false; }
};
struct CSerActionUnserialize
{
    constexpr bool ForRead() const { return true; }
};

template<typename Stream, typename T>
inline void SerReadWrite(Stream& s, const T& obj, CSerActionSerialize ser_action)
{
    ::Serialize(s, obj);
}

template<typename Stream, typename T>
inline void SerReadWrite(Stream& s, T& obj, CSerActionUnserialize ser_action)
{
    ::Unserialize(s, obj);
}

template<typename Stream, typename... Args>
inline void SerReadWriteMany(Stream& s, CSerActionSerialize ser_action, Args&&... args)
{
    ::SerializeMany(s, std::forward<Args>(args)...);
}

template<typename Stream, typename... Args>
inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&... args)
{
    ::UnserializeMany(s, args...);
}

It needs to be added inside the user’s custom typeADD_SERIALIZE_METHODSCall, after macro open:


template<typename Stream>  \

 void Serialize(Stream& s) const {  \

 NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \

 }  \

 template<typename Stream>  \

 void Unserialize(Stream& s) {  \

 SerializationOp(s, CSerActionUnserialize()); \

 }

This macro adds two member functions to the user-defined type:SerializeandUnserializeThey call the user-defined template member functions internallySerializationOp, inSerializationOpFunction internal, mainly usingREADWRITEandREADWRITEMANYMacro to complete the serialization and deserialization of each data member of the custom type.

#define READWRITE(obj)      (::SerReadWrite(s, (obj), ser_action))
#define READWRITEMANY(...)      (::SerReadWriteMany(s, ser_action, __VA_ARGS__))

struct CSerActionSerialize
{
    constexpr bool ForRead() const { return false; }
};
struct CSerActionUnserialize
{
    constexpr bool ForRead() const { return true; }
};

template<typename Stream, typename T>
inline void SerReadWrite(Stream& s, const T& obj, CSerActionSerialize ser_action)
{
    ::Serialize(s, obj);
}

template<typename Stream, typename T>
inline void SerReadWrite(Stream& s, T& obj, CSerActionUnserialize ser_action)
{
    ::Unserialize(s, obj);
}

template<typename Stream, typename... Args>
inline void SerReadWriteMany(Stream& s, CSerActionSerialize ser_action, Args&&... args)
{
    ::SerializeMany(s, std::forward<Args>(args)...);
}

template<typename Stream, typename... Args>
inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&... args)
{
    ::UnserializeMany(s, args...);
}

Here, serreadwrite and serreadwritemany have two overload implementations. The difference is that different types are passed in at the endCSerActionSerializeandCSerActionUnserializeAnd the formal parameter Ser_ Action is not used internally at all. After consulting the relevant information, a common mode of C + + generic programming is used here

Tag dispatch technology](https://akrzemi1.wordpress.co…Another explanation:[https://arne-mertz.de/2016/10…://arne-mertz.de/2016/10/tag-dispatch/),

By carrying different types, different overload implementations are selected at compile time. Cseractionserialize corresponds to the implementation of serialization, and cseractionunserialize corresponds to the implementation of deserialization.

SerializeManyandSerializeManyIt is implemented by variable length template parameter pack expansion technologySerializeManyFor example:


template<typename Stream>
void SerializeMany(Stream& s)
{
}

template<typename Stream, typename Arg>
void SerializeMany(Stream& s, Arg&& arg)
{
    ::Serialize(s, std::forward<Arg>(arg));
}

template<typename Stream, typename Arg, typename... Args>
void SerializeMany(Stream& s, Arg&& arg, Args&&... args)
{
    ::Serialize(s, std::forward<Arg>(arg));
    ::SerializeMany(s, std::forward<Args>(args)...);
}

SerializeManyThere are three overload implementations, which are assumed to be inverted from top to bottom and numbered as 1, 2, and 3 respectively. When we pass in more than two arguments, the compiler selects version 3. In version 3, a parameter pops up from the parameter pack, and then passes it to version 2. The remaining parameter list is passed to version 3 and called recursively. Until the parameter pack is empty, version 1 is selected.

This detour is so long that the final serialization actually uses the global namespaceSerializeTo complete the deserialization by callingUnserializerealization.

andSerializeandUnserializeThere are also a bunch of overload implementations. Bitcoin authors implement some common types of template specialization, for example, STD:: string. They mainly design prevector, STD:: vector, STD:: pair, STD:: map, STD:: set, STD:: unique_ ptr, std::share_ ptr 。 According to the matching degree of the parameter list, the template matching of C + + has different implementations, and the precision matching is preferred. Finally, the member function of type T is selected for implementation

template<typename Stream, typename T>
inline void Serialize(Stream& os, const T& a)
{
    a.Serialize(os);
}

template<typename Stream, typename T>
inline void Unserialize(Stream& is, T& a)
{
    a.Unserialize(is);
}

When serializing string, map, set, vector, prevector and other collection types that may contain multiple elements, the internal will callReadCompactSizeandWriteCompactSizeNumber of read and write elements of compact code:

template<typename Stream>
void WriteCompactSize(Stream& os, uint64_t nSize)
{
    if (nSize < 253)
    {
        ser_writedata8(os, nSize);
    }
    else if (nSize <= std::numeric_limits<unsigned short>::max())
    {
        ser_writedata8(os, 253);
        ser_writedata16(os, nSize);
    }
    else if (nSize <= std::numeric_limits<unsigned int>::max())
    {
        ser_writedata8(os, 254);
        ser_writedata32(os, nSize);
    }
    else
    {
        ser_writedata8(os, 255);
        ser_writedata64(os, nSize);
    }
    return;
}

template<typename Stream>
uint64_t ReadCompactSize(Stream& is)
{
    uint8_t chSize = ser_readdata8(is);
    uint64_t nSizeRet = 0;
    if (chSize < 253)
    {
        nSizeRet = chSize;
    }
    else if (chSize == 253)
    {
        nSizeRet = ser_readdata16(is);
        if (nSizeRet < 253)
            throw std::ios_base::failure("non-canonical ReadCompactSize()");
    }
    else if (chSize == 254)
    {
        nSizeRet = ser_readdata32(is);
        if (nSizeRet < 0x10000u)
            throw std::ios_base::failure("non-canonical ReadCompactSize()");
    }
    else
    {
        nSizeRet = ser_readdata64(is);
        if (nSizeRet < 0x100000000ULL)
            throw std::ios_base::failure("non-canonical ReadCompactSize()");
    }
    if (nSizeRet > (uint64_t)MAX_SIZE)
        throw std::ios_base::failure("ReadCompactSize(): size too large");
    return nSizeRet;
}

For the base type of bit width 1,2,4,8,SerializeandUnserializeFinally, the SER is called_ writedata, ser_readdata8Complete the implementation.

template<typename Stream> inline void Serialize(Stream& s, char a    ) { ser_writedata8(s, a); } // TODO Get rid of bare char
template<typename Stream> inline void Serialize(Stream& s, int8_t a  ) { ser_writedata8(s, a); }
template<typename Stream> inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); }
template<typename Stream> inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); }
template<typename Stream> inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); }
template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); }
template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); }
template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); }
template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); }
template<typename Stream> inline void Serialize(Stream& s, float a   ) { ser_writedata32(s, ser_float_to_uint32(a)); }
template<typename Stream> inline void Serialize(Stream& s, double a  ) { ser_writedata64(s, ser_double_to_uint64(a)); }

template<typename Stream> inline void Unserialize(Stream& s, char& a    ) { a = ser_readdata8(s); } // TODO Get rid of bare char
template<typename Stream> inline void Unserialize(Stream& s, int8_t& a  ) { a = ser_readdata8(s); }
template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); }
template<typename Stream> inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); }
template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); }
template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); }
template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); }
template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); }
template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); }
template<typename Stream> inline void Unserialize(Stream& s, float& a   ) { a = ser_uint32_to_float(ser_readdata32(s)); }
template<typename Stream> inline void Unserialize(Stream& s, double& a  ) { a = ser_uint64_to_double(ser_readdata64(s)); }

template<typename Stream> inline void Serialize(Stream& s, bool a)    { char f=a; ser_writedata8(s, f); }
template<typename Stream> inline void Unserialize(Stream& s, bool& a) { char f=ser_readdata8(s); a=f; }

In addition, the

struct deserialize_type {};
constexpr deserialize_type deserialize {};

As a type of tag, tag object is mainly used for multiple implementations. The signature has the following forms:

template <typename Stream> 
T::T(deserialize_type, Stream& s)

At present, the main type of the destructor is the transaction destructor

template <typename Stream>
    CTransaction(deserialize_type, Stream& s) : CTransaction(CMutableTransaction(deserialize, s)) {}
    
    template <typename Stream>
    CMutableTransaction(deserialize_type, Stream& s) {
        Unserialize(s);
    }

Original address:https://mp.weixin.qq.com/s/_fhGCfkI0sT3fasSKWqdPA


This paper is written byYu Jian of Copernicus teamNo authorization is required for writing and reprinting.