X-Git-Url: https://git.novaco.in/?p=novacoin.git;a=blobdiff_plain;f=src%2Fserialize.h;h=bdcd830591d570dbde8aacb9d4ab8312b0cc406f;hp=61120fd496982d8e2362e254659e0abe647863e7;hb=415da519893e8fef7a10007fc82934385addd03d;hpb=09f2a990950512cbe418526d91838887a0dded2a diff --git a/src/serialize.h b/src/serialize.h index 61120fd..bdcd830 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1,8 +1,7 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2011 The Bitcoin developers -// Copyright (c) 2012 The PPCoin developers +// Copyright (c) 2009-2012 The Bitcoin developers // Distributed under the MIT/X11 software license, see the accompanying -// file license.txt or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SERIALIZE_H #define BITCOIN_SERIALIZE_H @@ -11,62 +10,31 @@ #include #include #include -#include +#include #include #include +#ifndef Q_MOC_RUN #include #include #include #include - -#if defined(_MSC_VER) || defined(__BORLANDC__) -typedef __int64 int64; -typedef unsigned __int64 uint64; -#else -typedef long long int64; -typedef unsigned long long uint64; -#endif -#if defined(_MSC_VER) && _MSC_VER < 1300 -#define for if (false) ; else for #endif -#ifdef WIN32 -#include -// This is used to attempt to keep keying material out of swap -// Note that VirtualLock does not provide this as a guarantee on Windows, -// but, in practice, memory that has been VirtualLock'd almost never gets written to -// the pagefile except in rare circumstances where memory is extremely low. -#include -#define mlock(p, n) VirtualLock((p), (n)); -#define munlock(p, n) VirtualUnlock((p), (n)); -#else -#include -#include -/* This comes from limits.h if it's not defined there set a sane default */ -#ifndef PAGESIZE -#include -#define PAGESIZE sysconf(_SC_PAGESIZE) -#endif -#define mlock(a,b) \ - mlock(((void *)(((size_t)(a)) & (~((PAGESIZE)-1)))),\ - (((((size_t)(a)) + (b) - 1) | ((PAGESIZE) - 1)) + 1) - (((size_t)(a)) & (~((PAGESIZE) - 1)))) -#define munlock(a,b) \ - munlock(((void *)(((size_t)(a)) & (~((PAGESIZE)-1)))),\ - (((((size_t)(a)) + (b) - 1) | ((PAGESIZE) - 1)) + 1) - (((size_t)(a)) & (~((PAGESIZE) - 1)))) +#if defined __USE_MINGW_ANSI_STDIO +#undef __USE_MINGW_ANSI_STDIO // This constant forces MinGW to conduct stupid behavior #endif +#include + +#include "allocators.h" +#include "version.h" + class CScript; class CDataStream; class CAutoFile; static const unsigned int MAX_SIZE = 0x02000000; -static const int VERSION = 50100; -static const char* pszSubVer = ""; -static const bool VERSION_IS_BETA = true; - -static const int PPCOIN_VERSION = 100; - // Used to bypass the rule against non-const reference to temporary // where it makes sense with wrappers such as CFlatData or CTxDB template @@ -90,11 +58,12 @@ enum // modifiers SER_SKIPSIG = (1 << 16), - SER_BLOCKHEADERONLY = (1 << 17), + SER_BLOCKHEADERONLY = (1 << 17) + }; #define IMPLEMENT_SERIALIZE(statements) \ - unsigned int GetSerializeSize(int nType=0, int nVersion=VERSION) const \ + unsigned int GetSerializeSize(int nType, int nVersion) const \ { \ CSerActionGetSerializeSize ser_action; \ const bool fGetSize = true; \ @@ -102,29 +71,32 @@ enum const bool fRead = false; \ unsigned int nSerSize = 0; \ ser_streamplaceholder s; \ + assert(fGetSize||fWrite||fRead); /* suppress warning */ \ s.nType = nType; \ s.nVersion = nVersion; \ {statements} \ return nSerSize; \ } \ template \ - void Serialize(Stream& s, int nType=0, int nVersion=VERSION) const \ + void Serialize(Stream& s, int nType, int nVersion) const \ { \ CSerActionSerialize ser_action; \ const bool fGetSize = false; \ const bool fWrite = true; \ const bool fRead = false; \ unsigned int nSerSize = 0; \ + assert(fGetSize||fWrite||fRead); /* suppress warning */ \ {statements} \ } \ template \ - void Unserialize(Stream& s, int nType=0, int nVersion=VERSION) \ + void Unserialize(Stream& s, int nType, int nVersion) \ { \ CSerActionUnserialize ser_action; \ const bool fGetSize = false; \ const bool fWrite = false; \ const bool fRead = true; \ unsigned int nSerSize = 0; \ + assert(fGetSize||fWrite||fRead); /* suppress warning */ \ {statements} \ } @@ -148,10 +120,10 @@ inline unsigned int GetSerializeSize(signed short a, int, int=0) { return size inline unsigned int GetSerializeSize(unsigned short a, int, int=0) { return sizeof(a); } inline unsigned int GetSerializeSize(signed int a, int, int=0) { return sizeof(a); } inline unsigned int GetSerializeSize(unsigned int a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(signed long a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(unsigned long a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(int64 a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(uint64 a, int, int=0) { return sizeof(a); } +//inline unsigned int GetSerializeSize(signed long a, int, int=0) { return sizeof(a); } +//inline unsigned int GetSerializeSize(unsigned long a, int, int=0) { return sizeof(a); } +inline unsigned int GetSerializeSize(int64_t a, int, int=0) { return sizeof(a); } +inline unsigned int GetSerializeSize(uint64_t a, int, int=0) { return sizeof(a); } inline unsigned int GetSerializeSize(float a, int, int=0) { return sizeof(a); } inline unsigned int GetSerializeSize(double a, int, int=0) { return sizeof(a); } @@ -162,10 +134,10 @@ template inline void Serialize(Stream& s, signed short a, int template inline void Serialize(Stream& s, unsigned short a, int, int=0) { WRITEDATA(s, a); } template inline void Serialize(Stream& s, signed int a, int, int=0) { WRITEDATA(s, a); } template inline void Serialize(Stream& s, unsigned int a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, signed long a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, unsigned long a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, int64 a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, uint64 a, int, int=0) { WRITEDATA(s, a); } +//template inline void Serialize(Stream& s, signed long a, int, int=0) { WRITEDATA(s, a); } +//template inline void Serialize(Stream& s, unsigned long a, int, int=0) { WRITEDATA(s, a); } +template inline void Serialize(Stream& s, int64_t a, int, int=0) { WRITEDATA(s, a); } +template inline void Serialize(Stream& s, uint64_t a, int, int=0) { WRITEDATA(s, a); } template inline void Serialize(Stream& s, float a, int, int=0) { WRITEDATA(s, a); } template inline void Serialize(Stream& s, double a, int, int=0) { WRITEDATA(s, a); } @@ -176,10 +148,10 @@ template inline void Unserialize(Stream& s, signed short& a, template inline void Unserialize(Stream& s, unsigned short& a, int, int=0) { READDATA(s, a); } template inline void Unserialize(Stream& s, signed int& a, int, int=0) { READDATA(s, a); } template inline void Unserialize(Stream& s, unsigned int& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, signed long& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, unsigned long& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, int64& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, uint64& a, int, int=0) { READDATA(s, a); } +//template inline void Unserialize(Stream& s, signed long& a, int, int=0) { READDATA(s, a); } +//template inline void Unserialize(Stream& s, unsigned long& a, int, int=0) { READDATA(s, a); } +template inline void Unserialize(Stream& s, int64_t& a, int, int=0) { READDATA(s, a); } +template inline void Unserialize(Stream& s, uint64_t& a, int, int=0) { READDATA(s, a); } template inline void Unserialize(Stream& s, float& a, int, int=0) { READDATA(s, a); } template inline void Unserialize(Stream& s, double& a, int, int=0) { READDATA(s, a); } @@ -189,14 +161,8 @@ template inline void Unserialize(Stream& s, bool& a, int, int=0 -#ifndef THROW_WITH_STACKTRACE -#define THROW_WITH_STACKTRACE(exception) \ -{ \ - LogStackTrace(); \ - throw (exception); \ -} -void LogStackTrace(); -#endif + + // // Compact size @@ -205,40 +171,40 @@ void LogStackTrace(); // size <= UINT_MAX -- 5 bytes (254 + 4 bytes) // size > UINT_MAX -- 9 bytes (255 + 8 bytes) // -inline unsigned int GetSizeOfCompactSize(uint64 nSize) +inline unsigned int GetSizeOfCompactSize(uint64_t nSize) { if (nSize < 253) return sizeof(unsigned char); - else if (nSize <= USHRT_MAX) return sizeof(unsigned char) + sizeof(unsigned short); - else if (nSize <= UINT_MAX) return sizeof(unsigned char) + sizeof(unsigned int); - else return sizeof(unsigned char) + sizeof(uint64); + else if (nSize <= std::numeric_limits::max()) return sizeof(unsigned char) + sizeof(unsigned short); + else if (nSize <= std::numeric_limits::max()) return sizeof(unsigned char) + sizeof(unsigned int); + else return sizeof(unsigned char) + sizeof(uint64_t); } template -void WriteCompactSize(Stream& os, uint64 nSize) +void WriteCompactSize(Stream& os, uint64_t nSize) { if (nSize < 253) { - unsigned char chSize = nSize; + unsigned char chSize = (unsigned char)nSize; WRITEDATA(os, chSize); } - else if (nSize <= USHRT_MAX) + else if (nSize <= std::numeric_limits::max()) { unsigned char chSize = 253; - unsigned short xSize = nSize; + unsigned short xSize = (unsigned short)nSize; WRITEDATA(os, chSize); WRITEDATA(os, xSize); } - else if (nSize <= UINT_MAX) + else if (nSize <= std::numeric_limits::max()) { unsigned char chSize = 254; - unsigned int xSize = nSize; + unsigned int xSize = (unsigned int)nSize; WRITEDATA(os, chSize); WRITEDATA(os, xSize); } else { unsigned char chSize = 255; - uint64 xSize = nSize; + uint64_t xSize = nSize; WRITEDATA(os, chSize); WRITEDATA(os, xSize); } @@ -246,11 +212,11 @@ void WriteCompactSize(Stream& os, uint64 nSize) } template -uint64 ReadCompactSize(Stream& is) +uint64_t ReadCompactSize(Stream& is) { unsigned char chSize; READDATA(is, chSize); - uint64 nSizeRet = 0; + uint64_t nSizeRet = 0; if (chSize < 253) { nSizeRet = chSize; @@ -269,22 +235,87 @@ uint64 ReadCompactSize(Stream& is) } else { - uint64 xSize; + uint64_t xSize; READDATA(is, xSize); nSizeRet = xSize; } - if (nSizeRet > (uint64)MAX_SIZE) - THROW_WITH_STACKTRACE(std::ios_base::failure("ReadCompactSize() : size too large")); + if (nSizeRet > (uint64_t)MAX_SIZE) + throw std::ios_base::failure("ReadCompactSize() : size too large"); return nSizeRet; } - - +// Variable-length integers: bytes are a MSB base-128 encoding of the number. +// The high bit in each byte signifies whether another digit follows. To make +// the encoding is one-to-one, one is subtracted from all but the last digit. +// Thus, the byte sequence a[] with length len, where all but the last byte +// has bit 128 set, encodes the number: +// +// (a[len-1] & 0x7F) + sum(i=1..len-1, 128^i*((a[len-i-1] & 0x7F)+1)) // -// Wrapper for serializing arrays and POD -// There's a clever template way to make arrays serialize normally, but MSVC6 doesn't support it +// Properties: +// * Very small (0-127: 1 byte, 128-16511: 2 bytes, 16512-2113663: 3 bytes) +// * Every integer has exactly one encoding +// * Encoding does not depend on size of original integer type +// * No redundancy: every (infinite) byte sequence corresponds to a list +// of encoded integers. // -#define FLATDATA(obj) REF(CFlatData((char*)&(obj), (char*)&(obj) + sizeof(obj))) +// 0: [0x00] 256: [0x81 0x00] +// 1: [0x01] 16383: [0xFE 0x7F] +// 127: [0x7F] 16384: [0xFF 0x00] +// 128: [0x80 0x00] 16511: [0x80 0xFF 0x7F] +// 255: [0x80 0x7F] 65535: [0x82 0xFD 0x7F] +// 2^32: [0x8E 0xFE 0xFE 0xFF 0x00] + +template +inline unsigned int GetSizeOfVarInt(I n) +{ + int nRet = 0; + for ( ; ; ) { + nRet++; + if (n <= 0x7F) + break; + n = (n >> 7) - 1; + } + return nRet; +} + +template +void WriteVarInt(Stream& os, I n) +{ + unsigned char tmp[(sizeof(n)*8+6)/7]; + int len=0; + for ( ; ; ) { + tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00); + if (n <= 0x7F) + break; + n = (n >> 7) - 1; + len++; + } + do { + WRITEDATA(os, tmp[len]); + } while(len--); +} + +template +I ReadVarInt(Stream& is) +{ + I n = 0; + for ( ; ; ) { + unsigned char chData; + READDATA(is, chData); + n = (n << 7) | (chData & 0x7F); + if (chData & 0x80) + n++; + else + return n; + } +} + +#define FLATDATA(obj) REF(CFlatData((char*)&(obj), (char*)&(obj) + sizeof(obj))) +#define VARINT(obj) REF(WrapVarInt(REF(obj))) + +/** Wrapper for serializing arrays and POD. + */ class CFlatData { protected: @@ -299,13 +330,13 @@ public: unsigned int GetSerializeSize(int, int=0) const { - return pend - pbegin; + return (unsigned int)(pend - pbegin); } template void Serialize(Stream& s, int, int=0) const { - s.write(pbegin, pend - pbegin); + s.write(pbegin, (int)(pend - pbegin)); } template @@ -315,49 +346,31 @@ public: } }; - - -// -// string stored as a fixed length field -// -template -class CFixedFieldString +template +class CVarInt { protected: - const std::string* pcstr; - std::string* pstr; + I &n; public: - explicit CFixedFieldString(const std::string& str) : pcstr(&str), pstr(NULL) { } - explicit CFixedFieldString(std::string& str) : pcstr(&str), pstr(&str) { } + CVarInt(I& nIn) : n(nIn) { } - unsigned int GetSerializeSize(int, int=0) const - { - return LEN; + unsigned int GetSerializeSize(int, int) const { + return GetSizeOfVarInt(n); } template - void Serialize(Stream& s, int, int=0) const - { - char pszBuf[LEN]; - strncpy(pszBuf, pcstr->c_str(), LEN); - s.write(pszBuf, LEN); + void Serialize(Stream &s, int, int) const { + WriteVarInt(s, n); } template - void Unserialize(Stream& s, int, int=0) - { - if (pstr == NULL) - THROW_WITH_STACKTRACE(std::ios_base::failure("CFixedFieldString::Unserialize : trying to unserialize to const string")); - char pszBuf[LEN+1]; - s.read(pszBuf, LEN); - pszBuf[LEN] = '\0'; - *pstr = pszBuf; + void Unserialize(Stream& s, int, int) { + n = ReadVarInt(s); } }; - - - +template +CVarInt WrapVarInt(I& n) { return CVarInt(n); } // // Forward declarations @@ -371,43 +384,43 @@ template void Unserialize(Stream& is, std::basic_st // vector template unsigned int GetSerializeSize_impl(const std::vector& v, int nType, int nVersion, const boost::true_type&); template unsigned int GetSerializeSize_impl(const std::vector& v, int nType, int nVersion, const boost::false_type&); -template inline unsigned int GetSerializeSize(const std::vector& v, int nType, int nVersion=VERSION); +template inline unsigned int GetSerializeSize(const std::vector& v, int nType, int nVersion); template void Serialize_impl(Stream& os, const std::vector& v, int nType, int nVersion, const boost::true_type&); template void Serialize_impl(Stream& os, const std::vector& v, int nType, int nVersion, const boost::false_type&); -template inline void Serialize(Stream& os, const std::vector& v, int nType, int nVersion=VERSION); +template inline void Serialize(Stream& os, const std::vector& v, int nType, int nVersion); template void Unserialize_impl(Stream& is, std::vector& v, int nType, int nVersion, const boost::true_type&); template void Unserialize_impl(Stream& is, std::vector& v, int nType, int nVersion, const boost::false_type&); -template inline void Unserialize(Stream& is, std::vector& v, int nType, int nVersion=VERSION); +template inline void Unserialize(Stream& is, std::vector& v, int nType, int nVersion); // others derived from vector -extern inline unsigned int GetSerializeSize(const CScript& v, int nType, int nVersion=VERSION); -template void Serialize(Stream& os, const CScript& v, int nType, int nVersion=VERSION); -template void Unserialize(Stream& is, CScript& v, int nType, int nVersion=VERSION); +extern inline unsigned int GetSerializeSize(const CScript& v, int nType, int nVersion); +template void Serialize(Stream& os, const CScript& v, int nType, int nVersion); +template void Unserialize(Stream& is, CScript& v, int nType, int nVersion); // pair -template unsigned int GetSerializeSize(const std::pair& item, int nType, int nVersion=VERSION); -template void Serialize(Stream& os, const std::pair& item, int nType, int nVersion=VERSION); -template void Unserialize(Stream& is, std::pair& item, int nType, int nVersion=VERSION); +template unsigned int GetSerializeSize(const std::pair& item, int nType, int nVersion); +template void Serialize(Stream& os, const std::pair& item, int nType, int nVersion); +template void Unserialize(Stream& is, std::pair& item, int nType, int nVersion); // 3 tuple -template unsigned int GetSerializeSize(const boost::tuple& item, int nType, int nVersion=VERSION); -template void Serialize(Stream& os, const boost::tuple& item, int nType, int nVersion=VERSION); -template void Unserialize(Stream& is, boost::tuple& item, int nType, int nVersion=VERSION); +template unsigned int GetSerializeSize(const boost::tuple& item, int nType, int nVersion); +template void Serialize(Stream& os, const boost::tuple& item, int nType, int nVersion); +template void Unserialize(Stream& is, boost::tuple& item, int nType, int nVersion); // 4 tuple -template unsigned int GetSerializeSize(const boost::tuple& item, int nType, int nVersion=VERSION); -template void Serialize(Stream& os, const boost::tuple& item, int nType, int nVersion=VERSION); -template void Unserialize(Stream& is, boost::tuple& item, int nType, int nVersion=VERSION); +template unsigned int GetSerializeSize(const boost::tuple& item, int nType, int nVersion); +template void Serialize(Stream& os, const boost::tuple& item, int nType, int nVersion); +template void Unserialize(Stream& is, boost::tuple& item, int nType, int nVersion); // map -template unsigned int GetSerializeSize(const std::map& m, int nType, int nVersion=VERSION); -template void Serialize(Stream& os, const std::map& m, int nType, int nVersion=VERSION); -template void Unserialize(Stream& is, std::map& m, int nType, int nVersion=VERSION); +template unsigned int GetSerializeSize(const std::map& m, int nType, int nVersion); +template void Serialize(Stream& os, const std::map& m, int nType, int nVersion); +template void Unserialize(Stream& is, std::map& m, int nType, int nVersion); // set -template unsigned int GetSerializeSize(const std::set& m, int nType, int nVersion=VERSION); -template void Serialize(Stream& os, const std::set& m, int nType, int nVersion=VERSION); -template void Unserialize(Stream& is, std::set& m, int nType, int nVersion=VERSION); +template unsigned int GetSerializeSize(const std::set& m, int nType, int nVersion); +template void Serialize(Stream& os, const std::set& m, int nType, int nVersion); +template void Unserialize(Stream& is, std::set& m, int nType, int nVersion); @@ -420,19 +433,19 @@ template void Unserializ // Thanks to Boost serialization for this idea. // template -inline unsigned int GetSerializeSize(const T& a, long nType, int nVersion=VERSION) +inline unsigned int GetSerializeSize(const T& a, long nType, int nVersion) { return a.GetSerializeSize((int)nType, nVersion); } template -inline void Serialize(Stream& os, const T& a, long nType, int nVersion=VERSION) +inline void Serialize(Stream& os, const T& a, long nType, int nVersion) { a.Serialize(os, (int)nType, nVersion); } template -inline void Unserialize(Stream& is, T& a, long nType, int nVersion=VERSION) +inline void Unserialize(Stream& is, T& a, long nType, int nVersion) { a.Unserialize(is, (int)nType, nVersion); } @@ -447,7 +460,7 @@ inline void Unserialize(Stream& is, T& a, long nType, int nVersion=VERSION) template unsigned int GetSerializeSize(const std::basic_string& str, int, int) { - return GetSizeOfCompactSize(str.size()) + str.size() * sizeof(str[0]); + return (unsigned int)(GetSizeOfCompactSize(str.size()) + str.size() * sizeof(str[0])); } template @@ -455,13 +468,13 @@ void Serialize(Stream& os, const std::basic_string& str, int, int) { WriteCompactSize(os, str.size()); if (!str.empty()) - os.write((char*)&str[0], str.size() * sizeof(str[0])); + os.write((char*)&str[0], (int)(str.size() * sizeof(str[0]))); } template void Unserialize(Stream& is, std::basic_string& str, int, int) { - unsigned int nSize = ReadCompactSize(is); + unsigned int nSize = (unsigned int)(ReadCompactSize(is)); str.resize(nSize); if (nSize != 0) is.read((char*)&str[0], nSize * sizeof(str[0])); @@ -475,7 +488,7 @@ void Unserialize(Stream& is, std::basic_string& str, int, int) template unsigned int GetSerializeSize_impl(const std::vector& v, int nType, int nVersion, const boost::true_type&) { - return (GetSizeOfCompactSize(v.size()) + v.size() * sizeof(T)); + return (unsigned int)(GetSizeOfCompactSize(v.size()) + v.size() * sizeof(T)); } template @@ -499,7 +512,7 @@ void Serialize_impl(Stream& os, const std::vector& v, int nType, int nVers { WriteCompactSize(os, v.size()); if (!v.empty()) - os.write((char*)&v[0], v.size() * sizeof(T)); + os.write((char*)&v[0], (int)(v.size() * sizeof(T))); } template @@ -520,13 +533,9 @@ inline void Serialize(Stream& os, const std::vector& v, int nType, int nVe template void Unserialize_impl(Stream& is, std::vector& v, int nType, int nVersion, const boost::true_type&) { - //unsigned int nSize = ReadCompactSize(is); - //v.resize(nSize); - //is.read((char*)&v[0], nSize * sizeof(T)); - // Limit size per read so bogus size value won't cause out of memory v.clear(); - unsigned int nSize = ReadCompactSize(is); + unsigned int nSize = (unsigned int)(ReadCompactSize(is)); unsigned int i = 0; while (i < nSize) { @@ -540,13 +549,8 @@ void Unserialize_impl(Stream& is, std::vector& v, int nType, int nVersion, template void Unserialize_impl(Stream& is, std::vector& v, int nType, int nVersion, const boost::false_type&) { - //unsigned int nSize = ReadCompactSize(is); - //v.resize(nSize); - //for (std::vector::iterator vi = v.begin(); vi != v.end(); ++vi) - // Unserialize(is, (*vi), nType, nVersion); - v.clear(); - unsigned int nSize = ReadCompactSize(is); + unsigned int nSize = (unsigned int)(ReadCompactSize(is)); unsigned int i = 0; unsigned int nMid = 0; while (nMid < nSize) @@ -704,7 +708,7 @@ template void Unserialize(Stream& is, std::map& m, int nType, int nVersion) { m.clear(); - unsigned int nSize = ReadCompactSize(is); + unsigned int nSize = (unsigned int)(ReadCompactSize(is)); typename std::map::iterator mi = m.begin(); for (unsigned int i = 0; i < nSize; i++) { @@ -793,61 +797,19 @@ struct ser_streamplaceholder -// -// Allocator that locks its contents from being paged -// out of memory and clears its contents before deletion. -// -template -struct secure_allocator : public std::allocator -{ - // MSVC8 default copy constructor is broken - typedef std::allocator base; - typedef typename base::size_type size_type; - typedef typename base::difference_type difference_type; - typedef typename base::pointer pointer; - typedef typename base::const_pointer const_pointer; - typedef typename base::reference reference; - typedef typename base::const_reference const_reference; - typedef typename base::value_type value_type; - secure_allocator() throw() {} - secure_allocator(const secure_allocator& a) throw() : base(a) {} - template - secure_allocator(const secure_allocator& a) throw() : base(a) {} - ~secure_allocator() throw() {} - template struct rebind - { typedef secure_allocator<_Other> other; }; - - T* allocate(std::size_t n, const void *hint = 0) - { - T *p; - p = std::allocator::allocate(n, hint); - if (p != NULL) - mlock(p, sizeof(T) * n); - return p; - } - - void deallocate(T* p, std::size_t n) - { - if (p != NULL) - { - memset(p, 0, sizeof(T) * n); - munlock(p, sizeof(T) * n); - } - std::allocator::deallocate(p, n); - } -}; +typedef std::vector > CSerializeData; -// -// Double ended buffer combining vector and stream-like interfaces. -// >> and << read and write unformatted data using the above serialization templates. -// Fills with data in linear time; some stringstream implementations take N^2 time. -// +/** Double ended buffer combining vector and stream-like interfaces. + * + * >> and << read and write unformatted data using the above serialization templates. + * Fills with data in linear time; some stringstream implementations take N^2 time. + */ class CDataStream { protected: - typedef std::vector > vector_type; + typedef CSerializeData vector_type; vector_type vch; unsigned int nReadPos; short state; @@ -866,39 +828,39 @@ public: typedef vector_type::const_iterator const_iterator; typedef vector_type::reverse_iterator reverse_iterator; - explicit CDataStream(int nTypeIn=SER_NETWORK, int nVersionIn=VERSION) + explicit CDataStream(int nTypeIn, int nVersionIn) { Init(nTypeIn, nVersionIn); } - CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn=SER_NETWORK, int nVersionIn=VERSION) : vch(pbegin, pend) + CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) { Init(nTypeIn, nVersionIn); } #if !defined(_MSC_VER) || _MSC_VER >= 1300 - CDataStream(const char* pbegin, const char* pend, int nTypeIn=SER_NETWORK, int nVersionIn=VERSION) : vch(pbegin, pend) + CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) { Init(nTypeIn, nVersionIn); } #endif - CDataStream(const vector_type& vchIn, int nTypeIn=SER_NETWORK, int nVersionIn=VERSION) : vch(vchIn.begin(), vchIn.end()) + CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) { Init(nTypeIn, nVersionIn); } - CDataStream(const std::vector& vchIn, int nTypeIn=SER_NETWORK, int nVersionIn=VERSION) : vch(vchIn.begin(), vchIn.end()) + CDataStream(const std::vector& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) { Init(nTypeIn, nVersionIn); } - CDataStream(const std::vector& vchIn, int nTypeIn=SER_NETWORK, int nVersionIn=VERSION) : vch((char*)&vchIn.begin()[0], (char*)&vchIn.end()[0]) + CDataStream(const std::vector& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) { Init(nTypeIn, nVersionIn); } - void Init(int nTypeIn=SER_NETWORK, int nVersionIn=VERSION) + void Init(int nTypeIn, int nVersionIn) { nReadPos = 0; nType = nTypeIn; @@ -943,21 +905,26 @@ public: iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); } void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); } +#ifdef _MSC_VER void insert(iterator it, const_iterator first, const_iterator last) { - if (it == vch.begin() + nReadPos && last - first <= nReadPos) + assert(last - first >= 0); + if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room - nReadPos -= (last - first); + nReadPos -= (unsigned int)(last - first); memcpy(&vch[nReadPos], &first[0], last - first); } else vch.insert(it, first, last); } +#endif +#ifndef _MSC_VER void insert(iterator it, std::vector::const_iterator first, std::vector::const_iterator last) { - if (it == vch.begin() + nReadPos && last - first <= nReadPos) + assert(last - first >= 0); + if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room nReadPos -= (last - first); @@ -966,14 +933,16 @@ public: else vch.insert(it, first, last); } +#endif #if !defined(_MSC_VER) || _MSC_VER >= 1300 void insert(iterator it, const char* first, const char* last) { - if (it == vch.begin() + nReadPos && last - first <= nReadPos) + assert(last - first >= 0); + if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room - nReadPos -= (last - first); + nReadPos -= (unsigned int)(last - first); memcpy(&vch[nReadPos], &first[0], last - first); } else @@ -1010,7 +979,7 @@ public: } else { - nReadPos = (last - vch.begin()); + nReadPos = (unsigned int)(last - vch.begin()); return last; } } @@ -1029,7 +998,7 @@ public: // Rewind by n characters if the buffer hasn't been compacted yet if (n > nReadPos) return false; - nReadPos -= n; + nReadPos -= (unsigned int)n; return true; } @@ -1041,17 +1010,17 @@ public: { state |= bits; if (state & exceptmask) - THROW_WITH_STACKTRACE(std::ios_base::failure(psz)); + throw std::ios_base::failure(psz); } bool eof() const { return size() == 0; } - bool fail() const { return state & (std::ios::badbit | std::ios::failbit); } + bool fail() const { return (state & (std::ios::badbit | std::ios::failbit)) != 0; } bool good() const { return !eof() && (state == 0); } void clear(short n) { state = n; } // name conflict with vector clear() short exceptions() { return exceptmask; } short exceptions(short mask) { short prev = exceptmask; exceptmask = mask; setstate(0, "CDataStream"); return prev; } CDataStream* rdbuf() { return this; } - int in_avail() { return size(); } + int in_avail() { return (int)(size()); } void SetType(int n) { nType = n; } int GetType() { return nType; } @@ -1071,7 +1040,7 @@ public: { setstate(std::ios::failbit, "CDataStream::read() : end of data"); memset(pch, 0, nSize); - nSize = vch.size() - nReadPos; + nSize = (int)(vch.size() - nReadPos); } memcpy(pch, &vch[nReadPos], nSize); nReadPos = 0; @@ -1091,10 +1060,7 @@ public: if (nReadPosNext >= vch.size()) { if (nReadPosNext > vch.size()) - { setstate(std::ios::failbit, "CDataStream::ignore() : end of data"); - nSize = vch.size() - nReadPos; - } nReadPos = 0; vch.clear(); return (*this); @@ -1112,7 +1078,7 @@ public: } template - void Serialize(Stream& s, int nType=0, int nVersion=VERSION) const + void Serialize(Stream& s, int nType, int nVersion) const { // Special case: stream << stream concatenates like stream += stream if (!vch.empty()) @@ -1141,58 +1107,12 @@ public: ::Unserialize(*this, obj, nType, nVersion); return (*this); } -}; -#ifdef TESTCDATASTREAM -// VC6sp6 -// CDataStream: -// n=1000 0 seconds -// n=2000 0 seconds -// n=4000 0 seconds -// n=8000 0 seconds -// n=16000 0 seconds -// n=32000 0 seconds -// n=64000 1 seconds -// n=128000 1 seconds -// n=256000 2 seconds -// n=512000 4 seconds -// n=1024000 8 seconds -// n=2048000 16 seconds -// n=4096000 32 seconds -// stringstream: -// n=1000 1 seconds -// n=2000 1 seconds -// n=4000 13 seconds -// n=8000 87 seconds -// n=16000 400 seconds -// n=32000 1660 seconds -// n=64000 6749 seconds -// n=128000 27241 seconds -// n=256000 109804 seconds -#include -int main(int argc, char *argv[]) -{ - vector vch(0xcc, 250); - printf("CDataStream:\n"); - for (int n = 1000; n <= 4500000; n *= 2) - { - CDataStream ss; - time_t nStart = time(NULL); - for (int i = 0; i < n; i++) - ss.write((char*)&vch[0], vch.size()); - printf("n=%-10d %d seconds\n", n, time(NULL) - nStart); - } - printf("stringstream:\n"); - for (int n = 1000; n <= 4500000; n *= 2) - { - stringstream ss; - time_t nStart = time(NULL); - for (int i = 0; i < n; i++) - ss.write((char*)&vch[0], vch.size()); - printf("n=%-10d %d seconds\n", n, time(NULL) - nStart); + void GetAndClear(CSerializeData &data) { + vch.swap(data); + CSerializeData().swap(vch); } -} -#endif +}; @@ -1203,12 +1123,12 @@ int main(int argc, char *argv[]) -// -// Automatic closing wrapper for FILE* -// - Will automatically close the file when it goes out of scope if not null. -// - If you're returning the file pointer, return file.release(). -// - If you need to close the file early, use file.fclose() instead of fclose(file). -// +/** RAII wrapper for FILE*. + * + * Will automatically close the file when it goes out of scope if not null. + * If you're returning the file pointer, return file.release(). + * If you need to close the file early, use file.fclose() instead of fclose(file). + */ class CAutoFile { protected: @@ -1219,9 +1139,7 @@ public: int nType; int nVersion; - typedef FILE element_type; - - CAutoFile(FILE* filenew=NULL, int nTypeIn=SER_DISK, int nVersionIn=VERSION) + CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) { file = filenew; nType = nTypeIn; @@ -1258,10 +1176,10 @@ public: { state |= bits; if (state & exceptmask) - THROW_WITH_STACKTRACE(std::ios_base::failure(psz)); + throw std::ios_base::failure(psz); } - bool fail() const { return state & (std::ios::badbit | std::ios::failbit); } + bool fail() const { return (state & (std::ios::badbit | std::ios::failbit)) != 0; } bool good() const { return state == 0; } void clear(short n = 0) { state = n; } short exceptions() { return exceptmask; } @@ -1274,7 +1192,7 @@ public: void ReadVersion() { *this >> nVersion; } void WriteVersion() { *this << nVersion; } - CAutoFile& read(char* pch, int nSize) + CAutoFile& read(char* pch, size_t nSize) { if (!file) throw std::ios_base::failure("CAutoFile::read : file handle is NULL"); @@ -1283,7 +1201,7 @@ public: return (*this); } - CAutoFile& write(const char* pch, int nSize) + CAutoFile& write(const char* pch, size_t nSize) { if (!file) throw std::ios_base::failure("CAutoFile::write : file handle is NULL"); @@ -1320,4 +1238,148 @@ public: } }; +/** Wrapper around a FILE* that implements a ring buffer to + * deserialize from. It guarantees the ability to rewind + * a given number of bytes. */ +class CBufferedFile +{ +private: + FILE *src; // source file + uint64_t nSrcPos; // how many bytes have been read from source + uint64_t nReadPos; // how many bytes have been read from this + uint64_t nReadLimit; // up to which position we're allowed to read + uint64_t nRewind; // how many bytes we guarantee to rewind + std::vector vchBuf; // the buffer + + short state; + short exceptmask; + +protected: + void setstate(short bits, const char *psz) { + state |= bits; + if (state & exceptmask) + throw std::ios_base::failure(psz); + } + + // read data from the source to fill the buffer + bool Fill() { + unsigned int pos = (unsigned int)(nSrcPos % vchBuf.size()); + unsigned int readNow = (unsigned int)(vchBuf.size() - pos); + unsigned int nAvail = (unsigned int)(vchBuf.size() - (nSrcPos - nReadPos) - nRewind); + if (nAvail < readNow) + readNow = nAvail; + if (readNow == 0) + return false; + size_t read = fread((void*)&vchBuf[pos], 1, readNow, src); + if (read == 0) { + setstate(std::ios_base::failbit, feof(src) ? "CBufferedFile::Fill : end of file" : "CBufferedFile::Fill : fread failed"); + return false; + } else { + nSrcPos += read; + return true; + } + } + +public: + int nType; + int nVersion; + + CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) : + src(fileIn), nSrcPos(0), nReadPos(0), nReadLimit(std::numeric_limits::max()), nRewind(nRewindIn), vchBuf(nBufSize, 0), + state(0), exceptmask(std::ios_base::badbit | std::ios_base::failbit), nType(nTypeIn), nVersion(nVersionIn) { + } + + // check whether no error occurred + bool good() const { + return state == 0; + } + + // check whether we're at the end of the source file + bool eof() const { + return nReadPos == nSrcPos && feof(src); + } + + // read a number of bytes + CBufferedFile& read(char *pch, size_t nSize) { + if (nSize + nReadPos > nReadLimit) + throw std::ios_base::failure("Read attempted past buffer limit"); + if (nSize + nRewind > vchBuf.size()) + throw std::ios_base::failure("Read larger than buffer size"); + while (nSize > 0) { + if (nReadPos == nSrcPos) + Fill(); + unsigned int pos = (unsigned int)(nReadPos % vchBuf.size()); + size_t nNow = nSize; + if (nNow + pos > vchBuf.size()) + nNow = vchBuf.size() - pos; + if (nNow + nReadPos > nSrcPos) + nNow = (size_t)(nSrcPos - nReadPos); + memcpy(pch, &vchBuf[pos], nNow); + nReadPos += nNow; + pch += nNow; + nSize -= nNow; + } + return (*this); + } + + // return the current reading position + uint64_t GetPos() { + return nReadPos; + } + + // rewind to a given reading position + bool SetPos(uint64_t nPos) { + nReadPos = nPos; + if (nReadPos + nRewind < nSrcPos) { + nReadPos = nSrcPos - nRewind; + return false; + } else if (nReadPos > nSrcPos) { + nReadPos = nSrcPos; + return false; + } else { + return true; + } + } + + bool Seek(uint64_t nPos) { + long nLongPos = (long)nPos; + if (nPos != (uint64_t)nLongPos) + return false; + if (fseek(src, nLongPos, SEEK_SET)) + return false; + nLongPos = ftell(src); + nSrcPos = nLongPos; + nReadPos = nLongPos; + state = 0; + return true; + } + + // prevent reading beyond a certain position + // no argument removes the limit + bool SetLimit(uint64_t nPos = std::numeric_limits::max()) { + if (nPos < nReadPos) + return false; + nReadLimit = nPos; + return true; + } + + template + CBufferedFile& operator>>(T& obj) { + // Unserialize from this stream + ::Unserialize(*this, obj, nType, nVersion); + return (*this); + } + + // search for a given byte in the stream, and remain positioned on it + void FindByte(char ch) { + for ( ; ; ) { + if (nReadPos == nSrcPos) + Fill(); + if (vchBuf[nReadPos % vchBuf.size()] == ch) + break; + nReadPos++; + } + } +}; + #endif