X-Git-Url: https://git.novaco.in/?p=novacoin.git;a=blobdiff_plain;f=src%2Fdb.cpp;h=e12e11e9f77c6965144c64e5fca561d6028d2df4;hp=6c5e7918c874df8ab72fac69c07a43cb9afa109f;hb=1c1980bccd1bcccdb03c69ebbe03ad51e08f343a;hpb=d67b0434f24a629372000b0bd573ea9013ca3d90 diff --git a/src/db.cpp b/src/db.cpp index 6c5e791..e12e11e 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -3,133 +3,292 @@ // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "headers.h" #include "db.h" #include "net.h" -#include +#include "util.h" +#include "main.h" +#include "ui_interface.h" #include #include +#ifndef WIN32 +#include "sys/stat.h" +#endif + using namespace std; using namespace boost; unsigned int nWalletDBUpdated; -uint64 nAccountingEntryNumber = 0; - +extern bool fUseMemoryLog; // // CDB // -static CCriticalSection cs_db; -static bool fDbEnvInit = false; -DbEnv dbenv(0); -static map mapFileUseCount; -static map mapDb; -static int64 nTxn = 0; +CDBEnv bitdb; -static void EnvShutdown() +void CDBEnv::EnvShutdown() { if (!fDbEnvInit) return; fDbEnvInit = false; - try - { - dbenv.close(0); - } - catch (const DbException& e) + int ret = dbenv.close(0); + if (ret != 0) + printf("EnvShutdown exception: %s (%d)\n", DbEnv::strerror(ret), ret); + if (!fMockDb) + DbEnv(0).remove(strPath.c_str(), 0); +} + +CDBEnv::CDBEnv() : fDetachDB(false), fDbEnvInit(false), fMockDb(false), dbenv(DB_CXX_NO_EXCEPTIONS) { } + +CDBEnv::~CDBEnv() +{ + EnvShutdown(); +} + +void CDBEnv::Close() +{ + EnvShutdown(); +} + +bool CDBEnv::Open(boost::filesystem::path pathEnv_) +{ + if (fDbEnvInit) + return true; + + if (fShutdown) + return false; + + pathEnv = pathEnv_; + filesystem::path pathDataDir = pathEnv; + strPath = pathDataDir.string(); + filesystem::path pathLogDir = pathDataDir / "database"; + filesystem::create_directory(pathLogDir); + filesystem::path pathErrorFile = pathDataDir / "db.log"; + printf("dbenv.open LogDir=%s ErrorFile=%s\n", pathLogDir.string().c_str(), pathErrorFile.string().c_str()); + + unsigned int nEnvFlags = 0; + if (GetBoolArg("-privdb", true)) + nEnvFlags |= DB_PRIVATE; + + int nDbCache = GetArgInt("-dbcache", 25); + dbenv.set_lg_dir(pathLogDir.string().c_str()); + dbenv.set_cachesize(nDbCache / 1024, (nDbCache % 1024)*1048576, 1); + dbenv.set_lg_bsize(1048576); + dbenv.set_lg_max(10485760); + + // Bugfix: Bump lk_max_locks default to 537000, to safely handle reorgs with up to 5 blocks reversed + // dbenv.set_lk_max_locks(10000); + dbenv.set_lk_max_locks(537000); + + dbenv.set_lk_max_objects(10000); + dbenv.set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug + dbenv.set_flags(DB_AUTO_COMMIT, 1); + dbenv.set_flags(DB_TXN_WRITE_NOSYNC, 1); +#ifdef DB_LOG_AUTO_REMOVE + dbenv.log_set_config(DB_LOG_AUTO_REMOVE, 1); +#endif + int ret = dbenv.open(strPath.c_str(), + DB_CREATE | + DB_INIT_LOCK | + DB_INIT_LOG | + DB_INIT_MPOOL | + DB_INIT_TXN | + DB_THREAD | + DB_RECOVER | + nEnvFlags, + S_IRUSR | S_IWUSR); + if (ret != 0) + return error("CDB() : error %s (%d) opening database environment", DbEnv::strerror(ret), ret); + + fDbEnvInit = true; + fMockDb = false; + +#ifndef USE_LEVELDB + // Check that the number of locks is sufficient (to prevent chain fork possibility, read http://bitcoin.org/may15 for more info) + u_int32_t nMaxLocks; + if (!dbenv.get_lk_max_locks(&nMaxLocks)) { - printf("EnvShutdown exception: %s (%d)\n", e.what(), e.get_errno()); + int nBlocks, nDeepReorg; + std::string strMessage; + + nBlocks = nMaxLocks / 48768; + nDeepReorg = (nBlocks - 1) / 2; + + printf("Final lk_max_locks is %u, sufficient for (worst case) %d block%s in a single transaction (up to a %d-deep reorganization)\n", nMaxLocks, nBlocks, (nBlocks == 1) ? "" : "s", nDeepReorg); + if (nDeepReorg < 3) + { + if (nBlocks < 1) + strMessage = strprintf(_("Warning: DB_CONFIG has set_lk_max_locks %u, which may be too low for a single block. If this limit is reached, NovaCoin may stop working."), nMaxLocks); + else + strMessage = strprintf(_("Warning: DB_CONFIG has set_lk_max_locks %u, which may be too low for a common blockchain reorganization. If this limit is reached, NovaCoin may stop working."), nMaxLocks); + + strMiscWarning = strMessage; + printf("*** %s\n", strMessage.c_str()); + } } - DbEnv(0).remove(GetDataDir().c_str(), 0); +#endif + + return true; } -class CDBInit +void CDBEnv::MakeMock() { -public: - CDBInit() + if (fDbEnvInit) + throw runtime_error("CDBEnv::MakeMock(): already initialized"); + + if (fShutdown) + throw runtime_error("CDBEnv::MakeMock(): during shutdown"); + + printf("CDBEnv::MakeMock()\n"); + + dbenv.set_cachesize(1, 0, 1); + dbenv.set_lg_bsize(10485760*4); + dbenv.set_lg_max(10485760); + dbenv.set_lk_max_locks(10000); + dbenv.set_lk_max_objects(10000); + dbenv.set_flags(DB_AUTO_COMMIT, 1); +#ifdef DB_LOG_IN_MEMORY + dbenv.log_set_config(DB_LOG_IN_MEMORY, fUseMemoryLog ? 1 : 0); +#endif + int ret = dbenv.open(NULL, + DB_CREATE | + DB_INIT_LOCK | + DB_INIT_LOG | + DB_INIT_MPOOL | + DB_INIT_TXN | + DB_THREAD | + DB_PRIVATE, + S_IRUSR | S_IWUSR); + if (ret > 0) + throw runtime_error(strprintf("CDBEnv::MakeMock(): error %d opening database environment", ret)); + + fDbEnvInit = true; + fMockDb = true; +} + +CDBEnv::VerifyResult CDBEnv::Verify(std::string strFile, bool (*recoverFunc)(CDBEnv& dbenv, std::string strFile)) +{ + LOCK(cs_db); + assert(mapFileUseCount.count(strFile) == 0); + + Db db(&dbenv, 0); + int result = db.verify(strFile.c_str(), NULL, NULL, 0); + if (result == 0) + return VERIFY_OK; + else if (recoverFunc == NULL) + return RECOVER_FAIL; + + // Try to recover: + bool fRecovered = (*recoverFunc)(*this, strFile); + return (fRecovered ? RECOVER_OK : RECOVER_FAIL); +} + +bool CDBEnv::Salvage(std::string strFile, bool fAggressive, + std::vector& vResult) +{ + LOCK(cs_db); + assert(mapFileUseCount.count(strFile) == 0); + + u_int32_t flags = DB_SALVAGE; + if (fAggressive) flags |= DB_AGGRESSIVE; + + stringstream strDump; + + Db db(&dbenv, 0); + int result = db.verify(strFile.c_str(), NULL, &strDump, flags); + if (result != 0) { + printf("ERROR: db salvage failed\n"); + return false; } - ~CDBInit() + + // Format of bdb dump is ascii lines: + // header lines... + // HEADER=END + // hexadecimal key + // hexadecimal value + // ... repeated + // DATA=END + + string strLine; + while (!strDump.eof() && strLine != "HEADER=END") + getline(strDump, strLine); // Skip past header + + std::string keyHex, valueHex; + while (!strDump.eof() && keyHex != "DATA=END") { - EnvShutdown(); + getline(strDump, keyHex); + if (keyHex != "DATA=END") + { + getline(strDump, valueHex); + vResult.push_back(make_pair(ParseHex(keyHex),ParseHex(valueHex))); + } } + + return (result == 0); } -instance_of_cdbinit; -CDB::CDB(const char* pszFile, const char* pszMode) : pdb(NULL) +void CDBEnv::CheckpointLSN(std::string strFile) +{ + dbenv.txn_checkpoint(0, 0, 0); + if (fMockDb) + return; + dbenv.lsn_reset(strFile.c_str(), 0); +} + + +CDB::CDB(const char *pszFile, const char* pszMode) : + pdb(NULL), activeTxn(NULL) { int ret; if (pszFile == NULL) return; fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); - bool fCreate = strchr(pszMode, 'c'); + bool fCreate = strchr(pszMode, 'c') != NULL; unsigned int nFlags = DB_THREAD; if (fCreate) nFlags |= DB_CREATE; - CRITICAL_BLOCK(cs_db) { - if (!fDbEnvInit) - { - if (fShutdown) - return; - string strDataDir = GetDataDir(); - string strLogDir = strDataDir + "/database"; - filesystem::create_directory(strLogDir.c_str()); - string strErrorFile = strDataDir + "/db.log"; - printf("dbenv.open strLogDir=%s strErrorFile=%s\n", strLogDir.c_str(), strErrorFile.c_str()); - - int nDbCache = GetArg("-dbcache", 25); - dbenv.set_lg_dir(strLogDir.c_str()); - dbenv.set_cachesize(nDbCache / 1024, (nDbCache % 1024)*1048576, 1); - dbenv.set_lg_bsize(1048576); - dbenv.set_lg_max(10485760); - dbenv.set_lk_max_locks(10000); - dbenv.set_lk_max_objects(10000); - dbenv.set_errfile(fopen(strErrorFile.c_str(), "a")); /// debug - dbenv.set_flags(DB_AUTO_COMMIT, 1); - dbenv.log_set_config(DB_LOG_AUTO_REMOVE, 1); - ret = dbenv.open(strDataDir.c_str(), - DB_CREATE | - DB_INIT_LOCK | - DB_INIT_LOG | - DB_INIT_MPOOL | - DB_INIT_TXN | - DB_THREAD | - DB_RECOVER, - S_IRUSR | S_IWUSR); - if (ret > 0) - throw runtime_error(strprintf("CDB() : error %d opening database environment", ret)); - fDbEnvInit = true; - } + LOCK(bitdb.cs_db); + if (!bitdb.Open(GetDataDir())) + throw runtime_error("env open failed"); strFile = pszFile; - ++mapFileUseCount[strFile]; - pdb = mapDb[strFile]; + ++bitdb.mapFileUseCount[strFile]; + pdb = bitdb.mapDb[strFile]; if (pdb == NULL) { - pdb = new Db(&dbenv, 0); + pdb = new Db(&bitdb.dbenv, 0); + + bool fMockDb = bitdb.IsMock(); + if (fMockDb) + { + DbMpoolFile*mpf = pdb->get_mpf(); + ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); + if (ret != 0) + throw runtime_error(strprintf("CDB() : failed to configure for no temp file backing for database %s", pszFile)); + } ret = pdb->open(NULL, // Txn pointer - pszFile, // Filename + fMockDb ? NULL : pszFile, // Filename "main", // Logical db name DB_BTREE, // Database type nFlags, // Flags 0); - if (ret > 0) + if (ret != 0) { delete pdb; pdb = NULL; - CRITICAL_BLOCK(cs_db) - --mapFileUseCount[strFile]; - strFile = ""; + --bitdb.mapFileUseCount[strFile]; + strFile.clear(); throw runtime_error(strprintf("CDB() : can't open database file %s, error %d", pszFile, ret)); } @@ -141,45 +300,49 @@ CDB::CDB(const char* pszFile, const char* pszMode) : pdb(NULL) fReadOnly = fTmp; } - mapDb[strFile] = pdb; + bitdb.mapDb[strFile] = pdb; } } } +static bool IsChainFile(std::string strFile) +{ + if (strFile == "blkindex.dat") + return true; + + return false; +} + void CDB::Close() { if (!pdb) return; - if (!vTxn.empty()) - vTxn.front()->abort(); - vTxn.clear(); + if (activeTxn) + activeTxn->abort(); + activeTxn = NULL; pdb = NULL; // Flush database activity from memory pool to disk log unsigned int nMinutes = 0; if (fReadOnly) nMinutes = 1; - if (strFile == "addr.dat") + if (IsChainFile(strFile)) nMinutes = 2; - if (strFile == "blkindex.dat" && IsInitialBlockDownload()) + if (IsChainFile(strFile) && IsInitialBlockDownload()) nMinutes = 5; - if (nMinutes == 0 || nTxn > 200000) + bitdb.dbenv.txn_checkpoint(nMinutes ? GetArgUInt("-dblogsize", 100)*1024 : 0, nMinutes, 0); + { - nTxn = 0; - nMinutes = 0; + LOCK(bitdb.cs_db); + --bitdb.mapFileUseCount[strFile]; } - - dbenv.txn_checkpoint(0, nMinutes, 0); - - CRITICAL_BLOCK(cs_db) - --mapFileUseCount[strFile]; } -void static CloseDb(const string& strFile) +void CDBEnv::CloseDb(const string& strFile) { - CRITICAL_BLOCK(cs_db) { + LOCK(cs_db); if (mapDb[strFile] != NULL) { // Close the database handle @@ -191,27 +354,35 @@ void static CloseDb(const string& strFile) } } +bool CDBEnv::RemoveDb(const string& strFile) +{ + this->CloseDb(strFile); + + LOCK(cs_db); + int rc = dbenv.dbremove(NULL, strFile.c_str(), NULL, DB_AUTO_COMMIT); + return (rc == 0); +} + bool CDB::Rewrite(const string& strFile, const char* pszSkip) { while (!fShutdown) { - CRITICAL_BLOCK(cs_db) { - if (!mapFileUseCount.count(strFile) || mapFileUseCount[strFile] == 0) + LOCK(bitdb.cs_db); + if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0) { // Flush log data to the dat file - CloseDb(strFile); - dbenv.txn_checkpoint(0, 0, 0); - dbenv.lsn_reset(strFile.c_str(), 0); - mapFileUseCount.erase(strFile); + bitdb.CloseDb(strFile); + bitdb.CheckpointLSN(strFile); + bitdb.mapFileUseCount.erase(strFile); bool fSuccess = true; printf("Rewriting %s...\n", strFile.c_str()); string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} CDB db(strFile.c_str(), "r"); - Db* pdbCopy = new Db(&dbenv, 0); - + Db* pdbCopy = new Db(&bitdb.dbenv, 0); + int ret = pdbCopy->open(NULL, // Txn pointer strFileRes.c_str(), // Filename "main", // Logical db name @@ -223,13 +394,13 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip) printf("Cannot create database file %s\n", strFileRes.c_str()); fSuccess = false; } - + Dbc* pcursor = db.GetCursor(); - if (pcursor) + if (pcursor) { while (fSuccess) { - CDataStream ssKey; - CDataStream ssValue; + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + CDataStream ssValue(SER_DISK, CLIENT_VERSION); int ret = db.ReadAtCursor(pcursor, ssKey, ssValue, DB_NEXT); if (ret == DB_NOTFOUND) { @@ -242,9 +413,14 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip) fSuccess = false; break; } - if (pszSkip && - strncmp(&ssKey[0], pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) - continue; + + if (pszSkip != NULL) + { + size_t pszSkipLen = strlen(pszSkip); + if (strncmp(&ssKey[0], pszSkip, std::min(ssKey.size(), pszSkipLen)) == 0) + continue; + } + if (strncmp(&ssKey[0], "\x07version", 8) == 0) { // Update version: @@ -257,10 +433,11 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip) if (ret2 > 0) fSuccess = false; } + } if (fSuccess) { db.Close(); - CloseDb(strFile); + bitdb.CloseDb(strFile); if (pdbCopy->close(0)) fSuccess = false; delete pdbCopy; @@ -268,10 +445,10 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip) } if (fSuccess) { - Db dbA(&dbenv, 0); + Db dbA(&bitdb.dbenv, 0); if (dbA.remove(strFile.c_str(), NULL, 0)) fSuccess = false; - Db dbB(&dbenv, 0); + Db dbB(&bitdb.dbenv, 0); if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0)) fSuccess = false; } @@ -286,15 +463,16 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip) } -void DBFlush(bool fShutdown) +void CDBEnv::Flush(bool fShutdown) { + int64_t nStart = GetTimeMillis(); // Flush log data to the actual data file // on all files that are not in use - printf("DBFlush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started"); + printf("Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started"); if (!fDbEnvInit) return; - CRITICAL_BLOCK(cs_db) { + LOCK(cs_db); map::iterator mi = mapFileUseCount.begin(); while (mi != mapFileUseCount.end()) { @@ -305,917 +483,131 @@ void DBFlush(bool fShutdown) { // Move log data to the dat file CloseDb(strFile); + printf("%s checkpoint\n", strFile.c_str()); dbenv.txn_checkpoint(0, 0, 0); - printf("%s flush\n", strFile.c_str()); - dbenv.lsn_reset(strFile.c_str(), 0); + if (!IsChainFile(strFile) || fDetachDB) { + printf("%s detach\n", strFile.c_str()); + if (!fMockDb) + dbenv.lsn_reset(strFile.c_str(), 0); + } + printf("%s closed\n", strFile.c_str()); mapFileUseCount.erase(mi++); } else mi++; } + printf("DBFlush(%s)%s ended %15" PRId64 "ms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started", GetTimeMillis() - nStart); if (fShutdown) { char** listp; if (mapFileUseCount.empty()) { dbenv.log_archive(&listp, DB_ARCH_REMOVE); - EnvShutdown(); + Close(); } } } } - - - - // -// CTxDB +// CAddrDB // -bool CTxDB::ReadTxIndex(uint256 hash, CTxIndex& txindex) -{ - assert(!fClient); - txindex.SetNull(); - return Read(make_pair(string("tx"), hash), txindex); -} - -bool CTxDB::UpdateTxIndex(uint256 hash, const CTxIndex& txindex) -{ - assert(!fClient); - nTxn++; - return Write(make_pair(string("tx"), hash), txindex); -} - -bool CTxDB::AddTxIndex(const CTransaction& tx, const CDiskTxPos& pos, int nHeight) -{ - assert(!fClient); - - // Add to tx index - uint256 hash = tx.GetHash(); - CTxIndex txindex(pos, tx.vout.size()); - nTxn++; - return Write(make_pair(string("tx"), hash), txindex); -} - -bool CTxDB::EraseTxIndex(const CTransaction& tx) -{ - assert(!fClient); - uint256 hash = tx.GetHash(); - - return Erase(make_pair(string("tx"), hash)); -} - -bool CTxDB::ContainsTx(uint256 hash) -{ - assert(!fClient); - return Exists(make_pair(string("tx"), hash)); -} - -bool CTxDB::ReadOwnerTxes(uint160 hash160, int nMinHeight, vector& vtx) -{ - assert(!fClient); - vtx.clear(); - - // Get cursor - Dbc* pcursor = GetCursor(); - if (!pcursor) - return false; - - unsigned int fFlags = DB_SET_RANGE; - loop - { - // Read next record - CDataStream ssKey; - if (fFlags == DB_SET_RANGE) - ssKey << string("owner") << hash160 << CDiskTxPos(0, 0, 0); - CDataStream ssValue; - int ret = ReadAtCursor(pcursor, ssKey, ssValue, fFlags); - fFlags = DB_NEXT; - if (ret == DB_NOTFOUND) - break; - else if (ret != 0) - { - pcursor->close(); - return false; - } - - // Unserialize - string strType; - uint160 hashItem; - CDiskTxPos pos; - int nItemHeight; - - try { - ssKey >> strType >> hashItem >> pos; - ssValue >> nItemHeight; - } - catch (std::exception &e) { - return error("%s() : deserialize error", __PRETTY_FUNCTION__); - } - - // Read transaction - if (strType != "owner" || hashItem != hash160) - break; - if (nItemHeight >= nMinHeight) - { - vtx.resize(vtx.size()+1); - if (!vtx.back().ReadFromDisk(pos)) - { - pcursor->close(); - return false; - } - } - } - - pcursor->close(); - return true; -} - -bool CTxDB::ReadDiskTx(uint256 hash, CTransaction& tx, CTxIndex& txindex) -{ - assert(!fClient); - tx.SetNull(); - if (!ReadTxIndex(hash, txindex)) - return false; - return (tx.ReadFromDisk(txindex.pos)); -} - -bool CTxDB::ReadDiskTx(uint256 hash, CTransaction& tx) -{ - CTxIndex txindex; - return ReadDiskTx(hash, tx, txindex); -} - -bool CTxDB::ReadDiskTx(COutPoint outpoint, CTransaction& tx, CTxIndex& txindex) -{ - return ReadDiskTx(outpoint.hash, tx, txindex); -} - -bool CTxDB::ReadDiskTx(COutPoint outpoint, CTransaction& tx) -{ - CTxIndex txindex; - return ReadDiskTx(outpoint.hash, tx, txindex); -} - -bool CTxDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) -{ - return Write(make_pair(string("blockindex"), blockindex.GetBlockHash()), blockindex); -} - -bool CTxDB::EraseBlockIndex(uint256 hash) -{ - return Erase(make_pair(string("blockindex"), hash)); -} -bool CTxDB::ReadHashBestChain(uint256& hashBestChain) +CAddrDB::CAddrDB() { - return Read(string("hashBestChain"), hashBestChain); + pathAddr = GetDataDir() / "peers.dat"; } -bool CTxDB::WriteHashBestChain(uint256 hashBestChain) +bool CAddrDB::Write(const CAddrMan& addr) { - return Write(string("hashBestChain"), hashBestChain); -} + // Generate random temporary filename + unsigned short randv = 0; + RAND_bytes((unsigned char *)&randv, sizeof(randv)); + std::string tmpfn = strprintf("peers.dat.%04x", randv); -bool CTxDB::ReadBestInvalidWork(CBigNum& bnBestInvalidWork) -{ - return Read(string("bnBestInvalidWork"), bnBestInvalidWork); -} + // serialize addresses, checksum data up to that point, then append csum + CDataStream ssPeers(SER_DISK, CLIENT_VERSION); + ssPeers << FLATDATA(pchMessageStart); + ssPeers << addr; + uint256 hash = Hash(ssPeers.begin(), ssPeers.end()); + ssPeers << hash; -bool CTxDB::WriteBestInvalidWork(CBigNum bnBestInvalidWork) -{ - return Write(string("bnBestInvalidWork"), bnBestInvalidWork); -} - -CBlockIndex static * InsertBlockIndex(uint256 hash) -{ - if (hash == 0) - return NULL; - - // Return existing - map::iterator mi = mapBlockIndex.find(hash); - if (mi != mapBlockIndex.end()) - return (*mi).second; - - // Create new - CBlockIndex* pindexNew = new CBlockIndex(); - if (!pindexNew) - throw runtime_error("LoadBlockIndex() : new CBlockIndex failed"); - mi = mapBlockIndex.insert(make_pair(hash, pindexNew)).first; - pindexNew->phashBlock = &((*mi).first); - - return pindexNew; -} - -bool CTxDB::LoadBlockIndex() -{ - // Get database cursor - Dbc* pcursor = GetCursor(); - if (!pcursor) - return false; - - // Load mapBlockIndex - unsigned int fFlags = DB_SET_RANGE; - loop - { - // Read next record - CDataStream ssKey; - if (fFlags == DB_SET_RANGE) - ssKey << make_pair(string("blockindex"), uint256(0)); - CDataStream ssValue; - int ret = ReadAtCursor(pcursor, ssKey, ssValue, fFlags); - fFlags = DB_NEXT; - if (ret == DB_NOTFOUND) - break; - else if (ret != 0) - return false; - - // Unserialize - - try { - string strType; - ssKey >> strType; - if (strType == "blockindex") - { - CDiskBlockIndex diskindex; - ssValue >> diskindex; - - // Construct block index object - CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); - pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); - pindexNew->pnext = InsertBlockIndex(diskindex.hashNext); - pindexNew->nFile = diskindex.nFile; - pindexNew->nBlockPos = diskindex.nBlockPos; - pindexNew->nHeight = diskindex.nHeight; - pindexNew->nVersion = diskindex.nVersion; - pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; - pindexNew->nTime = diskindex.nTime; - pindexNew->nBits = diskindex.nBits; - pindexNew->nNonce = diskindex.nNonce; - - // Watch for genesis block - if (pindexGenesisBlock == NULL && diskindex.GetBlockHash() == hashGenesisBlock) - pindexGenesisBlock = pindexNew; - - if (!pindexNew->CheckIndex()) - return error("LoadBlockIndex() : CheckIndex failed at %d", pindexNew->nHeight); - } - else - { - break; - } - } // try - catch (std::exception &e) { - return error("%s() : deserialize error", __PRETTY_FUNCTION__); - } - } - pcursor->close(); + // open temp output file, and associate with CAutoFile + boost::filesystem::path pathTmp = GetDataDir() / tmpfn; + FILE *file = fopen(pathTmp.string().c_str(), "wb"); + CAutoFile fileout = CAutoFile(file, SER_DISK, CLIENT_VERSION); + if (!fileout) + return error("CAddrman::Write() : open failed"); - // Calculate bnChainWork - vector > vSortedByHeight; - vSortedByHeight.reserve(mapBlockIndex.size()); - BOOST_FOREACH(const PAIRTYPE(uint256, CBlockIndex*)& item, mapBlockIndex) - { - CBlockIndex* pindex = item.second; - vSortedByHeight.push_back(make_pair(pindex->nHeight, pindex)); + // Write and commit header, data + try { + fileout << ssPeers; } - sort(vSortedByHeight.begin(), vSortedByHeight.end()); - BOOST_FOREACH(const PAIRTYPE(int, CBlockIndex*)& item, vSortedByHeight) - { - CBlockIndex* pindex = item.second; - pindex->bnChainWork = (pindex->pprev ? pindex->pprev->bnChainWork : 0) + pindex->GetBlockWork(); + catch (const std::exception&) { + return error("CAddrman::Write() : I/O error"); } + FileCommit(fileout); + fileout.fclose(); - // Load hashBestChain pointer to end of best chain - if (!ReadHashBestChain(hashBestChain)) - { - if (pindexGenesisBlock == NULL) - return true; - return error("CTxDB::LoadBlockIndex() : hashBestChain not loaded"); - } - if (!mapBlockIndex.count(hashBestChain)) - return error("CTxDB::LoadBlockIndex() : hashBestChain not found in the block index"); - pindexBest = mapBlockIndex[hashBestChain]; - nBestHeight = pindexBest->nHeight; - bnBestChainWork = pindexBest->bnChainWork; - printf("LoadBlockIndex(): hashBestChain=%s height=%d\n", hashBestChain.ToString().substr(0,20).c_str(), nBestHeight); - - // Load bnBestInvalidWork, OK if it doesn't exist - ReadBestInvalidWork(bnBestInvalidWork); - - // Verify blocks in the best chain - int nCheckLevel = GetArg("-checklevel", 1); - int nCheckDepth = GetArg( "-checkblocks", 2500); - if (nCheckDepth == 0) - nCheckDepth = 1000000000; // suffices until the year 19000 - if (nCheckDepth > nBestHeight) - nCheckDepth = nBestHeight; - printf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); - CBlockIndex* pindexFork = NULL; - map, CBlockIndex*> mapBlockPos; - for (CBlockIndex* pindex = pindexBest; pindex && pindex->pprev; pindex = pindex->pprev) - { - if (pindex->nHeight < nBestHeight-nCheckDepth) - break; - CBlock block; - if (!block.ReadFromDisk(pindex)) - return error("LoadBlockIndex() : block.ReadFromDisk failed"); - // check level 1: verify block validity - if (nCheckLevel>0 && !block.CheckBlock()) - { - printf("LoadBlockIndex() : *** found bad block at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString().c_str()); - pindexFork = pindex->pprev; - } - // check level 2: verify transaction index validity - if (nCheckLevel>1) - { - pair pos = make_pair(pindex->nFile, pindex->nBlockPos); - mapBlockPos[pos] = pindex; - BOOST_FOREACH(const CTransaction &tx, block.vtx) - { - uint256 hashTx = tx.GetHash(); - CTxIndex txindex; - if (ReadTxIndex(hashTx, txindex)) - { - // check level 3: checker transaction hashes - if (nCheckLevel>2 || pindex->nFile != txindex.pos.nFile || pindex->nBlockPos != txindex.pos.nBlockPos) - { - // either an error or a duplicate transaction - CTransaction txFound; - if (!txFound.ReadFromDisk(txindex.pos)) - { - printf("LoadBlockIndex() : *** cannot read mislocated transaction %s\n", hashTx.ToString().c_str()); - pindexFork = pindex->pprev; - } - else - if (txFound.GetHash() != hashTx) // not a duplicate tx - { - printf("LoadBlockIndex(): *** invalid tx position for %s\n", hashTx.ToString().c_str()); - pindexFork = pindex->pprev; - } - } - // check level 4: check whether spent txouts were spent within the main chain - int nOutput = 0; - if (nCheckLevel>3) - { - BOOST_FOREACH(const CDiskTxPos &txpos, txindex.vSpent) - { - if (!txpos.IsNull()) - { - pair posFind = make_pair(txpos.nFile, txpos.nBlockPos); - if (!mapBlockPos.count(posFind)) - { - printf("LoadBlockIndex(): *** found bad spend at %d, hashBlock=%s, hashTx=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString().c_str(), hashTx.ToString().c_str()); - pindexFork = pindex->pprev; - } - // check level 6: check whether spent txouts were spent by a valid transaction that consume them - if (nCheckLevel>5) - { - CTransaction txSpend; - if (!txSpend.ReadFromDisk(txpos)) - { - printf("LoadBlockIndex(): *** cannot read spending transaction of %s:%i from disk\n", hashTx.ToString().c_str(), nOutput); - pindexFork = pindex->pprev; - } - else if (!txSpend.CheckTransaction()) - { - printf("LoadBlockIndex(): *** spending transaction of %s:%i is invalid\n", hashTx.ToString().c_str(), nOutput); - pindexFork = pindex->pprev; - } - else - { - bool fFound = false; - BOOST_FOREACH(const CTxIn &txin, txSpend.vin) - if (txin.prevout.hash == hashTx && txin.prevout.n == nOutput) - fFound = true; - if (!fFound) - { - printf("LoadBlockIndex(): *** spending transaction of %s:%i does not spend it\n", hashTx.ToString().c_str(), nOutput); - pindexFork = pindex->pprev; - } - } - } - } - nOutput++; - } - } - } - // check level 5: check whether all prevouts are marked spent - if (nCheckLevel>4) - { - BOOST_FOREACH(const CTxIn &txin, tx.vin) - { - CTxIndex txindex; - if (ReadTxIndex(txin.prevout.hash, txindex)) - if (txindex.vSpent.size()-1 < txin.prevout.n || txindex.vSpent[txin.prevout.n].IsNull()) - { - printf("LoadBlockIndex(): *** found unspent prevout %s:%i in %s\n", txin.prevout.hash.ToString().c_str(), txin.prevout.n, hashTx.ToString().c_str()); - pindexFork = pindex->pprev; - } - } - } - } - } - } - if (pindexFork) - { - // Reorg back to the fork - printf("LoadBlockIndex() : *** moving best chain pointer back to block %d\n", pindexFork->nHeight); - CBlock block; - if (!block.ReadFromDisk(pindexFork)) - return error("LoadBlockIndex() : block.ReadFromDisk failed"); - CTxDB txdb; - block.SetBestChain(txdb, pindexFork); - } + // replace existing peers.dat, if any, with new peers.dat.XXXX + if (!RenameOver(pathTmp, pathAddr)) + return error("CAddrman::Write() : Rename-into-place failed"); return true; } - - - - -// -// CAddrDB -// - -bool CAddrDB::WriteAddrman(const CAddrMan& addrman) -{ - return Write(string("addrman"), addrman); -} - -bool CAddrDB::LoadAddresses() +bool CAddrDB::Read(CAddrMan& addr) { - if (Read(string("addrman"), addrman)) - { - printf("Loaded %i addresses\n", addrman.size()); - return true; - } - - // Read pre-0.6 addr records + // open input file, and associate with CAutoFile + FILE *file = fopen(pathAddr.string().c_str(), "rb"); + CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION); + if (!filein) + return error("CAddrman::Read() : open failed"); - vector vAddr; - vector > vDelete; + // use file size to size memory buffer + int fileSize = GetFilesize(filein); + int dataSize = fileSize - sizeof(uint256); + //Don't try to resize to a negative number if file is small + if ( dataSize < 0 ) dataSize = 0; + vector vchData; + vchData.resize(dataSize); + uint256 hashIn; - // Get cursor - Dbc* pcursor = GetCursor(); - if (!pcursor) - return false; - - loop - { - // Read next record - CDataStream ssKey; - CDataStream ssValue; - int ret = ReadAtCursor(pcursor, ssKey, ssValue); - if (ret == DB_NOTFOUND) - break; - else if (ret != 0) - return false; - - // Unserialize - string strType; - ssKey >> strType; - if (strType == "addr") - { - CAddress addr; - ssValue >> addr; - vAddr.push_back(addr); - } + // read data and checksum from file + try { + filein.read((char *)&vchData[0], dataSize); + filein >> hashIn; } - pcursor->close(); - - addrman.Add(vAddr, CNetAddr("0.0.0.0")); - printf("Loaded %i addresses\n", addrman.size()); - - // Note: old records left; we ran into hangs-on-startup - // bugs for some users who (we think) were running after - // an unclean shutdown. - - return true; -} - -bool LoadAddresses() -{ - return CAddrDB("cr+").LoadAddresses(); -} - - - - -// -// CWalletDB -// - -bool CWalletDB::WriteName(const string& strAddress, const string& strName) -{ - nWalletDBUpdated++; - return Write(make_pair(string("name"), strAddress), strName); -} - -bool CWalletDB::EraseName(const string& strAddress) -{ - // This should only be used for sending addresses, never for receiving addresses, - // receiving addresses must always have an address book entry if they're not change return. - nWalletDBUpdated++; - return Erase(make_pair(string("name"), strAddress)); -} - -bool CWalletDB::ReadAccount(const string& strAccount, CAccount& account) -{ - account.SetNull(); - return Read(make_pair(string("acc"), strAccount), account); -} - -bool CWalletDB::WriteAccount(const string& strAccount, const CAccount& account) -{ - return Write(make_pair(string("acc"), strAccount), account); -} - -bool CWalletDB::WriteAccountingEntry(const CAccountingEntry& acentry) -{ - return Write(boost::make_tuple(string("acentry"), acentry.strAccount, ++nAccountingEntryNumber), acentry); -} - -int64 CWalletDB::GetAccountCreditDebit(const string& strAccount) -{ - list entries; - ListAccountCreditDebit(strAccount, entries); - - int64 nCreditDebit = 0; - BOOST_FOREACH (const CAccountingEntry& entry, entries) - nCreditDebit += entry.nCreditDebit; - - return nCreditDebit; -} - -void CWalletDB::ListAccountCreditDebit(const string& strAccount, list& entries) -{ - bool fAllAccounts = (strAccount == "*"); - - Dbc* pcursor = GetCursor(); - if (!pcursor) - throw runtime_error("CWalletDB::ListAccountCreditDebit() : cannot create DB cursor"); - unsigned int fFlags = DB_SET_RANGE; - loop - { - // Read next record - CDataStream ssKey; - if (fFlags == DB_SET_RANGE) - ssKey << boost::make_tuple(string("acentry"), (fAllAccounts? string("") : strAccount), uint64(0)); - CDataStream ssValue; - int ret = ReadAtCursor(pcursor, ssKey, ssValue, fFlags); - fFlags = DB_NEXT; - if (ret == DB_NOTFOUND) - break; - else if (ret != 0) - { - pcursor->close(); - throw runtime_error("CWalletDB::ListAccountCreditDebit() : error scanning DB"); - } - - // Unserialize - string strType; - ssKey >> strType; - if (strType != "acentry") - break; - CAccountingEntry acentry; - ssKey >> acentry.strAccount; - if (!fAllAccounts && acentry.strAccount != strAccount) - break; - - ssValue >> acentry; - entries.push_back(acentry); + catch (const std::exception&) { + return error("CAddrman::Read() 2 : I/O error or stream data corrupted"); } + filein.fclose(); - pcursor->close(); -} - - -int CWalletDB::LoadWallet(CWallet* pwallet) -{ - pwallet->vchDefaultKey.clear(); - int nFileVersion = 0; - vector vWalletUpgrade; - bool fIsEncrypted = false; + CDataStream ssPeers(vchData, SER_DISK, CLIENT_VERSION); - //// todo: shouldn't we catch exceptions and try to recover and continue? - CRITICAL_BLOCK(pwallet->cs_wallet) - { - int nMinVersion = 0; - if (Read((string)"minversion", nMinVersion)) - { - if (nMinVersion > CLIENT_VERSION) - return DB_TOO_NEW; - pwallet->LoadMinVersion(nMinVersion); - } + // verify stored checksum matches input data + uint256 hashTmp = Hash(ssPeers.begin(), ssPeers.end()); + if (hashIn != hashTmp) + return error("CAddrman::Read() : checksum mismatch; data corrupted"); - // Get cursor - Dbc* pcursor = GetCursor(); - if (!pcursor) - { - printf("Error getting wallet database cursor\n"); - return DB_CORRUPT; - } + unsigned char pchMsgTmp[4]; + try { + // de-serialize file header (pchMessageStart magic number) and + ssPeers >> FLATDATA(pchMsgTmp); - loop - { - // Read next record - CDataStream ssKey; - CDataStream ssValue; - int ret = ReadAtCursor(pcursor, ssKey, ssValue); - if (ret == DB_NOTFOUND) - break; - else if (ret != 0) - { - printf("Error reading next record from wallet database\n"); - return DB_CORRUPT; - } - - // Unserialize - // Taking advantage of the fact that pair serialization - // is just the two items serialized one after the other - string strType; - ssKey >> strType; - if (strType == "name") - { - string strAddress; - ssKey >> strAddress; - ssValue >> pwallet->mapAddressBook[strAddress]; - } - else if (strType == "tx") - { - uint256 hash; - ssKey >> hash; - CWalletTx& wtx = pwallet->mapWallet[hash]; - ssValue >> wtx; - wtx.BindWallet(pwallet); - - if (wtx.GetHash() != hash) - printf("Error in wallet.dat, hash mismatch\n"); - - // Undo serialize changes in 31600 - if (31404 <= wtx.fTimeReceivedIsTxTime && wtx.fTimeReceivedIsTxTime <= 31703) - { - if (!ssValue.empty()) - { - char fTmp; - char fUnused; - ssValue >> fTmp >> fUnused >> wtx.strFromAccount; - printf("LoadWallet() upgrading tx ver=%d %d '%s' %s\n", wtx.fTimeReceivedIsTxTime, fTmp, wtx.strFromAccount.c_str(), hash.ToString().c_str()); - wtx.fTimeReceivedIsTxTime = fTmp; - } - else - { - printf("LoadWallet() repairing tx ver=%d %s\n", wtx.fTimeReceivedIsTxTime, hash.ToString().c_str()); - wtx.fTimeReceivedIsTxTime = 0; - } - vWalletUpgrade.push_back(hash); - } + // verify the network matches ours + if (memcmp(pchMsgTmp, pchMessageStart, sizeof(pchMsgTmp))) + return error("CAddrman::Read() : invalid network magic number"); - //// debug print - //printf("LoadWallet %s\n", wtx.GetHash().ToString().c_str()); - //printf(" %12"PRI64d" %s %s %s\n", - // wtx.vout[0].nValue, - // DateTimeStrFormat("%x %H:%M:%S", wtx.GetBlockTime()).c_str(), - // wtx.hashBlock.ToString().substr(0,20).c_str(), - // wtx.mapValue["message"].c_str()); - } - else if (strType == "acentry") - { - string strAccount; - ssKey >> strAccount; - uint64 nNumber; - ssKey >> nNumber; - if (nNumber > nAccountingEntryNumber) - nAccountingEntryNumber = nNumber; - } - else if (strType == "key" || strType == "wkey") - { - vector vchPubKey; - ssKey >> vchPubKey; - CKey key; - if (strType == "key") - { - CPrivKey pkey; - ssValue >> pkey; - key.SetPubKey(vchPubKey); - key.SetPrivKey(pkey); - if (key.GetPubKey() != vchPubKey) - { - printf("Error reading wallet database: CPrivKey pubkey inconsistency\n"); - return DB_CORRUPT; - } - if (!key.IsValid()) - { - printf("Error reading wallet database: invalid CPrivKey\n"); - return DB_CORRUPT; - } - } - else - { - CWalletKey wkey; - ssValue >> wkey; - key.SetPubKey(vchPubKey); - key.SetPrivKey(wkey.vchPrivKey); - if (key.GetPubKey() != vchPubKey) - { - printf("Error reading wallet database: CWalletKey pubkey inconsistency\n"); - return DB_CORRUPT; - } - if (!key.IsValid()) - { - printf("Error reading wallet database: invalid CWalletKey\n"); - return DB_CORRUPT; - } - } - if (!pwallet->LoadKey(key)) - { - printf("Error reading wallet database: LoadKey failed\n"); - return DB_CORRUPT; - } - } - else if (strType == "mkey") - { - unsigned int nID; - ssKey >> nID; - CMasterKey kMasterKey; - ssValue >> kMasterKey; - if(pwallet->mapMasterKeys.count(nID) != 0) - { - printf("Error reading wallet database: duplicate CMasterKey id %u\n", nID); - return DB_CORRUPT; - } - pwallet->mapMasterKeys[nID] = kMasterKey; - if (pwallet->nMasterKeyMaxID < nID) - pwallet->nMasterKeyMaxID = nID; - } - else if (strType == "ckey") - { - vector vchPubKey; - ssKey >> vchPubKey; - vector vchPrivKey; - ssValue >> vchPrivKey; - if (!pwallet->LoadCryptedKey(vchPubKey, vchPrivKey)) - { - printf("Error reading wallet database: LoadCryptedKey failed\n"); - return DB_CORRUPT; - } - fIsEncrypted = true; - } - else if (strType == "defaultkey") - { - ssValue >> pwallet->vchDefaultKey; - } - else if (strType == "pool") - { - int64 nIndex; - ssKey >> nIndex; - pwallet->setKeyPool.insert(nIndex); - } - else if (strType == "version") - { - ssValue >> nFileVersion; - if (nFileVersion == 10300) - nFileVersion = 300; - } - else if (strType == "cscript") - { - uint160 hash; - ssKey >> hash; - CScript script; - ssValue >> script; - if (!pwallet->LoadCScript(script)) - { - printf("Error reading wallet database: LoadCScript failed\n"); - return DB_CORRUPT; - } - } - } - pcursor->close(); + // de-serialize address data into one CAddrMan object + ssPeers >> addr; } - - BOOST_FOREACH(uint256 hash, vWalletUpgrade) - WriteTx(hash, pwallet->mapWallet[hash]); - - printf("nFileVersion = %d\n", nFileVersion); - - - // Rewrite encrypted wallets of versions 0.4.0 and 0.5.0rc: - if (fIsEncrypted && (nFileVersion == 40000 || nFileVersion == 50000)) - return DB_NEED_REWRITE; - - if (nFileVersion < CLIENT_VERSION) // Update - { - // Get rid of old debug.log file in current directory - if (nFileVersion <= 105 && !pszSetDataDir[0]) - unlink("debug.log"); - - WriteVersion(CLIENT_VERSION); + catch (const std::exception&) { + return error("CAddrman::Read() : I/O error or stream data corrupted"); } - return DB_LOAD_OK; -} - -void ThreadFlushWalletDB(void* parg) -{ - const string& strFile = ((const string*)parg)[0]; - static bool fOneThread; - if (fOneThread) - return; - fOneThread = true; - if (!GetBoolArg("-flushwallet", true)) - return; - - unsigned int nLastSeen = nWalletDBUpdated; - unsigned int nLastFlushed = nWalletDBUpdated; - int64 nLastWalletUpdate = GetTime(); - while (!fShutdown) - { - Sleep(500); - - if (nLastSeen != nWalletDBUpdated) - { - nLastSeen = nWalletDBUpdated; - nLastWalletUpdate = GetTime(); - } - - if (nLastFlushed != nWalletDBUpdated && GetTime() - nLastWalletUpdate >= 2) - { - TRY_CRITICAL_BLOCK(cs_db) - { - // Don't do this if any databases are in use - int nRefCount = 0; - map::iterator mi = mapFileUseCount.begin(); - while (mi != mapFileUseCount.end()) - { - nRefCount += (*mi).second; - mi++; - } - - if (nRefCount == 0 && !fShutdown) - { - map::iterator mi = mapFileUseCount.find(strFile); - if (mi != mapFileUseCount.end()) - { - printf("%s ", DateTimeStrFormat("%x %H:%M:%S", GetTime()).c_str()); - printf("Flushing wallet.dat\n"); - nLastFlushed = nWalletDBUpdated; - int64 nStart = GetTimeMillis(); - - // Flush wallet.dat so it's self contained - CloseDb(strFile); - dbenv.txn_checkpoint(0, 0, 0); - dbenv.lsn_reset(strFile.c_str(), 0); - - mapFileUseCount.erase(mi++); - printf("Flushed wallet.dat %"PRI64d"ms\n", GetTimeMillis() - nStart); - } - } - } - } - } + return true; } -bool BackupWallet(const CWallet& wallet, const string& strDest) -{ - if (!wallet.fFileBacked) - return false; - while (!fShutdown) - { - CRITICAL_BLOCK(cs_db) - { - if (!mapFileUseCount.count(wallet.strWalletFile) || mapFileUseCount[wallet.strWalletFile] == 0) - { - // Flush log data to the dat file - CloseDb(wallet.strWalletFile); - dbenv.txn_checkpoint(0, 0, 0); - dbenv.lsn_reset(wallet.strWalletFile.c_str(), 0); - mapFileUseCount.erase(wallet.strWalletFile); - - // Copy wallet.dat - filesystem::path pathSrc(GetDataDir() + "/" + wallet.strWalletFile); - filesystem::path pathDest(strDest); - if (filesystem::is_directory(pathDest)) - pathDest = pathDest / wallet.strWalletFile; - - try { -#if BOOST_VERSION >= 104000 - filesystem::copy_file(pathSrc, pathDest, filesystem::copy_option::overwrite_if_exists); -#else - filesystem::copy_file(pathSrc, pathDest); -#endif - printf("copied wallet.dat to %s\n", pathDest.string().c_str()); - return true; - } catch(const filesystem::filesystem_error &e) { - printf("error copying wallet.dat to %s - %s\n", pathDest.string().c_str(), e.what()); - return false; - } - } - } - Sleep(100); - } - return false; -}