1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2012 The Bitcoin developers
3 // Distributed under the MIT/X11 software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
11 #include <boost/filesystem.hpp>
12 #include <boost/filesystem/fstream.hpp>
21 unsigned int nWalletDBUpdated;
22 extern bool fUseMemoryLog;
31 void CDBEnv::EnvShutdown()
37 int ret = dbenv.close(0);
39 printf("EnvShutdown exception: %s (%d)\n", DbEnv::strerror(ret), ret);
41 DbEnv(0u).remove(strPath.c_str(), 0);
44 CDBEnv::CDBEnv() : fDbEnvInit(false), fMockDb(false), dbenv(DB_CXX_NO_EXCEPTIONS) { }
56 bool CDBEnv::Open(boost::filesystem::path pathEnv_)
65 boost::filesystem::path pathDataDir = pathEnv;
66 strPath = pathDataDir.string();
67 boost::filesystem::path pathLogDir = pathDataDir / "database";
68 boost::filesystem::create_directory(pathLogDir);
69 boost::filesystem::path pathErrorFile = pathDataDir / "db.log";
70 printf("dbenv.open LogDir=%s ErrorFile=%s\n", pathLogDir.string().c_str(), pathErrorFile.string().c_str());
72 unsigned int nEnvFlags = 0;
73 if (GetBoolArg("-privdb", true))
74 nEnvFlags |= DB_PRIVATE;
76 int nDbCache = GetArgInt("-dbcache", 25);
77 dbenv.set_lg_dir(pathLogDir.string().c_str());
78 dbenv.set_cachesize(nDbCache / 1024, (nDbCache % 1024)*1048576, 1);
79 dbenv.set_lg_bsize(1048576);
80 dbenv.set_lg_max(10485760);
82 // Bugfix: Bump lk_max_locks default to 537000, to safely handle reorgs with up to 5 blocks reversed
83 // dbenv.set_lk_max_locks(10000);
84 dbenv.set_lk_max_locks(537000);
86 dbenv.set_lk_max_objects(10000);
87 dbenv.set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug
88 dbenv.set_flags(DB_AUTO_COMMIT, 1);
89 dbenv.set_flags(DB_TXN_WRITE_NOSYNC, 1);
90 #ifdef DB_LOG_AUTO_REMOVE
91 dbenv.log_set_config(DB_LOG_AUTO_REMOVE, 1);
93 int ret = dbenv.open(strPath.c_str(),
104 return error("CDB() : error %s (%d) opening database environment", DbEnv::strerror(ret), ret);
112 void CDBEnv::MakeMock()
115 throw runtime_error("CDBEnv::MakeMock(): already initialized");
118 throw runtime_error("CDBEnv::MakeMock(): during shutdown");
120 printf("CDBEnv::MakeMock()\n");
122 dbenv.set_cachesize(1, 0, 1);
123 dbenv.set_lg_bsize(10485760*4);
124 dbenv.set_lg_max(10485760);
125 dbenv.set_lk_max_locks(10000);
126 dbenv.set_lk_max_objects(10000);
127 dbenv.set_flags(DB_AUTO_COMMIT, 1);
128 #ifdef DB_LOG_IN_MEMORY
129 dbenv.log_set_config(DB_LOG_IN_MEMORY, fUseMemoryLog ? 1 : 0);
131 int ret = dbenv.open(NULL,
141 throw runtime_error(strprintf("CDBEnv::MakeMock(): error %d opening database environment", ret));
147 CDBEnv::VerifyResult CDBEnv::Verify(std::string strFile, bool (*recoverFunc)(CDBEnv& dbenv, std::string strFile))
150 assert(mapFileUseCount.count(strFile) == 0);
153 int result = db.verify(strFile.c_str(), NULL, NULL, 0);
156 else if (recoverFunc == NULL)
160 bool fRecovered = (*recoverFunc)(*this, strFile);
161 return (fRecovered ? RECOVER_OK : RECOVER_FAIL);
164 bool CDBEnv::Salvage(std::string strFile, bool fAggressive,
165 std::vector<CDBEnv::KeyValPair >& vResult)
168 assert(mapFileUseCount.count(strFile) == 0);
170 u_int32_t flags = DB_SALVAGE;
171 if (fAggressive) flags |= DB_AGGRESSIVE;
173 stringstream strDump;
176 int result = db.verify(strFile.c_str(), NULL, &strDump, flags);
179 printf("ERROR: db salvage failed\n");
183 // Format of bdb dump is ascii lines:
192 while (!strDump.eof() && strLine != "HEADER=END")
193 getline(strDump, strLine); // Skip past header
195 std::string keyHex, valueHex;
196 while (!strDump.eof() && keyHex != "DATA=END")
198 getline(strDump, keyHex);
199 if (keyHex != "DATA=END")
201 getline(strDump, valueHex);
202 vResult.push_back(make_pair(ParseHex(keyHex),ParseHex(valueHex)));
206 return (result == 0);
210 void CDBEnv::CheckpointLSN(std::string strFile)
212 dbenv.txn_checkpoint(0, 0, 0);
215 dbenv.lsn_reset(strFile.c_str(), 0);
219 CDB::CDB(const char *pszFile, const char* pszMode) :
220 pdb(NULL), activeTxn(NULL)
226 fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
227 bool fCreate = strchr(pszMode, 'c') != NULL;
228 unsigned int nFlags = DB_THREAD;
234 if (!bitdb.Open(GetDataDir()))
235 throw runtime_error("env open failed");
238 ++bitdb.mapFileUseCount[strFile];
239 pdb = bitdb.mapDb[strFile];
242 pdb = new Db(&bitdb.dbenv, 0);
244 bool fMockDb = bitdb.IsMock();
247 DbMpoolFile*mpf = pdb->get_mpf();
248 ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
250 throw runtime_error(strprintf("CDB() : failed to configure for no temp file backing for database %s", pszFile));
253 ret = pdb->open(NULL, // Txn pointer
254 fMockDb ? NULL : pszFile, // Filename
255 "main", // Logical db name
256 DB_BTREE, // Database type
264 --bitdb.mapFileUseCount[strFile];
266 throw runtime_error(strprintf("CDB() : can't open database file %s, error %d", pszFile, ret));
269 if (fCreate && !Exists(string("version")))
271 bool fTmp = fReadOnly;
273 WriteVersion(CLIENT_VERSION);
277 bitdb.mapDb[strFile] = pdb;
291 // Flush database activity from memory pool to disk log
292 unsigned int nMinutes = 0;
296 bitdb.dbenv.txn_checkpoint(nMinutes ? GetArgUInt("-dblogsize", 100)*1024 : 0, nMinutes, 0);
300 --bitdb.mapFileUseCount[strFile];
304 void CDBEnv::CloseDb(const string& strFile)
308 if (mapDb[strFile] != NULL)
310 // Close the database handle
311 Db* pdb = mapDb[strFile];
314 mapDb[strFile] = NULL;
319 bool CDBEnv::RemoveDb(const string& strFile)
321 this->CloseDb(strFile);
324 int rc = dbenv.dbremove(NULL, strFile.c_str(), NULL, DB_AUTO_COMMIT);
328 bool CDB::Rewrite(const string& strFile, const char* pszSkip)
334 if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0)
336 // Flush log data to the dat file
337 bitdb.CloseDb(strFile);
338 bitdb.CheckpointLSN(strFile);
339 bitdb.mapFileUseCount.erase(strFile);
341 bool fSuccess = true;
342 printf("Rewriting %s...\n", strFile.c_str());
343 string strFileRes = strFile + ".rewrite";
344 { // surround usage of db with extra {}
345 CDB db(strFile.c_str(), "r");
346 Db* pdbCopy = new Db(&bitdb.dbenv, 0);
348 int ret = pdbCopy->open(NULL, // Txn pointer
349 strFileRes.c_str(), // Filename
350 "main", // Logical db name
351 DB_BTREE, // Database type
356 printf("Cannot create database file %s\n", strFileRes.c_str());
360 Dbc* pcursor = db.GetCursor();
364 CDataStream ssKey(SER_DISK, CLIENT_VERSION);
365 CDataStream ssValue(SER_DISK, CLIENT_VERSION);
366 int ret = db.ReadAtCursor(pcursor, ssKey, ssValue, DB_NEXT);
367 if (ret == DB_NOTFOUND)
381 size_t pszSkipLen = strlen(pszSkip);
382 if (strncmp(&ssKey[0], pszSkip, std::min(ssKey.size(), pszSkipLen)) == 0)
386 if (strncmp(&ssKey[0], "\x07version", 8) == 0)
390 ssValue << CLIENT_VERSION;
392 Dbt datKey(&ssKey[0], ssKey.size());
393 Dbt datValue(&ssValue[0], ssValue.size());
394 int ret2 = pdbCopy->put(NULL, &datKey, &datValue, DB_NOOVERWRITE);
402 bitdb.CloseDb(strFile);
403 if (pdbCopy->close(0))
410 Db dbA(&bitdb.dbenv, 0);
411 if (dbA.remove(strFile.c_str(), NULL, 0))
413 Db dbB(&bitdb.dbenv, 0);
414 if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0))
418 printf("Rewriting of %s FAILED!\n", strFileRes.c_str());
428 void CDBEnv::Flush(bool fShutdown)
430 int64_t nStart = GetTimeMillis();
431 // Flush log data to the actual data file
432 // on all files that are not in use
433 printf("Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started");
438 map<string, int>::iterator mi = mapFileUseCount.begin();
439 while (mi != mapFileUseCount.end())
441 string strFile = (*mi).first;
442 int nRefCount = (*mi).second;
443 printf("%s refcount=%d\n", strFile.c_str(), nRefCount);
446 // Move log data to the dat file
448 printf("%s checkpoint\n", strFile.c_str());
449 dbenv.txn_checkpoint(0, 0, 0);
450 printf("%s detach\n", strFile.c_str());
452 dbenv.lsn_reset(strFile.c_str(), 0);
453 printf("%s closed\n", strFile.c_str());
454 mapFileUseCount.erase(mi++);
459 printf("DBFlush(%s)%s ended %15" PRId64 "ms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started", GetTimeMillis() - nStart);
463 if (mapFileUseCount.empty())
465 dbenv.log_archive(&listp, DB_ARCH_REMOVE);
477 unsigned char CAddrDB::pchMessageStart[4] = {};
481 pathAddr = GetDataDir() / "peers.dat";
484 bool CAddrDB::Write(const CAddrMan& addr)
486 // Generate random temporary filename
487 unsigned short randv = 0;
488 GetRandBytes((unsigned char *)&randv, sizeof(randv));
489 std::string tmpfn = strprintf("peers.dat.%04x", randv);
491 // serialize addresses, checksum data up to that point, then append csum
492 CDataStream ssPeers(SER_DISK, CLIENT_VERSION);
493 ssPeers << FLATDATA(CAddrDB::pchMessageStart);
495 uint256 hash = Hash(ssPeers.begin(), ssPeers.end());
498 // open temp output file, and associate with CAutoFile
499 boost::filesystem::path pathTmp = GetDataDir() / tmpfn;
500 FILE *file = fopen(pathTmp.string().c_str(), "wb");
501 CAutoFile fileout = CAutoFile(file, SER_DISK, CLIENT_VERSION);
503 return error("CAddrman::Write() : open failed");
505 // Write and commit header, data
509 catch (const std::exception&) {
510 return error("CAddrman::Write() : I/O error");
515 // replace existing peers.dat, if any, with new peers.dat.XXXX
516 if (!RenameOver(pathTmp, pathAddr))
517 return error("CAddrman::Write() : Rename-into-place failed");
522 bool CAddrDB::Read(CAddrMan& addr)
524 // open input file, and associate with CAutoFile
525 FILE *file = fopen(pathAddr.string().c_str(), "rb");
526 CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION);
528 return error("CAddrman::Read() : open failed");
530 // use file size to size memory buffer
531 int fileSize = GetFilesize(filein);
532 int dataSize = fileSize - sizeof(uint256);
533 //Don't try to resize to a negative number if file is small
534 if ( dataSize < 0 ) dataSize = 0;
535 vector<unsigned char> vchData;
536 vchData.resize(dataSize);
539 // read data and checksum from file
541 filein.read((char *)&vchData[0], dataSize);
544 catch (const std::exception&) {
545 return error("CAddrman::Read() 2 : I/O error or stream data corrupted");
549 CDataStream ssPeers(vchData, SER_DISK, CLIENT_VERSION);
551 // verify stored checksum matches input data
552 uint256 hashTmp = Hash(ssPeers.begin(), ssPeers.end());
553 if (hashIn != hashTmp)
554 return error("CAddrman::Read() : checksum mismatch; data corrupted");
556 unsigned char pchMsgTmp[4];
558 // de-serialize file header (CAddrDB::pchMessageStart magic number) and
559 ssPeers >> FLATDATA(pchMsgTmp);
561 // verify the network matches ours
562 if (memcmp(pchMsgTmp, CAddrDB::pchMessageStart, sizeof(pchMsgTmp)))
563 return error("CAddrman::Read() : invalid network magic number");
565 // de-serialize address data into one CAddrMan object
568 catch (const std::exception&) {
569 return error("CAddrman::Read() : I/O error or stream data corrupted");