From 768e5d52fb295b000940f6a806c3d4bfc3e4f54d Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Sat, 12 May 2012 01:24:27 -0400 Subject: [PATCH 1/2] Add new utility functions FileCommit(), RenameOver() --- src/main.h | 8 +------- src/util.cpp | 21 +++++++++++++++++++++ src/util.h | 2 ++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/main.h b/src/main.h index 5ac5547a3..331b16b50 100644 --- a/src/main.h +++ b/src/main.h @@ -956,13 +956,7 @@ public: // Flush stdio buffers and commit to disk before returning fflush(fileout); if (!IsInitialBlockDownload() || (nBestHeight+1) % 500 == 0) - { -#ifdef WIN32 - _commit(_fileno(fileout)); -#else - fsync(fileno(fileout)); -#endif - } + FileCommit(fileout); return true; } diff --git a/src/util.cpp b/src/util.cpp index 823d00a4e..82c16feda 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -910,6 +910,27 @@ void CreatePidFile(const boost::filesystem::path &path, pid_t pid) } } +bool RenameOver(boost::filesystem::path src, boost::filesystem::path dest) +{ +#ifdef WIN32 + return MoveFileEx(src.string().c_str(), dest.string().c_str(), + MOVEFILE_REPLACE_EXISTING); +#else + int rc = std::rename(src.string().c_str(), dest.string().c_str()); + return (rc == 0); +#endif /* WIN32 */ +} + +void FileCommit(FILE *fileout) +{ + fflush(fileout); // harmless if redundantly called +#ifdef WIN32 + _commit(_fileno(fileout)); +#else + fsync(fileno(fileout)); +#endif +} + int GetFilesize(FILE* file) { int nSavePos = ftell(file); diff --git a/src/util.h b/src/util.h index 8e65fa786..f7bdaf533 100644 --- a/src/util.h +++ b/src/util.h @@ -152,7 +152,9 @@ std::string EncodeBase64(const std::string& str); void ParseParameters(int argc, const char*const argv[]); bool WildcardMatch(const char* psz, const char* mask); bool WildcardMatch(const std::string& str, const std::string& mask); +void FileCommit(FILE *fileout); int GetFilesize(FILE* file); +bool RenameOver(boost::filesystem::path src, boost::filesystem::path dest); boost::filesystem::path GetDefaultDataDir(); const boost::filesystem::path &GetDataDir(bool fNetSpecific = true); boost::filesystem::path GetConfigFile(); From 928d3a011cc66c7f907c4d053f674ea77dc611cc Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Wed, 16 May 2012 22:11:19 -0400 Subject: [PATCH 2/2] CAddrDB: Replace BDB-managed addr.dat with internally managed peers.dat --- src/db.cpp | 129 +++++++++++++++++++++++++++++++-------------------- src/db.h | 17 +++---- src/init.cpp | 12 +++-- src/net.cpp | 7 ++- src/net.h | 1 - 5 files changed, 100 insertions(+), 66 deletions(-) diff --git a/src/db.cpp b/src/db.cpp index 5bd052820..5b36a8f1f 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -164,8 +164,6 @@ void CDB::Close() unsigned int nMinutes = 0; if (fReadOnly) nMinutes = 1; - if (strFile == "addr.dat") - nMinutes = 2; if (strFile == "blkindex.dat") nMinutes = 2; if (strFile == "blkindex.dat" && IsInitialBlockDownload()) @@ -310,7 +308,7 @@ void DBFlush(bool fShutdown) CloseDb(strFile); printf("%s checkpoint\n", strFile.c_str()); dbenv.txn_checkpoint(0, 0, 0); - if ((strFile != "blkindex.dat" && strFile != "addr.dat") || fDetachDB) { + if (strFile != "blkindex.dat" || fDetachDB) { printf("%s detach\n", strFile.c_str()); dbenv.lsn_reset(strFile.c_str(), 0); } @@ -737,65 +735,96 @@ bool CTxDB::LoadBlockIndex() // CAddrDB // -bool CAddrDB::WriteAddrman(const CAddrMan& addrman) + +CAddrDB::CAddrDB() { - return Write(string("addrman"), addrman); + pathAddr = GetDataDir() / "peers.dat"; } -bool CAddrDB::LoadAddresses() +bool CAddrDB::Write(const CAddrMan& addr) { - if (Read(string("addrman"), addrman)) - { - printf("Loaded %i addresses\n", addrman.size()); - return true; + // Generate random temporary filename + unsigned short randv = 0; + RAND_bytes((unsigned char *)&randv, sizeof(randv)); + std::string tmpfn = strprintf("peers.dat.%04x", randv); + + // serialize addresses, checksum data up to that point, then append csum + CDataStream ssPeers(SER_DISK, CLIENT_VERSION); + ssPeers << FLATDATA(pchMessageStart); + ssPeers << addr; + uint256 hash = Hash(ssPeers.begin(), ssPeers.end()); + ssPeers << hash; + + // open temp output file, and associate with CAutoFile + boost::filesystem::path pathTmp = GetDataDir() / tmpfn; + FILE *file = fopen(pathTmp.string().c_str(), "wb"); + CAutoFile fileout = CAutoFile(file, SER_DISK, CLIENT_VERSION); + if (!fileout) + return error("CAddrman::Write() : open failed"); + + // Write and commit header, data + try { + fileout << ssPeers; } - - // Read pre-0.6 addr records + catch (std::exception &e) { + return error("CAddrman::Write() : I/O error"); + } + FileCommit(fileout); + fileout.fclose(); - vector vAddr; - vector > vDelete; + // replace existing peers.dat, if any, with new peers.dat.XXXX + if (!RenameOver(pathTmp, pathAddr)) + return error("CAddrman::Write() : Rename-into-place failed"); - // Get cursor - Dbc* pcursor = GetCursor(); - if (!pcursor) - return false; - - loop - { - // Read next record - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue); - if (ret == DB_NOTFOUND) - break; - else if (ret != 0) - return false; + return true; +} - // Unserialize - string strType; - ssKey >> strType; - if (strType == "addr") - { - CAddress addr; - ssValue >> addr; - vAddr.push_back(addr); - } +bool CAddrDB::Read(CAddrMan& addr) +{ + // open input file, and associate with CAutoFile + FILE *file = fopen(pathAddr.string().c_str(), "rb"); + CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION); + if (!filein) + return error("CAddrman::Read() : open failed"); + + // use file size to size memory buffer + int fileSize = GetFilesize(filein); + int dataSize = fileSize - sizeof(uint256); + vector vchData; + vchData.resize(dataSize); + uint256 hashIn; + + // read data and checksum from file + try { + filein.read((char *)&vchData[0], dataSize); + filein >> hashIn; } - pcursor->close(); + catch (std::exception &e) { + return error("CAddrman::Read() 2 : I/O error or stream data corrupted"); + } + filein.fclose(); - addrman.Add(vAddr, CNetAddr("0.0.0.0")); - printf("Loaded %i addresses\n", addrman.size()); + CDataStream ssPeers(vchData, SER_DISK, CLIENT_VERSION); - // Note: old records left; we ran into hangs-on-startup - // bugs for some users who (we think) were running after - // an unclean shutdown. + // verify stored checksum matches input data + uint256 hashTmp = Hash(ssPeers.begin(), ssPeers.end()); + if (hashIn != hashTmp) + return error("CAddrman::Read() : checksum mismatch; data corrupted"); - return true; -} + // de-serialize address data + unsigned char pchMsgTmp[4]; + try { + ssPeers >> FLATDATA(pchMsgTmp); + ssPeers >> addr; + } + catch (std::exception &e) { + return error("CAddrman::Read() : I/O error or stream data corrupted"); + } -bool LoadAddresses() -{ - return CAddrDB("cr+").LoadAddresses(); -} + // finally, verify the network matches ours + if (memcmp(pchMsgTmp, pchMessageStart, sizeof(pchMsgTmp))) + return error("CAddrman::Read() : invalid network magic number"); + return true; +} diff --git a/src/db.h b/src/db.h index 3ce8f1758..4f8e5d04f 100644 --- a/src/db.h +++ b/src/db.h @@ -296,20 +296,15 @@ public: -/** Access to the (IP) address database (addr.dat) */ -class CAddrDB : public CDB +/** Access to the (IP) address database (peers.dat) */ +class CAddrDB { -public: - CAddrDB(const char* pszMode="r+") : CDB("addr.dat", pszMode) { } private: - CAddrDB(const CAddrDB&); - void operator=(const CAddrDB&); + boost::filesystem::path pathAddr; public: - bool WriteAddrman(const CAddrMan& addr); - bool LoadAddresses(); + CAddrDB(); + bool Write(const CAddrMan& addr); + bool Read(CAddrMan& addr); }; -bool LoadAddresses(); - - #endif // BITCOIN_DB_H diff --git a/src/init.cpp b/src/init.cpp index 9b12ab0aa..2a3e407e8 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -371,9 +371,15 @@ bool AppInit2() InitMessage(_("Loading addresses...")); printf("Loading addresses...\n"); nStart = GetTimeMillis(); - if (!LoadAddresses()) - strErrors << _("Error loading addr.dat") << "\n"; - printf(" addresses %15"PRI64d"ms\n", GetTimeMillis() - nStart); + + { + CAddrDB adb; + if (!adb.Read(addrman)) + printf("Invalid or missing peers.dat; recreating\n"); + } + + printf("Loaded %i addresses from peers.dat %"PRI64d"ms\n", + addrman.size(), GetTimeMillis() - nStart); InitMessage(_("Loading block index...")); printf("Loading block index...\n"); diff --git a/src/net.cpp b/src/net.cpp index 636db2952..24fd26ae5 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1278,8 +1278,13 @@ unsigned int pnSeed[] = void DumpAddresses() { + int64 nStart = GetTimeMillis(); + CAddrDB adb; - adb.WriteAddrman(addrman); + adb.Write(addrman); + + printf("Flushed %d addresses to peers.dat %"PRI64d"ms\n", + addrman.size(), GetTimeMillis() - nStart); } void ThreadDumpAddress2(void* parg) diff --git a/src/net.h b/src/net.h index 6c053b536..fcb5824d8 100644 --- a/src/net.h +++ b/src/net.h @@ -19,7 +19,6 @@ #include "protocol.h" #include "addrman.h" -class CAddrDB; class CRequestTracker; class CNode; class CBlockIndex;