From ac79502308d24c9ffc0be28c47c7a63c8f0465ed Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Fri, 15 May 2015 20:42:47 -0400 Subject: [PATCH 1/4] Move mdb_txn_safe implementation to cpp file --- src/blockchain_db/lmdb/db_lmdb.cpp | 54 +++++++++++++++++++++++++++++ src/blockchain_db/lmdb/db_lmdb.h | 55 +++--------------------------- 2 files changed, 58 insertions(+), 51 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 5126db40..211e2f3e 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -148,6 +148,60 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { + mdb_txn_safe::mdb_txn_safe() : m_txn(NULL) { } + + mdb_txn_safe::~mdb_txn_safe() + { + LOG_PRINT_L3("mdb_txn_safe: destructor"); + if (m_txn != NULL) + { + if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety + { + LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); + } + else + { + // Example of when this occurs: a lookup fails, so a read-only txn is + // aborted through this destructor. However, successful read-only txns + // ideally should have been committed when done and not end up here. + // + // NOTE: not sure if this is ever reached for a non-batch write + // transaction, but it's probably not ideal if it did. + LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); + } + mdb_txn_abort(m_txn); + } + } + + void mdb_txn_safe::commit(std::string message) + { + if (message.size() == 0) + { + message = "Failed to commit a transaction to the db"; + } + + if (mdb_txn_commit(m_txn)) + { + m_txn = NULL; + LOG_PRINT_L0(message); + throw DB_ERROR(message.c_str()); + } + m_txn = NULL; + } + + void mdb_txn_safe::abort() + { + LOG_PRINT_L3("mdb_txn_safe: abort()"); + if(m_txn != NULL) + { + mdb_txn_abort(m_txn); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL"); + } + } // If m_batch_active is set, a batch transaction exists beyond this class, such // as a batch import with verification enabled, or possibly (later) a batch diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 8f1e07e0..6c05de06 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -36,63 +36,16 @@ namespace cryptonote struct mdb_txn_safe { - mdb_txn_safe() : m_txn(NULL) { } - ~mdb_txn_safe() - { - LOG_PRINT_L3("mdb_txn_safe: destructor"); - if (m_txn != NULL) - { - if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety - { - LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); - } - else - { - // Example of when this occurs: a lookup fails, so a read-only txn is - // aborted through this destructor. However, successful read-only txns - // ideally should have been committed when done and not end up here. - // - // NOTE: not sure if this is ever reached for a non-batch write - // transaction, but it's probably not ideal if it did. - LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); - } - mdb_txn_abort(m_txn); - } - } + mdb_txn_safe(); + ~mdb_txn_safe(); - void commit(std::string message = "") - { - if (message.size() == 0) - { - message = "Failed to commit a transaction to the db"; - } - - if (mdb_txn_commit(m_txn)) - { - m_txn = NULL; - LOG_PRINT_L0(message); - throw DB_ERROR(message.c_str()); - } - m_txn = NULL; - } + void commit(std::string message = ""); // This should only be needed for batch transaction which must be ensured to // be aborted before mdb_env_close, not after. So we can't rely on // BlockchainLMDB destructor to call mdb_txn_safe destructor, as that's too late // to properly abort, since mdb_env_close would have been called earlier. - void abort() - { - LOG_PRINT_L3("mdb_txn_safe: abort()"); - if(m_txn != NULL) - { - mdb_txn_abort(m_txn); - m_txn = NULL; - } - else - { - LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL"); - } - } + void abort(); operator MDB_txn*() { From 7b7ef73c15b0b7775e504714fbdf1b0dee7f0461 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sat, 16 May 2015 22:05:54 -0400 Subject: [PATCH 2/4] LMDB should now dynamically resize the mapsize Some filesystems (*cough* NTFS *cough*) aren't good with sparse files, so this makes LMDB dynamically resize its mapsize as needed. Note: the check interval is currently every 10 blocks (for testing) and will probably need to change to 1000 or something. Default mapsize set to 1GiB. Blockchain conversion tools using batching will probably segfault, I'll fix that in the next commit. --- src/blockchain_db/lmdb/db_lmdb.cpp | 232 ++++++++++++++++++++--------- src/blockchain_db/lmdb/db_lmdb.h | 35 ++++- 2 files changed, 197 insertions(+), 70 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 211e2f3e..9579697d 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -148,74 +148,149 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { - mdb_txn_safe::mdb_txn_safe() : m_txn(NULL) { } +std::atomic mdb_txn_safe::num_active_txns{0}; +std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT; - mdb_txn_safe::~mdb_txn_safe() +mdb_txn_safe::mdb_txn_safe() : m_txn(NULL) +{ + while (creation_gate.test_and_set()); + num_active_txns++; + creation_gate.clear(); +} + +mdb_txn_safe::~mdb_txn_safe() +{ + LOG_PRINT_L3("mdb_txn_safe: destructor"); + if (m_txn != NULL) { - LOG_PRINT_L3("mdb_txn_safe: destructor"); - if (m_txn != NULL) + if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety { - if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety - { - LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); - } - else - { - // Example of when this occurs: a lookup fails, so a read-only txn is - // aborted through this destructor. However, successful read-only txns - // ideally should have been committed when done and not end up here. - // - // NOTE: not sure if this is ever reached for a non-batch write - // transaction, but it's probably not ideal if it did. - LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); - } - mdb_txn_abort(m_txn); - } - } - - void mdb_txn_safe::commit(std::string message) - { - if (message.size() == 0) - { - message = "Failed to commit a transaction to the db"; - } - - if (mdb_txn_commit(m_txn)) - { - m_txn = NULL; - LOG_PRINT_L0(message); - throw DB_ERROR(message.c_str()); - } - m_txn = NULL; - } - - void mdb_txn_safe::abort() - { - LOG_PRINT_L3("mdb_txn_safe: abort()"); - if(m_txn != NULL) - { - mdb_txn_abort(m_txn); - m_txn = NULL; + LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); } else { - LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL"); + // Example of when this occurs: a lookup fails, so a read-only txn is + // aborted through this destructor. However, successful read-only txns + // ideally should have been committed when done and not end up here. + // + // NOTE: not sure if this is ever reached for a non-batch write + // transaction, but it's probably not ideal if it did. + LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); + } + mdb_txn_abort(m_txn); + } + num_active_txns--; +} + +void mdb_txn_safe::commit(std::string message) +{ + if (message.size() == 0) + { + message = "Failed to commit a transaction to the db"; + } + + if (mdb_txn_commit(m_txn)) + { + m_txn = NULL; + LOG_PRINT_L0(message); + throw DB_ERROR(message.c_str()); + } + m_txn = NULL; +} + +void mdb_txn_safe::abort() +{ + LOG_PRINT_L3("mdb_txn_safe: abort()"); + if(m_txn != NULL) + { + mdb_txn_abort(m_txn); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL"); + } +} + +uint64_t mdb_txn_safe::num_active_tx() +{ + return num_active_txns; +} + +void mdb_txn_safe::prevent_new_txns() +{ + while (creation_gate.test_and_set()); +} + +void mdb_txn_safe::wait_no_active_txns() +{ + while (num_active_txns > 0); +} + +void mdb_txn_safe::allow_new_txns() +{ + creation_gate.clear(); +} + + + +void BlockchainLMDB::do_resize() +{ + MDB_envinfo mei; + + mdb_env_info(m_env, &mei); + + MDB_stat mst; + + mdb_env_stat(m_env, &mst); + + uint64_t new_mapsize = (double)mei.me_mapsize * RESIZE_FACTOR; + + new_mapsize += (new_mapsize % mst.ms_psize); + + mdb_txn_safe::prevent_new_txns(); + + if (m_write_txn != nullptr) + { + if (m_batch_active) + { + throw0(DB_ERROR("lmdb resizing not yet supported when batch transactions enabled!")); + } + else + { + throw0(DB_ERROR("attempting resize with write transaction in progress, this should not happen!")); } } -// If m_batch_active is set, a batch transaction exists beyond this class, such -// as a batch import with verification enabled, or possibly (later) a batch -// network sync. -// -// For some of the lookup methods, such as get_block_timestamp(), tx_exists(), -// and get_tx(), when m_batch_active is set, the lookup uses the batch -// transaction. This isn't only because the transaction is available, but it's -// necessary so that lookups include the database updates only present in the -// current batch write. -// -// A regular network sync without batch writes is expected to open a new read -// transaction, as those lookups are part of the validation done prior to the -// write for block and tx data, so no write transaction is open at the time. + mdb_txn_safe::wait_no_active_txns(); + + mdb_env_set_mapsize(m_env, new_mapsize); + + LOG_PRINT_L0("LMDB Mapsize increased." + << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" + << ", New: " << new_mapsize / (1024 * 1024) << "MiB"); + + mdb_txn_safe::allow_new_txns(); +} + +bool BlockchainLMDB::need_resize() +{ + MDB_envinfo mei; + + mdb_env_info(m_env, &mei); + + MDB_stat mst; + + mdb_env_stat(m_env, &mst); + + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + + if ((double)size_used / mei.me_mapsize > 0.8) + { + return true; + } + return false; +} void BlockchainLMDB::add_block( const block& blk , const size_t& block_size @@ -726,7 +801,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) if (mdb_env_set_maxdbs(m_env, 20)) throw0(DB_ERROR("Failed to set max number of dbs")); - size_t mapsize = 1LL << 34; + size_t mapsize = DEFAULT_MAPSIZE; if (auto result = mdb_env_set_mapsize(m_env, mapsize)) throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str())); if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644)) @@ -1710,16 +1785,23 @@ void BlockchainLMDB::batch_start() throw0(DB_ERROR("batch transactions not enabled")); if (m_batch_active) throw0(DB_ERROR("batch transaction already in progress")); + if (m_write_batch_txn != nullptr) + throw0(DB_ERROR("batch transaction already in progress")); if (m_write_txn) throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use")); check_open(); + + if (m_write_batch_txn == nullptr) + { + m_write_batch_txn = new mdb_txn_safe(); + } // NOTE: need to make sure it's destroyed properly when done - if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + if (mdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); // indicates this transaction is for batch transactions, but not whether it's // active - m_write_batch_txn.m_batch_txn = true; - m_write_txn = &m_write_batch_txn; + m_write_batch_txn->m_batch_txn = true; + m_write_txn = m_write_batch_txn; m_batch_active = true; LOG_PRINT_L3("batch transaction: begin"); } @@ -1731,7 +1813,10 @@ void BlockchainLMDB::batch_commit() throw0(DB_ERROR("batch transactions not enabled")); if (! m_batch_active) throw0(DB_ERROR("batch transaction not in progress")); + if (m_write_batch_txn == nullptr) + throw0(DB_ERROR("batch transaction not in progress")); check_open(); + LOG_PRINT_L3("batch transaction: committing..."); TIME_MEASURE_START(time1); m_write_txn->commit(); @@ -1739,11 +1824,8 @@ void BlockchainLMDB::batch_commit() time_commit1 += time1; LOG_PRINT_L3("batch transaction: committed"); - if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - if (! m_write_batch_txn.m_batch_txn) - throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction")); - m_write_txn = &m_write_batch_txn; + m_write_txn = nullptr; + delete m_write_batch_txn; } void BlockchainLMDB::batch_stop() @@ -1753,6 +1835,8 @@ void BlockchainLMDB::batch_stop() throw0(DB_ERROR("batch transactions not enabled")); if (! m_batch_active) throw0(DB_ERROR("batch transaction not in progress")); + if (m_write_batch_txn == nullptr) + throw0(DB_ERROR("batch transaction not in progress")); check_open(); LOG_PRINT_L3("batch transaction: committing..."); TIME_MEASURE_START(time1); @@ -1761,6 +1845,7 @@ void BlockchainLMDB::batch_stop() time_commit1 += time1; // for destruction of batch transaction m_write_txn = nullptr; + delete m_write_batch_txn; m_batch_active = false; LOG_PRINT_L3("batch transaction: end"); } @@ -1776,7 +1861,7 @@ void BlockchainLMDB::batch_abort() // for destruction of batch transaction m_write_txn = nullptr; // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called. - m_write_batch_txn.abort(); + m_write_batch_txn->abort(); m_batch_active = false; LOG_PRINT_L3("batch transaction: aborted"); } @@ -1798,6 +1883,15 @@ uint64_t BlockchainLMDB::add_block( const block& blk LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + if (m_height % 10 == 0) + { + if (need_resize()) + { + LOG_PRINT_L0("LMDB memory map needs resized, doing that now."); + do_resize(); + } + } + mdb_txn_safe txn; if (! m_batch_active) { diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 6c05de06..ec552a0f 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -26,6 +26,8 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once +#include + #include "blockchain_db/blockchain_db.h" #include "cryptonote_protocol/blobdatatype.h" // for type blobdata @@ -57,11 +59,34 @@ struct mdb_txn_safe return &m_txn; } + uint64_t num_active_tx(); + + static void prevent_new_txns(); + static void wait_no_active_txns(); + static void allow_new_txns(); + MDB_txn* m_txn; bool m_batch_txn = false; + static std::atomic num_active_txns; + + // could use a mutex here, but this should be sufficient. + static std::atomic_flag creation_gate; }; +// If m_batch_active is set, a batch transaction exists beyond this class, such +// as a batch import with verification enabled, or possibly (later) a batch +// network sync. +// +// For some of the lookup methods, such as get_block_timestamp(), tx_exists(), +// and get_tx(), when m_batch_active is set, the lookup uses the batch +// transaction. This isn't only because the transaction is available, but it's +// necessary so that lookups include the database updates only present in the +// current batch write. +// +// A regular network sync without batch writes is expected to open a new read +// transaction, as those lookups are part of the validation done prior to the +// write for block and tx data, so no write transaction is open at the time. class BlockchainLMDB : public BlockchainDB { public: @@ -174,6 +199,10 @@ public: virtual void pop_block(block& blk, std::vector& txs); private: + void do_resize(); + + bool need_resize(); + virtual void add_block( const block& blk , const size_t& block_size , const difficulty_type& cumulative_difficulty @@ -258,10 +287,14 @@ private: uint64_t m_num_outputs; std::string m_folder; mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn - mdb_txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB + mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB bool m_batch_transactions; // support for batch transactions bool m_batch_active; // whether batch transaction is in progress + + constexpr static uint64_t DEFAULT_MAPSIZE = 1 << 30; + constexpr static float RESIZE_PERCENT = 0.8f; + constexpr static float RESIZE_FACTOR = 1.5f; }; } // namespace cryptonote From b0d849e0a475d575cdb34c6b42119b4380e220e5 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 18 May 2015 05:45:15 -0400 Subject: [PATCH 3/4] null out batch txn pointer as needed (BlockchainLMDB) --- src/blockchain_db/lmdb/db_lmdb.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 9579697d..93900a43 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -760,6 +760,7 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) m_batch_transactions = batch_transactions; m_write_txn = nullptr; + m_write_batch_txn = nullptr; m_batch_active = false; m_height = 0; } @@ -1791,10 +1792,8 @@ void BlockchainLMDB::batch_start() throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use")); check_open(); - if (m_write_batch_txn == nullptr) - { - m_write_batch_txn = new mdb_txn_safe(); - } + m_write_batch_txn = new mdb_txn_safe(); + // NOTE: need to make sure it's destroyed properly when done if (mdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1846,6 +1845,7 @@ void BlockchainLMDB::batch_stop() // for destruction of batch transaction m_write_txn = nullptr; delete m_write_batch_txn; + m_write_batch_txn = nullptr; m_batch_active = false; LOG_PRINT_L3("batch transaction: end"); } @@ -1863,6 +1863,7 @@ void BlockchainLMDB::batch_abort() // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called. m_write_batch_txn->abort(); m_batch_active = false; + m_write_batch_txn = nullptr; LOG_PRINT_L3("batch transaction: aborted"); } From 01076ae70006558be580a3625ecc738efc9adeb7 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 18 May 2015 06:18:31 -0400 Subject: [PATCH 4/4] Check if LMDB needs resize every 1000 blocks (this was 10 for testing purposes) --- src/blockchain_db/lmdb/db_lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 93900a43..fd2600ba 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1884,7 +1884,7 @@ uint64_t BlockchainLMDB::add_block( const block& blk LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - if (m_height % 10 == 0) + if (m_height % 1000 == 0) { if (need_resize()) {