diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 0a35325e..96afdb0e 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -377,7 +377,50 @@ void mdb_txn_safe::allow_new_txns() creation_gate.clear(); } +void lmdb_resized(MDB_env *env) +{ + mdb_txn_safe::prevent_new_txns(); + MGINFO("LMDB map resize detected."); + + MDB_envinfo mei; + + mdb_env_info(env, &mei); + uint64_t old = mei.me_mapsize; + + mdb_txn_safe::wait_no_active_txns(); + + int result = mdb_env_set_mapsize(env, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to set new mapsize: ", result).c_str())); + + mdb_env_info(env, &mei); + uint64_t new_mapsize = mei.me_mapsize; + + MGINFO("LMDB Mapsize increased." << " Old: " << old / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB"); + + mdb_txn_safe::allow_new_txns(); +} + +inline int lmdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn) +{ + int res = mdb_txn_begin(env, parent, flags, txn); + if (res == MDB_MAP_RESIZED) { + lmdb_resized(env); + res = mdb_txn_begin(env, parent, flags, txn); + } + return res; +} + +inline int lmdb_txn_renew(MDB_txn *txn) +{ + int res = mdb_txn_renew(txn); + if (res == MDB_MAP_RESIZED) { + lmdb_resized(mdb_txn_env(txn)); + res = mdb_txn_renew(txn); + } + return res; +} void BlockchainLMDB::do_resize(uint64_t increase_size) { @@ -561,7 +604,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con { LOG_PRINT_L1("No existing blocks to check for average block size"); } - else if (m_cum_count) + else if (m_cum_count >= num_prev_blocks) { avg_block_size = m_cum_size / m_cum_count; LOG_PRINT_L1("average block size across recent " << m_cum_count << " blocks: " << avg_block_size); @@ -570,6 +613,9 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con } else { + MDB_txn *rtxn; + mdb_txn_cursors *rcurs; + block_rtxn_start(&rtxn, &rcurs); for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num) { uint32_t block_size = get_block_size(block_num); @@ -578,6 +624,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con // some blocks were to be skipped for being outliers. ++num_blocks_used; } + block_rtxn_stop(); avg_block_size = total_block_size / num_blocks_used; LOG_PRINT_L1("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size); } @@ -702,7 +749,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons uint64_t m_height = height(); int result; - uint64_t tx_id = m_num_txs; + uint64_t tx_id = num_txs(); CURSOR(txs) CURSOR(tx_indices) @@ -735,7 +782,6 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str())); - m_num_txs++; return tx_id; } @@ -783,8 +829,6 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const // Don't delete the tx_indices entry until the end, after we're done with val_tx_id if (mdb_cursor_del(m_cur_tx_indices, 0)) throw1(DB_ERROR("Failed to add removal of tx index to db transaction")); - - m_num_txs--; } uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash, @@ -797,6 +841,7 @@ uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash, check_open(); mdb_txn_cursors *m_cursors = &m_wcursors; uint64_t m_height = height(); + uint64_t m_num_outputs = num_outputs(); int result = 0; @@ -849,7 +894,6 @@ uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash, if ((result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &data, MDB_APPENDDUP))) throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str())); - m_num_outputs++; return ok.amount_index; } @@ -934,8 +978,6 @@ void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_in result = mdb_cursor_del(m_cur_output_amounts, 0); if (result) throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast(out_index).append(": ")).c_str(), result).c_str())); - - m_num_outputs--; } void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) @@ -1000,7 +1042,7 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const void BlockchainLMDB::check_open() const { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); +// LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (!m_open) throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); } @@ -1155,16 +1197,6 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); uint64_t m_height = db_stats.ms_entries; - // get and keep current number of txs - if ((result = mdb_stat(txn, m_txs, &db_stats))) - throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str())); - m_num_txs = db_stats.ms_entries; - - // get and keep current number of outputs - if ((result = mdb_stat(txn, m_output_txs, &db_stats))) - throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str())); - m_num_outputs = db_stats.ms_entries; - bool compatible = true; MDB_val_copy k("version"); @@ -1270,7 +1302,7 @@ void BlockchainLMDB::reset() check_open(); mdb_txn_safe txn; - if (auto result = mdb_txn_begin(m_env, NULL, 0, txn)) + if (auto result = lmdb_txn_begin(m_env, NULL, 0, txn)) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); if (auto result = mdb_drop(txn, m_blocks, 0)) @@ -1304,7 +1336,6 @@ void BlockchainLMDB::reset() throw0(DB_ERROR(lmdb_error("Failed to write version to database: ", result).c_str())); txn.commit(); - m_num_outputs = 0; m_cum_size = 0; m_cum_count = 0; } @@ -1354,7 +1385,7 @@ void BlockchainLMDB::unlock() txn_ptr = m_write_txn; \ else \ { \ - if (auto mdb_res = mdb_txn_begin(m_env, NULL, flags, auto_txn)) \ + if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \ throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \ } \ @@ -1388,7 +1419,7 @@ void BlockchainLMDB::unlock() txn_ptr = m_write_txn; \ else \ { \ - if (auto mdb_res = mdb_txn_begin(m_env, NULL, flags, auto_txn)) \ + if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \ throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \ } \ @@ -1714,6 +1745,34 @@ uint64_t BlockchainLMDB::height() const return db_stats.ms_entries; } +uint64_t BlockchainLMDB::num_txs() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + TXN_PREFIX_RDONLY(); + int result; + + // get current height + MDB_stat db_stats; + if ((result = mdb_stat(m_txn, m_txs, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str())); + return db_stats.ms_entries; +} + +uint64_t BlockchainLMDB::num_outputs() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + TXN_PREFIX_RDONLY(); + int result; + + // get current height + MDB_stat db_stats; + if ((result = mdb_stat(m_txn, m_output_txs, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str())); + return db_stats.ms_entries; +} + bool BlockchainLMDB::tx_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2279,7 +2338,7 @@ bool BlockchainLMDB::batch_start(uint64_t batch_num_blocks) m_write_batch_txn = new mdb_txn_safe(); // NOTE: need to make sure it's destroyed properly when done - if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn)) + if (auto mdb_res = lmdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn)) { delete m_write_batch_txn; m_write_batch_txn = nullptr; @@ -2398,12 +2457,12 @@ bool BlockchainLMDB::block_rtxn_start(MDB_txn **mtxn, mdb_txn_cursors **mcur) co m_tinfo.reset(new mdb_threadinfo); memset(&m_tinfo->m_ti_rcursors, 0, sizeof(m_tinfo->m_ti_rcursors)); memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); - if (auto mdb_res = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn)) + if (auto mdb_res = lmdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn)) throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a read transaction for the db: ", mdb_res).c_str())); ret = true; } else if (!m_tinfo->m_ti_rflags.m_rf_txn) { - if (auto mdb_res = mdb_txn_renew(m_tinfo->m_ti_rtxn)) + if (auto mdb_res = lmdb_txn_renew(m_tinfo->m_ti_rtxn)) throw0(DB_ERROR_TXN_START(lmdb_error("Failed to renew a read transaction for the db: ", mdb_res).c_str())); ret = true; } @@ -2436,12 +2495,12 @@ void BlockchainLMDB::block_txn_start(bool readonly) m_tinfo.reset(new mdb_threadinfo); memset(&m_tinfo->m_ti_rcursors, 0, sizeof(m_tinfo->m_ti_rcursors)); memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); - if (auto mdb_res = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn)) + if (auto mdb_res = lmdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn)) throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a read transaction for the db: ", mdb_res).c_str())); didit = true; } else if (!m_tinfo->m_ti_rflags.m_rf_txn) { - if (auto mdb_res = mdb_txn_renew(m_tinfo->m_ti_rtxn)) + if (auto mdb_res = lmdb_txn_renew(m_tinfo->m_ti_rtxn)) throw0(DB_ERROR_TXN_START(lmdb_error("Failed to renew a read transaction for the db: ", mdb_res).c_str())); didit = true; } @@ -2467,7 +2526,7 @@ void BlockchainLMDB::block_txn_start(bool readonly) { m_writer = boost::this_thread::get_id(); m_write_txn = new mdb_txn_safe(); - if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, *m_write_txn)) + if (auto mdb_res = lmdb_txn_begin(m_env, NULL, 0, *m_write_txn)) { delete m_write_txn; m_write_txn = nullptr; @@ -2545,8 +2604,6 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c } } - uint64_t num_txs = m_num_txs; - uint64_t num_outputs = m_num_outputs; try { BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); @@ -2557,8 +2614,6 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c } catch (...) { - m_num_txs = num_txs; - m_num_outputs = num_outputs; block_txn_abort(); throw; } @@ -2573,8 +2628,6 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) block_txn_start(false); - uint64_t num_txs = m_num_txs; - uint64_t num_outputs = m_num_outputs; try { BlockchainDB::pop_block(blk, txs); @@ -2582,8 +2635,6 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) } catch (...) { - m_num_txs = num_txs; - m_num_outputs = num_outputs; block_txn_abort(); throw; } @@ -3257,8 +3308,6 @@ void BlockchainLMDB::migrate_0_1() lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); mdb_set_dupsort(txn, m_output_amounts, compare_uint64); txn.commit(); - m_num_txs = 0; - m_num_outputs = 0; } while(0); do { @@ -3329,12 +3378,9 @@ void BlockchainLMDB::migrate_0_1() throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str())); if (!i) { MDB_stat ms; - mdb_stat(txn, m_output_txs, &ms); - m_num_outputs = ms.ms_entries; mdb_stat(txn, m_txs, &ms); - m_num_txs = i = ms.ms_entries; + i = ms.ms_entries; if (i) { - m_num_txs = i; MDB_val_set(pk, "txblk"); result = mdb_cursor_get(c_props, &pk, &k, MDB_SET); if (result) diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index e7faf8cd..b4e3c717 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -312,6 +312,9 @@ private: virtual void remove_spent_key(const crypto::key_image& k_image); + uint64_t num_txs() const; + uint64_t num_outputs() const; + // Hard fork virtual void set_hard_fork_version(uint64_t height, uint8_t version); virtual uint8_t get_hard_fork_version(uint64_t height) const; @@ -369,10 +372,8 @@ private: MDB_dbi m_properties; - uint64_t m_num_txs; - uint64_t m_num_outputs; mutable uint64_t m_cum_size; // used in batch size estimation - mutable int m_cum_count; + mutable unsigned int m_cum_count; std::string m_folder; mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index a8128663..d5fefffc 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1986,12 +1986,14 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc return false; } + m_db->block_txn_start(true); resp.total_height = get_current_blockchain_height(); size_t count = 0; for(size_t i = resp.start_height; i < resp.total_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) { resp.m_block_ids.push_back(m_db->get_block_hash_from_height(i)); } + m_db->block_txn_stop(); return true; } //------------------------------------------------------------------ @@ -2022,6 +2024,7 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons } } + m_db->block_txn_start(true); total_height = get_current_blockchain_height(); size_t count = 0; for(size_t i = start_height; i < total_height && count < max_count; i++, count++) @@ -2032,6 +2035,7 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons get_transactions(blocks.back().first.tx_hashes, blocks.back().second, mis); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); } + m_db->block_txn_stop(); return true; } //------------------------------------------------------------------ diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index b940dba3..239ef5d8 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -219,40 +219,6 @@ namespace cryptonote return m_blockchain_storage.get_alternative_blocks_count(); } //----------------------------------------------------------------------------------------------- - bool core::lock_db_directory(const boost::filesystem::path &path) - { - // boost doesn't like locking directories... - const boost::filesystem::path lock_path = path / ".daemon_lock"; - - try - { - // ensure the file exists - std::ofstream(lock_path.string(), std::ios::out).close(); - - db_lock = boost::interprocess::file_lock(lock_path.string().c_str()); - LOG_PRINT_L1("Locking " << lock_path.string()); - if (!db_lock.try_lock()) - { - LOG_ERROR("Failed to lock " << lock_path.string()); - return false; - } - return true; - } - catch (const std::exception &e) - { - LOG_ERROR("Error trying to lock " << lock_path.string() << ": " << e.what()); - return false; - } - } - //----------------------------------------------------------------------------------------------- - bool core::unlock_db_directory() - { - db_lock.unlock(); - db_lock = boost::interprocess::file_lock(); - LOG_PRINT_L1("Blockchain directory successfully unlocked"); - return true; - } - //----------------------------------------------------------------------------------------------- bool core::init(const boost::program_options::variables_map& vm, const cryptonote::test_options *test_options) { start_time = std::time(nullptr); @@ -284,11 +250,6 @@ namespace cryptonote // make sure the data directory exists, and try to lock it CHECK_AND_ASSERT_MES (boost::filesystem::exists(folder) || boost::filesystem::create_directories(folder), false, std::string("Failed to create directory ").append(folder.string()).c_str()); - if (!lock_db_directory (folder)) - { - LOG_ERROR ("Failed to lock " << folder); - return false; - } // check for blockchain.bin try @@ -440,7 +401,6 @@ namespace cryptonote m_miner.stop(); m_mempool.deinit(); m_blockchain_storage.deinit(); - unlock_db_directory(); return true; } //----------------------------------------------------------------------------------------------- diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index fe1d1191..1463c2fe 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -763,24 +763,6 @@ namespace cryptonote */ bool relay_txpool_transactions(); - /** - * @brief locks a file in the BlockchainDB directory - * - * @param path the directory in which to place the file - * - * @return true if lock acquired successfully, otherwise false - */ - bool lock_db_directory(const boost::filesystem::path &path); - - /** - * @brief unlocks the db directory - * - * @note see lock_db_directory() - * - * @return true - */ - bool unlock_db_directory(); - bool m_test_drop_download = true; //!< whether or not to drop incoming blocks (for testing) uint64_t m_test_drop_download_height = 0; //!< height under which to drop incoming blocks, if doing so