kumi/danicoin
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge #32

Browse source 
Add support to BlockchainDB and BlockchainLMDB for batch transactions.

Add profiling to block and tx processing and DB operations.

Improve block and tx processing efficiency by less repeat hashing.

Move LMDB storage to "lmdb" subfolder.
 - Upon startup, if old LMDB files are detected, abort with a message for the user to move them to subfolder or delete them.

Update and fix log statements and formatting.
This commit is contained in:
Thomas Winget 2015-02-23 18:24:59 -05:00
parent b7d112f601 6485dacc2f
commit b5796da0fa
No known key found for this signature in database
GPG key ID: 58131A160789E630
7 changed files with 442 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/blockchain_converter/blockchain_converter.cpp
View file

@ -1,21 +1,21 @@
// Copyright (c) 2014, The Monero Project
//
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL

371
src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp
View file

@ -33,6 +33,7 @@
#include "cryptonote_core/cryptonote_format_utils.h"
#include "crypto/crypto.h"
#include "profile_tools.h"
using epee::string_tools::pod_to_hex;
@ -152,12 +153,13 @@ void BlockchainLMDB::add_block( const block& blk
, const size_t& block_size
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const crypto::hash& blk_hash
)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
MDB_val_copy<crypto::hash> val_h(get_block_hash(blk));
MDB_val_copy<crypto::hash> val_h(blk_hash);
MDB_val unused;
if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0)
throw1(BLOCK_EXISTS("Attempting to add block that's already in the db"));
@ -167,8 +169,11 @@ void BlockchainLMDB::add_block( const block& blk
MDB_val_copy<crypto::hash> parent_key(blk.prev_id);
MDB_val parent_h;
if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h))
{
LOG_PRINT_L3("m_height: " << m_height);
LOG_PRINT_L3("parent_key: " << blk.prev_id);
throw0(DB_ERROR("Failed to get top block hash to check for new block's parent"));
}
uint64_t parent_height = *(const uint64_t *)parent_h.mv_data;
if (parent_height != m_height - 1)
throw0(BLOCK_PARENT_DNE("Top block is not new block's parent"));
@ -177,8 +182,9 @@ void BlockchainLMDB::add_block( const block& blk
MDB_val_copy<uint64_t> key(m_height);
MDB_val_copy<blobdata> blob(block_to_blob(blk));
if (mdb_put(*m_write_txn, m_blocks, &key, &blob, 0))
throw0(DB_ERROR("Failed to add block blob to db transaction"));
auto res = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0);
if (res)
throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(res)).c_str()));
MDB_val_copy<size_t> sz(block_size);
if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0))
@ -239,12 +245,12 @@ void BlockchainLMDB::remove_block()
throw1(DB_ERROR("Failed to add removal of block hash to db transaction"));
}
void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx)
void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
MDB_val_copy<crypto::hash> val_h(get_transaction_hash(tx));
MDB_val_copy<crypto::hash> val_h(tx_hash);
MDB_val unused;
if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0)
throw1(TX_EXISTS("Attempting to add transaction that's already in the db"));
@ -382,7 +388,6 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo
check_open();
MDB_val_copy<uint64_t> k(out_index);
MDB_val v;
/****** Uncomment if ever outputs actually need to be stored in this manner
blobdata b;
@ -505,8 +510,8 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image)
char anything = '\0';
unused.mv_size = sizeof(char);
unused.mv_data = &anything;
if (mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0))
throw1(DB_ERROR("Error adding spent key image to db transaction"));
if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0))
throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str()));
}
void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image)
@ -596,15 +601,23 @@ void BlockchainLMDB::check_open() const
BlockchainLMDB::~BlockchainLMDB()
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
// batch transaction shouldn't be active at this point. If it is, consider it aborted.
if (m_batch_active)
batch_abort();
}
BlockchainLMDB::BlockchainLMDB()
BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
// initialize folder to something "safe" just in case
// someone accidentally misuses this class...
m_folder = "thishsouldnotexistbecauseitisgibberish";
m_open = false;
m_batch_transactions = batch_transactions;
m_write_txn = nullptr;
m_batch_active = false;
m_height = 0;
}
@ -627,6 +640,16 @@ void BlockchainLMDB::open(const std::string& filename)
throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str()));
}
// check for existing LMDB files in base directory
boost::filesystem::path old_files = direc.parent_path();
if (boost::filesystem::exists(old_files / "data.mdb") ||
boost::filesystem::exists(old_files / "lock.mdb"))
{
LOG_PRINT_L0("Found existing LMDB files in " << old_files.c_str());
LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart");
throw DB_ERROR("Database could not be opened");
}
m_folder = filename;
// set up lmdb environment
@ -672,7 +695,7 @@ void BlockchainLMDB::open(const std::string& filename)
lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices");
*************************************************/
lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_outputs");
lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys");
mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
mdb_set_dupsort(txn, m_tx_outputs, compare_uint64);
@ -706,6 +729,13 @@ void BlockchainLMDB::create(const std::string& filename)
void BlockchainLMDB::close()
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
if (m_batch_active)
{
LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction");
batch_abort();
}
this->sync();
// FIXME: not yet thread safe!!! Use with care.
mdb_env_close(m_env);
}
@ -713,7 +743,14 @@ void BlockchainLMDB::close()
void BlockchainLMDB::sync()
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
// LMDB documentation leads me to believe this is unnecessary
check_open();
// Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part
// MDB_NOMETASYNC. Force flush to be synchronous.
if (auto result = mdb_env_sync(m_env, true))
{
throw0(DB_ERROR(std::string("Failed to sync database").append(mdb_strerror(result)).c_str()));
}
}
void BlockchainLMDB::reset()
@ -753,7 +790,6 @@ void BlockchainLMDB::unlock()
check_open();
}
bool BlockchainLMDB::block_exists(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@ -769,7 +805,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const
if (get_result == MDB_NOTFOUND)
{
txn.commit();
LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << "not found in db");
LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
return false;
}
else if (get_result)
@ -854,12 +890,17 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const
check_open();
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
MDB_val_copy<uint64_t> key(height);
MDB_val result;
auto get_result = mdb_get(txn, m_block_timestamps, &key, &result);
auto get_result = mdb_get(*txn_ptr, m_block_timestamps, &key, &result);
if (get_result == MDB_NOTFOUND)
{
throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str()));
@ -867,7 +908,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db"));
txn.commit();
if (! m_batch_active)
txn.commit();
return *(const uint64_t*)result.mv_data;
}
@ -891,12 +933,18 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
check_open();
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
MDB_val_copy<uint64_t> key(height);
MDB_val result;
auto get_result = mdb_get(txn, m_block_sizes, &key, &result);
auto get_result = mdb_get(*txn_ptr, m_block_sizes, &key, &result);
if (get_result == MDB_NOTFOUND)
{
throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str()));
@ -904,22 +952,28 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a block size from the db"));
txn.commit();
if (! m_batch_active)
txn.commit();
return *(const size_t*)result.mv_data;
}
difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height);
check_open();
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
MDB_val_copy<uint64_t> key(height);
MDB_val result;
auto get_result = mdb_get(txn, m_block_diffs, &key, &result);
auto get_result = mdb_get(*txn_ptr, m_block_diffs, &key, &result);
if (get_result == MDB_NOTFOUND)
{
throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str()));
@ -927,7 +981,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t&
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db"));
txn.commit();
if (! m_batch_active)
txn.commit();
return *(difficulty_type*)result.mv_data;
}
@ -954,12 +1009,18 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh
check_open();
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
MDB_val_copy<uint64_t> key(height);
MDB_val result;
auto get_result = mdb_get(txn, m_block_coins, &key, &result);
auto get_result = mdb_get(*txn_ptr, m_block_coins, &key, &result);
if (get_result == MDB_NOTFOUND)
{
throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str()));
@ -967,7 +1028,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db"));
txn.commit();
if (! m_batch_active)
txn.commit();
return *(const uint64_t*)result.mv_data;
}
@ -977,20 +1039,28 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height)
check_open();
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
MDB_val_copy<uint64_t> key(height);
MDB_val result;
auto get_result = mdb_get(txn, m_block_hashes, &key, &result);
auto get_result = mdb_get(*txn_ptr, m_block_hashes, &key, &result);
if (get_result == MDB_NOTFOUND)
{
throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str()));
}
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a block hash from the db"));
throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: ").
append(mdb_strerror(get_result)).c_str()));
txn.commit();
if (! m_batch_active)
txn.commit();
return *(crypto::hash*)result.mv_data;
}
@ -1056,23 +1126,33 @@ uint64_t BlockchainLMDB::height() const
return m_height;
}
bool BlockchainLMDB::tx_exists(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
MDB_val_copy<crypto::hash> key(h);
MDB_val result;
auto get_result = mdb_get(txn, m_txs, &key, &result);
TIME_MEASURE_START(time1);
auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result);
TIME_MEASURE_FINISH(time1);
time_tx_exists += time1;
if (get_result == MDB_NOTFOUND)
{
txn.commit();
LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << "not found in db");
if (! m_batch_active)
txn.commit();
LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
return false;
}
else if (get_result)
@ -1094,7 +1174,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const
MDB_val result;
auto get_result = mdb_get(txn, m_tx_unlocks, &key, &result);
if (get_result == MDB_NOTFOUND)
throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str()));
throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash"));
@ -1107,14 +1187,20 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const
check_open();
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
MDB_val_copy<crypto::hash> key(h);
MDB_val result;
auto get_result = mdb_get(txn, m_txs, &key, &result);
auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result);
if (get_result == MDB_NOTFOUND)
throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str()));
throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch tx from hash"));
@ -1124,6 +1210,8 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const
transaction tx;
if (!parse_and_validate_tx_from_blob(bd, tx))
throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
if (! m_batch_active)
txn.commit();
return tx;
}
@ -1165,20 +1253,37 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
// If m_batch_active is set, a batch transaction exists beyond this class,
// such as a batch import with verification enabled, or possibly (later) a
// batch network sync.
//
// A regular network sync without batching would be expected to open a new
// read transaction here, as validation is done prior to the write for block
// and tx data.
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
MDB_val_copy<crypto::hash> key(h);
MDB_val result;
auto get_result = mdb_get(txn, m_tx_heights, &key, &result);
auto get_result = mdb_get(*txn_ptr, m_tx_heights, &key, &result);
if (get_result == MDB_NOTFOUND)
{
throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str()));
throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
}
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch tx height from hash"));
if (! m_batch_active)
txn.commit();
return *(const uint64_t*)result.mv_data;
}
@ -1321,13 +1426,18 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t&
check_open();
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
MDB_val_copy<uint64_t> k(index);
MDB_val v;
auto get_result = mdb_get(txn, m_output_txs, &k, &v);
auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v);
if (get_result == MDB_NOTFOUND)
throw1(OUTPUT_DNE("output with given index not in db"));
else if (get_result)
@ -1335,11 +1445,13 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t&
crypto::hash tx_hash = *(crypto::hash*)v.mv_data;
get_result = mdb_get(txn, m_output_indices, &k, &v);
get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v);
if (get_result == MDB_NOTFOUND)
throw1(OUTPUT_DNE("output with given index not in db"));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch output tx index"));
if (! m_batch_active)
txn.commit();
return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data);
}
@ -1350,10 +1462,15 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con
check_open();
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
lmdb_cur cur(txn, m_output_amounts);
txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
lmdb_cur cur(*txn_ptr, m_output_amounts);
MDB_val_copy<uint64_t> k(amount);
MDB_val v;
@ -1382,7 +1499,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con
cur.close();
txn.commit();
if (! m_batch_active)
txn.commit();
return get_output_tx_and_index_from_global(glob_index);
}
@ -1525,6 +1643,91 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const
return false;
}
void BlockchainLMDB::batch_start()
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
if (! m_batch_transactions)
throw0(DB_ERROR("batch transactions not enabled"));
if (m_batch_active)
throw0(DB_ERROR("batch transaction already in progress"));
if (m_write_txn)
throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use"));
check_open();
// NOTE: need to make sure it's destroyed properly when done
if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
// indicates this transaction is for batch transactions, but not whether it's
// active
m_write_batch_txn.m_batch_txn = true;
m_write_txn = &m_write_batch_txn;
m_batch_active = true;
LOG_PRINT_L3("batch transaction: begin");
}
void BlockchainLMDB::batch_commit()
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
if (! m_batch_transactions)
throw0(DB_ERROR("batch transactions not enabled"));
if (! m_batch_active)
throw0(DB_ERROR("batch transaction not in progress"));
check_open();
LOG_PRINT_L3("batch transaction: committing...");
TIME_MEASURE_START(time1);
m_write_txn->commit();
TIME_MEASURE_FINISH(time1);
time_commit1 += time1;
LOG_PRINT_L3("batch transaction: committed");
if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
if (! m_write_batch_txn.m_batch_txn)
throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction"));
m_write_txn = &m_write_batch_txn;
}
void BlockchainLMDB::batch_stop()
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
if (! m_batch_transactions)
throw0(DB_ERROR("batch transactions not enabled"));
if (! m_batch_active)
throw0(DB_ERROR("batch transaction not in progress"));
check_open();
LOG_PRINT_L3("batch transaction: committing...");
TIME_MEASURE_START(time1);
m_write_txn->commit();
TIME_MEASURE_FINISH(time1);
time_commit1 += time1;
// for destruction of batch transaction
m_write_txn = nullptr;
m_batch_active = false;
LOG_PRINT_L3("batch transaction: end");
}
void BlockchainLMDB::batch_abort()
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
if (! m_batch_transactions)
throw0(DB_ERROR("batch transactions not enabled"));
if (! m_batch_active)
throw0(DB_ERROR("batch transaction not in progress"));
check_open();
// for destruction of batch transaction
m_write_txn = nullptr;
// explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called.
m_write_batch_txn.abort();
m_batch_active = false;
LOG_PRINT_L3("batch transaction: aborted");
}
void BlockchainLMDB::set_batch_transactions(bool batch_transactions)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
m_batch_transactions = batch_transactions;
LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled"));
}
uint64_t BlockchainLMDB::add_block( const block& blk
, const size_t& block_size
, const difficulty_type& cumulative_difficulty
@ -1534,23 +1737,34 @@ uint64_t BlockchainLMDB::add_block( const block& blk
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, 0, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
m_write_txn = &txn;
if (! m_batch_active)
{
if (mdb_txn_begin(m_env, NULL, 0, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
m_write_txn = &txn;
}
uint64_t num_outputs = m_num_outputs;
try
{
BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
m_write_txn = NULL;
if (! m_batch_active)
{
m_write_txn = NULL;
txn.commit();
TIME_MEASURE_START(time1);
txn.commit();
TIME_MEASURE_FINISH(time1);
time_commit1 += time1;
}
}
catch (...)
{
m_num_outputs = num_outputs;
m_write_txn = NULL;
if (! m_batch_active)
m_write_txn = NULL;
throw;
}
@ -1559,18 +1773,27 @@ uint64_t BlockchainLMDB::add_block( const block& blk
void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
if (mdb_txn_begin(m_env, NULL, 0, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
m_write_txn = &txn;
if (! m_batch_active)
{
if (mdb_txn_begin(m_env, NULL, 0, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
m_write_txn = &txn;
}
uint64_t num_outputs = m_num_outputs;
try
{
BlockchainDB::pop_block(blk, txs);
m_write_txn = NULL;
if (! m_batch_active)
{
m_write_txn = NULL;
txn.commit();
txn.commit();
}
}
catch (...)
{

53
src/cryptonote_core/BlockchainDB_impl/db_lmdb.h
View file

@ -38,8 +38,23 @@ struct txn_safe
txn_safe() : m_txn(NULL) { }
~txn_safe()
{
if(m_txn != NULL)
LOG_PRINT_L3("txn_safe: destructor");
if (m_txn != NULL)
{
if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
{
LOG_PRINT_L0("WARNING: txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
}
else
{
// Example of when this occurs: a lookup fails, so a read-only txn is
// aborted through this destructor. However, successful read-only txns
// ideally should have been committed when done and not end up here.
//
// NOTE: not sure if this is ever reached for a non-batch write
// transaction, but it's probably not ideal if it did.
LOG_PRINT_L3("txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
}
mdb_txn_abort(m_txn);
}
}
@ -60,6 +75,24 @@ struct txn_safe
m_txn = NULL;
}
// This should only be needed for batch transaction which must be ensured to
// be aborted before mdb_env_close, not after. So we can't rely on
// BlockchainLMDB destructor to call txn_safe destructor, as that's too late
// to properly abort, since mdb_env_close would have been called earlier.
void abort()
{
LOG_PRINT_L3("txn_safe: abort()");
if(m_txn != NULL)
{
mdb_txn_abort(m_txn);
m_txn = NULL;
}
else
{
LOG_PRINT_L0("WARNING: txn_safe: abort() called, but m_txn is NULL");
}
}
operator MDB_txn*()
{
return m_txn;
@ -71,13 +104,14 @@ struct txn_safe
}
MDB_txn* m_txn;
bool m_batch_txn = false;
};
class BlockchainLMDB : public BlockchainDB
{
public:
BlockchainLMDB();
BlockchainLMDB(bool batch_transactions=false);
~BlockchainLMDB();
virtual void open(const std::string& filename);
@ -177,6 +211,12 @@ public:
, const std::vector<transaction>& txs
);
virtual void set_batch_transactions(bool batch_transactions);
virtual void batch_start();
virtual void batch_commit();
virtual void batch_stop();
virtual void batch_abort();
virtual void pop_block(block& blk, std::vector<transaction>& txs);
private:
@ -184,11 +224,12 @@ private:
, const size_t& block_size
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const crypto::hash& block_hash
);
virtual void remove_block();
virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx);
virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash);
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
@ -263,7 +304,11 @@ private:
uint64_t m_height;
uint64_t m_num_outputs;
std::string m_folder;
txn_safe* m_write_txn;
txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
bool m_batch_transactions; // support for batch transactions
bool m_batch_active; // whether batch transaction is in progress
};
} // namespace cryptonote

5
src/cryptonote_core/blockchain.cpp
View file

@ -237,8 +237,9 @@ bool Blockchain::init(const std::string& config_folder, bool testnet)
m_testnet = testnet;
boost::filesystem::path folder(m_config_folder);
folder /= "lmdb";
LOG_PRINT_L0("Loading blockchain...");
LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ...");
//FIXME: update filename for BlockchainDB
const std::string filename = folder.string();
@ -2288,7 +2289,7 @@ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc
return handle_block_to_main_chain(bl, id, bvc);
}
//------------------------------------------------------------------
void Blockchain::check_against_checkpoints(checkpoints& points, bool enforce)
void Blockchain::check_against_checkpoints(const checkpoints& points, bool enforce)
{
const auto& pts = points.get_points();

2
src/cryptonote_core/blockchain.h
View file

@ -141,7 +141,7 @@ namespace cryptonote
void print_blockchain_index();
void print_blockchain_outs(const std::string& file);
void check_against_checkpoints(checkpoints& points, bool enforce);
void check_against_checkpoints(const checkpoints& points, bool enforce);
void set_enforce_dns_checkpoints(bool enforce);
bool update_checkpoints(const std::string& file_path, bool check_dns);

68
src/cryptonote_core/blockchain_db.cpp
View file

@ -28,6 +28,7 @@
#include "cryptonote_core/blockchain_db.h"
#include "cryptonote_format_utils.h"
#include "profile_tools.h"
using epee::string_tools::pod_to_hex;
@ -41,11 +42,21 @@ void BlockchainDB::pop_block()
pop_block(blk, txs);
}
void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx)
void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr)
{
crypto::hash tx_hash = get_transaction_hash(tx);
crypto::hash tx_hash;
if (!tx_hash_ptr)
{
// should only need to compute hash for miner transactions
tx_hash = get_transaction_hash(tx);
LOG_PRINT_L3("null tx_hash_ptr - needed to compute: " << tx_hash);
}
else
{
tx_hash = *tx_hash_ptr;
}
add_transaction_data(blk_hash, tx);
add_transaction_data(blk_hash, tx, tx_hash);
// iterate tx.vout using indices instead of C++11 foreach syntax because
// we need the index
@ -73,18 +84,33 @@ uint64_t BlockchainDB::add_block( const block& blk
, const std::vector<transaction>& txs
)
{
TIME_MEASURE_START(time1);
crypto::hash blk_hash = get_block_hash(blk);
TIME_MEASURE_FINISH(time1);
time_blk_hash += time1;
// call out to subclass implementation to add the block & metadata
add_block(blk, block_size, cumulative_difficulty, coins_generated);
time1 = epee::misc_utils::get_tick_count();
add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash);
TIME_MEASURE_FINISH(time1);
time_add_block1 += time1;
// call out to add the transactions
time1 = epee::misc_utils::get_tick_count();
add_transaction(blk_hash, blk.miner_tx);
int tx_i = 0;
crypto::hash tx_hash = null_hash;
for (const transaction& tx : txs)
{
add_transaction(blk_hash, tx);
tx_hash = blk.tx_hashes[tx_i];
add_transaction(blk_hash, tx, &tx_hash);
++tx_i;
}
TIME_MEASURE_FINISH(time1);
time_add_transaction += time1;
++num_calls;
return height();
}
@ -119,4 +145,36 @@ void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
remove_transaction_data(tx_hash, tx);
}
void BlockchainDB::reset_stats()
{
num_calls = 0;
time_blk_hash = 0;
time_tx_exists = 0;
time_add_block1 = 0;
time_add_transaction = 0;
time_commit1 = 0;
}
void BlockchainDB::show_stats()
{
LOG_PRINT_L1(ENDL
<< "*********************************"
<< ENDL
<< "num_calls: " << num_calls
<< ENDL
<< "time_blk_hash: " << time_blk_hash << "ms"
<< ENDL
<< "time_tx_exists: " << time_tx_exists << "ms"
<< ENDL
<< "time_add_block1: " << time_add_block1 << "ms"
<< ENDL
<< "time_add_transaction: " << time_add_transaction << "ms"
<< ENDL
<< "time_commit1: " << time_commit1 << "ms"
<< ENDL
<< "*********************************"
<< ENDL
);
}
} // namespace cryptonote

24
src/cryptonote_core/blockchain_db.h
View file

@ -265,13 +265,14 @@ private:
, const size_t& block_size
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const crypto::hash& blk_hash
) = 0;
// tells the subclass to remove data about the top block
virtual void remove_block() = 0;
// tells the subclass to store the transaction and its metadata
virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) = 0;
virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0;
// tells the subclass to remove data about a transaction
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0;
@ -296,17 +297,33 @@ private:
void pop_block();
// helper function for add_transactions, to add each individual tx
void add_transaction(const crypto::hash& blk_hash, const transaction& tx);
void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL);
// helper function to remove transaction from blockchain
void remove_transaction(const crypto::hash& tx_hash);
uint64_t num_calls = 0;
uint64_t time_blk_hash = 0;
uint64_t time_add_block1 = 0;
uint64_t time_add_transaction = 0;
protected:
mutable uint64_t time_tx_exists = 0;
uint64_t time_commit1 = 0;
public:
// virtual dtor
virtual ~BlockchainDB() { };
// reset profiling stats
void reset_stats();
// show profiling stats
void show_stats();
// open the db at location <filename>, or create it if there isn't one.
virtual void open(const std::string& filename) = 0;
@ -336,6 +353,9 @@ public:
// release db lock
virtual void unlock() = 0;
virtual void batch_start() = 0;
virtual void batch_stop() = 0;
virtual void set_batch_transactions(bool) = 0;
// adds a block with the given metadata to the top of the blockchain, returns the new height
// NOTE: subclass implementations of this (or the functions it calls) need