From edef0bb77175d225f93ac68fdc9f2ba37971ed30 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 9 Mar 2015 16:21:44 -0400 Subject: [PATCH 1/8] Initial commit of BDB BlockchainDB implementation Basically verbatim copy of LMDB implementation, but with the guts ripped out and includes changed, etc. --- src/blockchain_db/berkeleydb/db_bdb.cpp | 465 ++++++++++++++++++++++++ src/blockchain_db/berkeleydb/db_bdb.h | 213 +++++++++++ 2 files changed, 678 insertions(+) create mode 100644 src/blockchain_db/berkeleydb/db_bdb.cpp create mode 100644 src/blockchain_db/berkeleydb/db_bdb.h diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp new file mode 100644 index 00000000..9f79c484 --- /dev/null +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -0,0 +1,465 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "db_bdb.h" + +#include +#include // std::unique_ptr +#include // memcpy + +#include "cryptonote_core/cryptonote_format_utils.h" +#include "crypto/crypto.h" +#include "profile_tools.h" + +using epee::string_tools::pod_to_hex; + +namespace +{ + +template +inline void throw0(const T &e) +{ + LOG_PRINT_L0(e.what()); + throw e; +} + +template +inline void throw1(const T &e) +{ + LOG_PRINT_L1(e.what()); + throw e; +} + +} // anonymous namespace + +namespace cryptonote +{ + +void BlockchainBDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& blk_hash + ) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::remove_block() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + +} + +// TODO: probably remove this function +void BlockchainBDB::remove_output(const tx_out& tx_output) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__ << " (unused version - does nothing)"); + return; +} + +void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amount) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::add_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +blobdata BlockchainBDB::output_to_blob(const tx_out& output) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +tx_out BlockchainBDB::output_from_blob(const blobdata& blob) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::check_open() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + if (!m_open) + throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); +} + +BlockchainBDB::~BlockchainBDB() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + +} + +BlockchainBDB::BlockchainBDB(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::open(const std::string& filename) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + +} + +// unused for now, create will happen on open if doesn't exist +void BlockchainBDB::create(const std::string& filename) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed."); +} + +void BlockchainBDB::close() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::sync() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::reset() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + // TODO: this +} + +std::vector BlockchainBDB::get_filenames() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +// TODO: this? +bool BlockchainBDB::lock() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + return false; +} + +// TODO: this? +void BlockchainBDB::unlock() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +bool BlockchainBDB::block_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +block BlockchainBDB::get_block(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +block_header BlockchainBDB::get_block_header(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +block BlockchainBDB::get_block_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_top_block_timestamp() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +size_t BlockchainBDB::get_block_size(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__ << " height: " << height); + check_open(); +} + +difficulty_type BlockchainBDB::get_block_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +std::vector BlockchainBDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +std::vector BlockchainBDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +crypto::hash BlockchainBDB::top_block_hash() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +block BlockchainBDB::get_top_block() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::height() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + return m_height; +} + +bool BlockchainBDB::tx_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_tx_unlock_time(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +transaction BlockchainBDB::get_tx(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_tx_count() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +std::vector BlockchainBDB::get_tx_list(const std::vector& hlist) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +//FIXME: make sure the random method used here is appropriate +uint64_t BlockchainBDB::get_random_output(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +crypto::public_key BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +tx_out BlockchainBDB::get_output(const crypto::hash& h, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +// As this is not used, its return is now a blank output. +// This will save on space in the db. +tx_out BlockchainBDB::get_output(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + return tx_out(); +} + +tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +std::vector BlockchainBDB::get_tx_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +std::vector BlockchainBDB::get_tx_amount_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + + + +bool BlockchainBDB::has_key_image(const crypto::key_image& img) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::batch_start() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_commit() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_stop() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_abort() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::set_batch_transactions(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +uint64_t BlockchainBDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::pop_block(block& blk, std::vector& txs) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +} // namespace cryptonote diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h new file mode 100644 index 00000000..32686e6b --- /dev/null +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -0,0 +1,213 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_protocol/blobdatatype.h" // for type blobdata + +namespace cryptonote +{ + +class BlockchainBDB : public BlockchainDB +{ +public: + BlockchainBDB(bool batch_transactions=false); + ~BlockchainBDB(); + + virtual void open(const std::string& filename); + + virtual void create(const std::string& filename); + + virtual void close(); + + virtual void sync(); + + virtual void reset(); + + virtual std::vector get_filenames() const; + + virtual bool lock(); + + virtual void unlock(); + + virtual bool block_exists(const crypto::hash& h) const; + + virtual block get_block(const crypto::hash& h) const; + + virtual uint64_t get_block_height(const crypto::hash& h) const; + + virtual block_header get_block_header(const crypto::hash& h) const; + + virtual block get_block_from_height(const uint64_t& height) const; + + virtual uint64_t get_block_timestamp(const uint64_t& height) const; + + virtual uint64_t get_top_block_timestamp() const; + + virtual size_t get_block_size(const uint64_t& height) const; + + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; + + virtual difficulty_type get_block_difficulty(const uint64_t& height) const; + + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; + + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; + + virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual crypto::hash top_block_hash() const; + + virtual block get_top_block() const; + + virtual uint64_t height() const; + + virtual bool tx_exists(const crypto::hash& h) const; + + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; + + virtual transaction get_tx(const crypto::hash& h) const; + + virtual uint64_t get_tx_count() const; + + virtual std::vector get_tx_list(const std::vector& hlist) const; + + virtual uint64_t get_tx_block_height(const crypto::hash& h) const; + + virtual uint64_t get_random_output(const uint64_t& amount) const; + + virtual uint64_t get_num_outputs(const uint64_t& amount) const; + + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const; + + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const; + + /** + * @brief get an output from its global index + * + * @param index global index of the output desired + * + * @return the output associated with the index. + * Will throw OUTPUT_DNE if not output has that global index. + * Will throw DB_ERROR if there is a non-specific LMDB error in fetching + */ + tx_out get_output(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; + + virtual std::vector get_tx_output_indices(const crypto::hash& h) const; + virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const; + + virtual bool has_key_image(const crypto::key_image& img) const; + + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ); + + virtual void set_batch_transactions(bool batch_transactions); + virtual void batch_start(); + virtual void batch_commit(); + virtual void batch_stop(); + virtual void batch_abort(); + + virtual void pop_block(block& blk, std::vector& txs); + +private: + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& block_hash + ); + + virtual void remove_block(); + + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); + + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); + + virtual void remove_output(const tx_out& tx_output); + + void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx); + + void remove_output(const uint64_t& out_index, const uint64_t amount); + void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); + + virtual void add_spent_key(const crypto::key_image& k_image); + + virtual void remove_spent_key(const crypto::key_image& k_image); + + /** + * @brief convert a tx output to a blob for storage + * + * @param output the output to convert + * + * @return the resultant blob + */ + blobdata output_to_blob(const tx_out& output); + + /** + * @brief convert a tx output blob to a tx output + * + * @param blob the blob to convert + * + * @return the resultant tx output + */ + tx_out output_from_blob(const blobdata& blob) const; + + /** + * @brief get the global index of the index-th output of the given amount + * + * @param amount the output amount + * @param index the index into the set of outputs of that amount + * + * @return the global index of the desired output + */ + uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const; + + void check_open() const; + + bool m_open; + uint64_t m_height; + uint64_t m_num_outputs; + std::string m_folder; + txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn + txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainBDB + + bool m_batch_transactions; // support for batch transactions + bool m_batch_active; // whether batch transaction is in progress +}; + +} // namespace cryptonote From 5112dc37d75cd7c08661591aade3572128db3f9f Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sat, 14 Mar 2015 17:24:51 -0400 Subject: [PATCH 2/8] Try to not pollute cryptonote namespace --- src/blockchain_db/lmdb/db_lmdb.cpp | 72 +++++++++++++++--------------- src/blockchain_db/lmdb/db_lmdb.h | 22 ++++----- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index ee49e182..1fd03990 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -553,7 +553,7 @@ uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const u LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -665,7 +665,7 @@ void BlockchainLMDB::open(const std::string& filename) throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str())); // get a read/write MDB_txn - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, 0, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -802,7 +802,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -835,7 +835,7 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -865,7 +865,7 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -896,8 +896,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -939,8 +939,8 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -969,8 +969,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1015,8 +1015,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1045,8 +1045,8 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1138,8 +1138,8 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1173,7 +1173,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1193,8 +1193,8 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1228,7 +1228,7 @@ uint64_t BlockchainLMDB::get_tx_count() const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1268,8 +1268,8 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const // read transaction here, as validation is done prior to the write for block // and tx data. - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1312,7 +1312,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1343,7 +1343,7 @@ crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t glob_index = get_output_global_index(amount, index); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1363,7 +1363,7 @@ tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1408,7 +1408,7 @@ tx_out BlockchainLMDB::get_output(const uint64_t& index) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1432,8 +1432,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1468,8 +1468,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1518,7 +1518,7 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& check_open(); std::vector index_vec; - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1563,7 +1563,7 @@ std::vector BlockchainLMDB::get_tx_amount_output_indices(const crypto: transaction tx = get_tx(h); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1634,7 +1634,7 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1745,7 +1745,7 @@ uint64_t BlockchainLMDB::add_block( const block& blk LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (! m_batch_active) { if (mdb_txn_begin(m_env, NULL, 0, txn)) @@ -1783,7 +1783,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (! m_batch_active) { if (mdb_txn_begin(m_env, NULL, 0, txn)) diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 1a684548..a04d0f6b 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -33,17 +33,17 @@ namespace cryptonote { -struct txn_safe +struct mdb_txn_safe { - txn_safe() : m_txn(NULL) { } - ~txn_safe() + mdb_txn_safe() : m_txn(NULL) { } + ~mdb_txn_safe() { - LOG_PRINT_L3("txn_safe: destructor"); + LOG_PRINT_L3("mdb_txn_safe: destructor"); if (m_txn != NULL) { if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety { - LOG_PRINT_L0("WARNING: txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); + LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); } else { @@ -53,7 +53,7 @@ struct txn_safe // // NOTE: not sure if this is ever reached for a non-batch write // transaction, but it's probably not ideal if it did. - LOG_PRINT_L3("txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); + LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); } mdb_txn_abort(m_txn); } @@ -77,11 +77,11 @@ struct txn_safe // This should only be needed for batch transaction which must be ensured to // be aborted before mdb_env_close, not after. So we can't rely on - // BlockchainLMDB destructor to call txn_safe destructor, as that's too late + // BlockchainLMDB destructor to call mdb_txn_safe destructor, as that's too late // to properly abort, since mdb_env_close would have been called earlier. void abort() { - LOG_PRINT_L3("txn_safe: abort()"); + LOG_PRINT_L3("mdb_txn_safe: abort()"); if(m_txn != NULL) { mdb_txn_abort(m_txn); @@ -89,7 +89,7 @@ struct txn_safe } else { - LOG_PRINT_L0("WARNING: txn_safe: abort() called, but m_txn is NULL"); + LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL"); } } @@ -306,8 +306,8 @@ private: uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; - txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn - txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB + mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn + mdb_txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB bool m_batch_transactions; // support for batch transactions bool m_batch_active; // whether batch transaction is in progress From 1bc89398b4cea4406418f7c385025a1d3c062054 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sat, 14 Mar 2015 19:28:34 -0400 Subject: [PATCH 3/8] BerkeleyDB BlockchainDB impl copy/paste/modify LMDB implementation code copy/paste/modified into the Berkeley DB implementation. Need to test if it builds, then if it works, and so on, but the code is all there. --- src/blockchain_db/berkeleydb/db_bdb.cpp | 1233 ++++++++++++++++++++++- src/blockchain_db/berkeleydb/db_bdb.h | 80 +- 2 files changed, 1304 insertions(+), 9 deletions(-) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 9f79c484..4b8065a0 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -54,6 +54,113 @@ inline void throw1(const T &e) throw e; } +// cursor needs to be closed when it goes out of scope, +// this helps if the function using it throws +struct bdb_cur +{ + bdb_cur(DbTxn* txn, Db* dbi) + { + if (dbi->cursor(txn, &m_cur, 0)) + throw0(cryptonote::DB_ERROR("Error opening db cursor")); + done = false; + } + + ~bdb_cur() { close(); } + + operator Dbc*() { return m_cur; } + operator Dbc**() { return &m_cur; } + + void close() + { + if (!done) + { + m_cur->close(); + done = true; + } + } + +private: + Dbc* m_cur; + bool done; +}; + +const char* const BDB_BLOCKS = "blocks"; +const char* const BDB_BLOCK_TIMESTAMPS = "block_timestamps"; +const char* const BDB_BLOCK_HEIGHTS = "block_heights"; +const char* const BDB_BLOCK_HASHES = "block_hashes"; +const char* const BDB_BLOCK_SIZES = "block_sizes"; +const char* const BDB_BLOCK_DIFFS = "block_diffs"; +const char* const BDB_BLOCK_COINS = "block_coins"; + +const char* const BDB_TXS = "txs"; +const char* const BDB_TX_UNLOCKS = "tx_unlocks"; +const char* const BDB_TX_HEIGHTS = "tx_heights"; +const char* const BDB_TX_OUTPUTS = "tx_outputs"; + +const char* const BDB_OUTPUT_TXS = "output_txs"; +const char* const BDB_OUTPUT_INDICES = "output_indices"; +const char* const BDB_OUTPUT_AMOUNTS = "output_amounts"; +const char* const BDB_OUTPUT_KEYS = "output_keys"; + +const char* const BDB_SPENT_KEYS = "spent_keys"; + +template +struct Dbt_copy: public Dbt +{ + Dbt_copy(const T &t): t_copy(t) + { + init(); + } + + Dbt_copy() + { + init(); + } + + void init() + { + set_data(&t_copy); + set_size(sizeof(T)); + set_flags(DB_DBT_USERMEM); + } + + operator T() + { + return t_copy; + } +private: + T t_copy; +}; + +template<> +struct Dbt_copy: public Dbt +{ + Dbt_copy(const cryptonote::blobdata &bd) : m_data(new char[bd.size()]) + { + memcpy(m_data.get(), bd.data(), bd.size()); + set_data(&m_data); + set_size(bd.size()); + } +private: + std::unique_ptr m_data; +}; + +struct Dbt_safe : public Dbt +{ + Dbt_safe() + { + set_data(NULL); + } + ~Dbt_safe() + { + void* buf = get_data(); + if (buf != NULL) + { + free(buf); + } + } +}; + } // anonymous namespace namespace cryptonote @@ -68,36 +175,203 @@ void BlockchainBDB::add_block( const block& blk { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy val_h(blk_hash); + if (m_block_heights->exists(m_write_txn, &val_h, 0) == 0) + throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); + + if (m_height > 0) + { + Dbt_copy parent_key(blk.prev_id); + Dbt_copy parent_h; + if (m_block_heights->get(m_write_txn, &parent_key, &parent_h, 0)) + { + LOG_PRINT_L3("m_height: " << m_height); + LOG_PRINT_L3("parent_key: " << blk.prev_id); + throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); + } + uint64_t parent_height = parent_h; + if (parent_height != m_height - 1) + throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); + } + + Dbt_copy key(m_height); + + Dbt_copy blob(block_to_blob(blk)); + auto res = m_blocks->put(m_write_txn, &key, &blob, DB_APPEND); + if (res) + throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: "))); + + Dbt_copy sz(block_size); + if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, DB_APPEND)) + throw0(DB_ERROR("Failed to add block size to db transaction")); + + Dbt_copy ts(blk.timestamp); + if (m_block_timestamps->put(m_write_txn, &key, &ts, DB_APPEND)) + throw0(DB_ERROR("Failed to add block timestamp to db transaction")); + + Dbt_copy diff(cumulative_difficulty); + if (m_block_diffs->put(m_write_txn, &key, &diff, DB_APPEND)) + throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction")); + + Dbt_copy coinsgen(coins_generated); + if (m_block_coins->put(m_write_txn, &key, &coinsgen, DB_APPEND)) + throw0(DB_ERROR("Failed to add block total generated coins to db transaction")); + + if (m_block_heights->put(m_write_txn, &val_h, &key, DB_APPEND)) + throw0(DB_ERROR("Failed to add block height by hash to db transaction")); + + if (m_block_hashes->put(m_write_txn, &key, &val_h, DB_APPEND)) + throw0(DB_ERROR("Failed to add block hash to db transaction")); } void BlockchainBDB::remove_block() { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + if (m_height == 0) + throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); + + Dbt_copy k(m_height - 1); + Dbt_copy h; + if (m_block_hashes->get(m_write_txn, &k, &h, 0)) + throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); + + if (m_blocks->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block to db transaction")); + + if (m_block_sizes->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block size to db transaction")); + + if (m_block_diffs->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); + + if (m_block_coins->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); + + if (m_block_timestamps->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); + + if (m_block_heights->del(m_write_txn, &h, 0)) + throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); + + if (m_block_hashes->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); } void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy val_h(tx_hash); + + if (m_txs->exists(m_write_txn, &val_h, 0) == 0) + throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); + + Dbt_copy blob(tx_to_blob(tx)); + if (m_txs->put(m_write_txn, &val_h, &blob, DB_APPEND)) + throw0(DB_ERROR("Failed to add tx blob to db transaction")); + + Dbt_copy height(m_height); + if (m_tx_heights->put(m_write_txn, &val_h, &height, DB_APPEND)) + throw0(DB_ERROR("Failed to add tx block height to db transaction")); + + Dbt_copy unlock_time(tx.unlock_time); + if (m_tx_unlocks->put(m_write_txn, &val_h, &unlock_time, DB_APPEND)) + throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); } void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy val_h(tx_hash); + if (m_txs->exists(m_write_txn, &val_h, 0)) + throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + + if (m_txs->del(m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx to db transaction")); + if (m_tx_unlocks->del(m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); + if (m_tx_heights->del(m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); + + remove_tx_outputs(tx_hash, tx); + + if (m_tx_outputs->del(m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); + } void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy k(m_num_outputs); + Dbt_copy v(tx_hash); + + if (m_output_txs->put(m_write_txn, &k, &v, DB_APPEND)) + throw0(DB_ERROR("Failed to add output tx hash to db transaction")); + if (m_tx_outputs->put(m_write_txn, &v, &k, DB_APPEND)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + Dbt_copy val_local_index(local_index); + if (m_output_indices->put(m_write_txn, &k, &val_local_index, DB_APPEND)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + Dbt_copy val_amount(tx_output.amount); + if (auto result = m_output_amounts->put(m_write_txn, &val_amount, &k, DB_APPEND)) + throw0(DB_ERROR(std::string("Failed to add output amount to db transaction"))); + + if (tx_output.target.type() == typeid(txout_to_key)) + { + Dbt_copy val_pubkey(boost::get(tx_output.target).key); + if (m_output_keys->put(m_write_txn, &k, &val_pubkey, DB_APPEND)) + throw0(DB_ERROR("Failed to add output pubkey to db transaction")); + } + + m_num_outputs++; } void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) { LOG_PRINT_L3("BlockchainBDB::" << __func__); + bdb_cur cur(m_write_txn, m_tx_outputs); + + Dbt_copy k(tx_hash); + Dbt_copy v; + + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + { + throw0(DB_ERROR("Attempting to remove a tx's outputs, but none found.")); + } + else if (result) + { + throw0(DB_ERROR("DB error attempting to get an output")); + } + else + { + db_recno_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + for (uint64_t i = 0; i < num_elems; ++i) + { + const tx_out tx_output = tx.vout[i]; + remove_output(v, tx_output.amount); + if (i < num_elems - 1) + { + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + } + } + + cur.close(); } // TODO: probably remove this function @@ -111,40 +385,179 @@ void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amou { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy k(out_index); + + auto result = m_output_indices->del(m_write_txn, &k, 0); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); + } + + result = m_output_txs->del(m_write_txn, &k, 0); + // if (result != 0 && result != DB_NOTFOUND) + // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + } + + result = m_output_keys->del(m_write_txn, &k, 0); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); + } + else if (result) + throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); + + remove_amount_output_index(amount, out_index); + + m_num_outputs--; } void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_cur cur(m_write_txn, m_output_amounts); + + Dbt_copy k(amount); + Dbt_copy v; + + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + uint64_t amount_output_index = 0; + uint64_t goi = 0; + bool found_index = false; + for (uint64_t i = 0; i < num_elems; ++i) + { + goi = v; + if (goi == global_output_index) + { + amount_output_index = i; + found_index = true; + break; + } + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + if (found_index) + { + // found the amount output index + // now delete it + result = (Dbc*)cur->del(0); + if (result) + throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast(amount_output_index)).c_str())); + } + else + { + // not found + throw1(OUTPUT_DNE("Failed to find amount output index")); + } + cur.close(); } void BlockchainBDB::add_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy val_key(k_image); + if (m_spent_keys->exists(m_write_txn, &val_key, 0) == 0) + throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); + + Dbt_copy val('\0'); + if (auto result = m_spent_keys->put(m_write_txn, &val_key, &val, DB_APPEND)) + throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: "))); } void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy k(k_image); + auto result = m_spent_keys->del(m_write_txn, &k, 0); + if (result != 0 && result != DB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of key image to db transaction")); } blobdata BlockchainBDB::output_to_blob(const tx_out& output) { LOG_PRINT_L3("BlockchainBDB::" << __func__); + blobdata b; + if (!t_serializable_object_to_blob(output, b)) + throw1(DB_ERROR("Error serializing output to blob")); + return b; } tx_out BlockchainBDB::output_from_blob(const blobdata& blob) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); + std::stringstream ss; + ss << blob; + binary_archive ba(ss); + tx_out o; + + if (!(::serialization::serialize(ba, o))) + throw1(DB_ERROR("Error deserializing tx output blob")); + + return o; } uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const { - LOG_PRINT_L3("BlockchainBDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy k(amount); + Dbt_copy v; + + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems; + (Dbc*)cur->count(&num_elems, 0); + + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + for (uint64_t i = 0; i < index; ++i) + { + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + + uint64_t glob_index = v; + + cur.close(); + + txn.commit(); + + return glob_index; } void BlockchainBDB::check_open() const @@ -158,17 +571,160 @@ BlockchainBDB::~BlockchainBDB() { LOG_PRINT_L3("BlockchainBDB::" << __func__); + if (m_open) + { + close(); + } } BlockchainBDB::BlockchainBDB(bool batch_transactions) { LOG_PRINT_L3("BlockchainBDB::" << __func__); + // initialize folder to something "safe" just in case + // someone accidentally misuses this class... + m_folder = "thishsouldnotexistbecauseitisgibberish"; + m_open = false; + + m_batch_transactions = batch_transactions; + m_write_txn = nullptr; + m_height = 0; } void BlockchainBDB::open(const std::string& filename) { LOG_PRINT_L3("BlockchainBDB::" << __func__); + if (m_open) + throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); + + boost::filesystem::path direc(filename); + if (boost::filesystem::exists(direc)) + { + if (!boost::filesystem::is_directory(direc)) + throw0(DB_OPEN_FAILURE("DB needs a directory path, but a file was passed")); + } + else + { + if (!boost::filesystem::create_directory(direc)) + throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); + } + + m_folder = filename; + + try + { + + //Create BerkeleyDB environment + m_env = new DbEnv(0); // no flags needed for DbEnv + + uint32_t db_env_open_flags = DB_CREATE | DB_INIT_MPOOL | DB_INIT_LOCK + | DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER + | DB_THREAD; + + // last parameter left 0, files will be created with default rw access + m_env->open(filename.c_str(), db_env_open_flags, 0); + + // begin transaction to init dbs + bdb_txn_safe txn; + m_env->txn_begin(NULL, txn, 0); + + // create Dbs in the environment + m_blocks = new Db(m_env); + m_block_heights = new Db(m_env); + m_block_hashes = new Db(m_env); + m_block_timestamps = new Db(m_env); + m_block_sizes = new Db(m_env); + m_block_diffs = new Db(m_env); + m_block_coins = new Db(m_env); + + m_txs = new Db(m_env); + m_tx_unlocks = new Db(m_env); + m_tx_heights = new Db(m_env); + m_tx_outputs = new Db(m_env); + + m_output_txs = new Db(m_env); + m_output_indices = new Db(m_env); + m_output_amounts = new Db(m_env); + m_output_keys = new Db(m_env); + + m_spent_keys = new Db(m_env); + + // Tell DB about Dbs that need duplicate support + // Note: no need to tell about sorting, + // as the default is insertion order, which we want + m_tx_outputs->set_flags(DB_DUP); + m_output_amounts->set_flags(DB_DUP); + + // Tell DB about fixed-size values. + m_block_heights->set_re_len(sizeof(uint64_t)); + m_block_hashes->set_re_len(sizeof(crypto::hash)); + m_block_timestamps->set_re_len(sizeof(uint64_t)); + m_block_sizes->set_re_len(sizeof(size_t)); // should really store block size as uint64_t... + m_block_diffs->set_re_len(sizeof(difficulty_type)); + m_block_coins->set_re_len(sizeof(uint64_t)); + + m_tx_unlocks->set_re_len(sizeof(uint64_t)); + m_tx_heights->set_re_len(sizeof(uint64_t)); + m_tx_outputs->set_re_len(sizeof(uint64_t)); + + m_output_txs->set_re_len(sizeof(uint64_t)); + m_output_indices->set_re_len(sizeof(uint64_t)); + m_output_amounts->set_re_len(sizeof(uint64_t)); + m_output_keys->set_re_len(sizeof(crypto::public_key)); + + m_spent_keys->set_re_len(sizeof(crypto::key_image)); + + //TODO: Find out if we need to do Db::set_flags(DB_RENUMBER) + // for the RECNO databases. We shouldn't as we're only + // inserting/removing from the end, but we'll see. + + // open Dbs in the environment + // m_tx_outputs and m_output_amounts must be DB_HASH or DB_BTREE + // because they need duplicate entry support. The rest are DB_RECNO, + // as it seems that will be the most performant choice. + m_blocks->open(txn, BDB_BLOCKS, NULL, DB_RECNO, DB_CREATE, 0); + + m_block_timestamps->open(txn, BDB_BLOCK_TIMESTAMPS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_heights->open(txn, BDB_BLOCK_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_hashes->open(txn, BDB_BLOCK_HASHES, NULL, DB_RECNO, DB_CREATE, 0); + m_block_sizes->open(txn, BDB_BLOCK_SIZES, NULL, DB_RECNO, DB_CREATE, 0); + m_block_diffs->open(txn, BDB_BLOCK_DIFFS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_coins->open(txn, BDB_BLOCK_COINS, NULL, DB_RECNO, DB_CREATE, 0); + + m_txs->open(txn, BDB_TXS, NULL, DB_RECNO, DB_CREATE, 0); + m_tx_unlocks->open(txn, BDB_TX_UNLOCKS, NULL, DB_RECNO, DB_CREATE, 0); + m_tx_heights->open(txn, BDB_TX_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0); + m_tx_outputs->open(txn, BDB_TX_OUTPUTS, NULL, DB_HASH, DB_CREATE, 0); + + m_output_txs->open(txn, BDB_OUTPUT_TXS, NULL, DB_RECNO, DB_CREATE, 0); + m_output_indices->open(txn, BDB_OUTPUT_INDICES, NULL, DB_RECNO, DB_CREATE, 0); + m_output_amounts->open(txn, BDB_OUTPUT_AMOUNTS, NULL, DB_HASH, DB_CREATE, 0); + m_output_keys->open(txn, BDB_OUTPUT_KEYS, NULL, DB_RECNO, DB_CREATE, 0); + + m_spent_keys->open(txn, BDB_SPENT_KEYS, NULL, DB_RECNO, DB_CREATE, 0); + + DB_BTREE_STAT* stats; + + // DB_FAST_STAT can apparently cause an incorrect number of records + // to be returned. The flag should be set to 0 instead if this proves + // to be the case. + m_blocks->stat(txn, &stats, DB_FAST_STAT); + m_height = stats->bt_nkeys; + delete stats; + + // see above comment about DB_FAST_STAT + m_output_indices->stat(txn, &stats, DB_FAST_STAT); + m_num_outputs = stats->bt_nkeys; + delete stats; + + txn->commit(); + } + catch (const std::exception& e) + { + throw0(DB_OPEN_FAILURE(e.what())); + } + + m_open = true; } // unused for now, create will happen on open if doesn't exist @@ -180,24 +736,117 @@ void BlockchainBDB::create(const std::string& filename) void BlockchainBDB::close() { - LOG_PRINT_L3("BlockchainBDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + this->sync(); + + // FIXME: not yet thread safe!!! Use with care. + m_env->close(DB_FORCESYNC); } void BlockchainBDB::sync() { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + try + { + m_blocks->sync(); + m_block_heights->sync(); + m_block_hashes->sync(); + m_block_timestamps->sync(); + m_block_sizes->sync(); + m_block_diffs->sync(); + m_block_coins->sync(); + + m_txs->sync(); + m_tx_unlocks->sync(); + m_tx_heights->sync(); + m_tx_outputs->sync(); + + m_output_txs->sync(); + m_output_indices->sync(); + m_output_amounts->sync(); + m_output_keys->sync(); + + m_spent_keys->sync(); + } + catch (const std::exception& e) + { + throw0(DB_ERROR(std::string("Failed to sync database: ").append(e.what()))); + } } void BlockchainBDB::reset() { - LOG_PRINT_L3("BlockchainBDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); // TODO: this } std::vector BlockchainBDB::get_filenames() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + std::vector filenames; + + char *fname, *dbname; + + m_blocks->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_heights->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_hashes->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_timestamps->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_sizes->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_diffs->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_coins->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_txs->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_tx_unlocks->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_tx_heights->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_tx_outputs->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_output_txs->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_output_indices->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_output_amounts->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_output_keys->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_spent_keys->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + return filenames; +} + +std::string BlockchainBDB::get_db_name() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + + return std::string("BerkeleyDB"); } // TODO: this? @@ -219,96 +868,296 @@ bool BlockchainBDB::block_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + + auto get_result = m_block_heights->exists(txn, &key, 0); + if (get_result == DB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch block index from hash")); + + txn.commit(); + return true; } block BlockchainBDB::get_block(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + return get_block_from_height(get_block_height(h)); } uint64_t BlockchainBDB::get_block_height(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + Dbt_copy result; + + auto get_result = m_block_heights->get(txn, &key, &result, 0); + if (get_result == MDB_NOTFOUND) + throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); + + txn.commit(); + return result; } block_header BlockchainBDB::get_block_header(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + // block_header object is automatically cast from block object + return get_block(h); } block BlockchainBDB::get_block_from_height(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_safe result; + auto get_result = mdb_get(txn, m_blocks, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast(height)).append(" failed -- block not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block from the db")); + + txn.commit(); + + blobdata bd; + bd.assign(reinterpret_cast(result.get_data()), result.get_size()); + + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + + return b; } uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_copy result; + auto get_result = m_block_timestamps->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast(height)).append(" failed -- timestamp not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); + + txn.commit(); + return result; } uint64_t BlockchainBDB::get_top_block_timestamp() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + // if no blocks, return 0 + if (m_height == 0) + { + return 0; + } + + return get_block_timestamp(m_height - 1); } size_t BlockchainBDB::get_block_size(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_copy result; + auto get_result = m_block_sizes->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); + + txn.commit(); + return result; } difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__ << " height: " << height); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_copy result; + auto get_result = m_block_diffs->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); + + txn.commit(); + return result; } difficulty_type BlockchainBDB::get_block_difficulty(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + difficulty_type diff1 = 0; + difficulty_type diff2 = 0; + + diff1 = get_block_cumulative_difficulty(height); + if (height != 0) + { + diff2 = get_block_cumulative_difficulty(height - 1); + } + + return diff1 - diff2; } uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_copy result; + auto get_result = m_block_coins->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); + + txn.commit(); + return result; } crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_copy result; + auto get_result = m_block_hashes->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). + append(mdb_strerror(get_result)).c_str())); + + txn.commit(); + return result; } std::vector BlockchainBDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + std::vector v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_from_height(height)); + } + + return v; } std::vector BlockchainBDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + std::vector v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_hash_from_height(height)); + } + + return v; } crypto::hash BlockchainBDB::top_block_hash() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + if (m_height != 0) + { + return get_block_hash_from_height(m_height - 1); + } + + return null_hash; } block BlockchainBDB::get_top_block() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + if (m_height != 0) + { + return get_block_from_height(m_height - 1); + } + + block b; + return b; } uint64_t BlockchainBDB::height() const @@ -323,36 +1172,137 @@ bool BlockchainBDB::tx_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + + TIME_MEASURE_START(time1); + auto get_result = m_txs->exists(txn, &key, 0); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; + if (get_result == DB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); + + return true; } uint64_t BlockchainBDB::get_tx_unlock_time(const crypto::hash& h) const { - LOG_PRINT_L3("BlockchainBDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + Dbt_copy result; + auto get_result = m_tx_unlocks->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); + + return result; } transaction BlockchainBDB::get_tx(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + Dbt_safe result; + auto get_result = m_txs->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx from hash")); + + blobdata bd; + bd.assign(reinterpret_cast(result.get_data()), result.get_size()); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + + txn.commit(); + + return tx; } uint64_t BlockchainBDB::get_tx_count() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + DB_BTREE_STAT* stats; + + // DB_FAST_STAT can apparently cause an incorrect number of records + // to be returned. The flag should be set to 0 instead if this proves + // to be the case. + m_txs->stat(txn, &stats, DB_FAST_STAT); + auto num_txs = stats->bt_nkeys; + delete stats; + + txn.commit(); + + return num_txs; } std::vector BlockchainBDB::get_tx_list(const std::vector& hlist) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + std::vector v; + + for (auto& h : hlist) + { + v.push_back(get_tx(h)); + } + + return v; } uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + Dbt_copy result; + auto get_result = m_tx_heights->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + + txn.commit(); + + return result; } //FIXME: make sure the random method used here is appropriate @@ -360,24 +1310,71 @@ uint64_t BlockchainBDB::get_random_output(const uint64_t& amount) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + uint64_t num_outputs = get_num_outputs(amount); + if (num_outputs == 0) + throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist")); + + return crypto::rand() % num_outputs; } uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy k(amount); + Dbt_copy v; + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + { + return 0; + } + else if (result) + throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); + + size_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + txn.commit(); + + return num_elems; } crypto::public_key BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + uint64_t glob_index = get_output_global_index(amount, index); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy k(glob_index); + Dbt_copy v; + auto get_result = m_output_keys->get(txn, &k, &v, 0); + if (get_result == DB_NOTFOUND) + throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + + return v; } +// As this is not used, its return is now a blank output. +// This will save on space in the db. tx_out BlockchainBDB::get_output(const crypto::hash& h, const uint64_t& index) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); + return tx_out(); } // As this is not used, its return is now a blank output. @@ -392,24 +1389,181 @@ tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy k(index); + Dbt_copy v; + + auto get_result = m_output_txs->get(txn, &k, &v, 0); + if (get_result == DB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + crypto::hash tx_hash = v; + + Dbt_copy result; + get_result = m_output_indices->get(txn, &k, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx index")); + + txn.commit(); + + return tx_out_index(tx_hash, result); } tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy k(amount); + Dbt_copy v; + + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + for (uint64_t i = 0; i < index; ++i) + { + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + + uint64_t glob_index = v; + + cur.close(); + + txn.commit(); + + return get_output_tx_and_index_from_global(glob_index); } std::vector BlockchainBDB::get_tx_output_indices(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + std::vector index_vec; + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_tx_outputs); + + Dbt_copy k(h); + Dbt_copy v; + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + for (uint64_t i = 0; i < num_elems; ++i) + { + index_vec.push_back(v); + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + + cur.close(); + txn.commit(); + + return index_vec; } std::vector BlockchainBDB::get_tx_amount_output_indices(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + std::vector index_vec; + std::vector index_vec2; + + // get the transaction's global output indices first + index_vec = get_tx_output_indices(h); + // these are next used to obtain the amount output indices + + transaction tx = get_tx(h); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + uint64_t i = 0; + uint64_t global_index; + for (const auto& vout : tx.vout) + { + uint64_t amount = vout.amount; + + global_index = index_vec[i]; + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy k(amount); + Dbt_copy v; + + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + uint64_t amount_output_index = 0; + uint64_t output_index = 0; + bool found_index = false; + for (uint64_t j = 0; j < num_elems; ++j) + { + output_index = v; + if (output_index == global_index) + { + amount_output_index = j; + found_index = true; + break; + } + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + if (found_index) + { + index_vec2.push_back(amount_output_index); + } + else + { + // not found + cur.close(); + txn.commit(); + throw1(OUTPUT_DNE("specified output not found in db")); + } + + cur.close(); + ++i; + } + + txn.commit(); + + return index_vec2; } @@ -418,8 +1572,25 @@ bool BlockchainBDB::has_key_image(const crypto::key_image& img) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy val_key(img); + if (m_spent_keys->exists(txn, &val_key, 0) == 0) + { + txn.commit(); + return true; + } + + txn.commit(); + return false; } +// Ostensibly BerkeleyDB has batch transaction support built-in, +// so the following few functions will be NOP. + void BlockchainBDB::batch_start() { LOG_PRINT_L3("BlockchainBDB::" << __func__); @@ -442,7 +1613,9 @@ void BlockchainBDB::batch_abort() void BlockchainBDB::set_batch_transactions(bool batch_transactions) { - LOG_PRINT_L3("BlockchainBDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + m_batch_transactions = batch_transactions; + LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } uint64_t BlockchainBDB::add_block( const block& blk @@ -454,12 +1627,60 @@ uint64_t BlockchainBDB::add_block( const block& blk { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + m_write_txn = NULL; + + TIME_MEASURE_START(time1); + txn.commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + } + } + catch (...) + { + m_num_outputs = num_outputs; + m_write_txn = NULL; + throw; + } + + return ++m_height; } void BlockchainBDB::pop_block(block& blk, std::vector& txs) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::pop_block(blk, txs); + + m_write_txn = NULL; + txn.commit(); + } + catch (...) + { + m_num_outputs = num_outputs; + m_write_txn = NULL; + throw; + } + + --m_height; } } // namespace cryptonote diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index 32686e6b..b28afbf2 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -25,12 +25,66 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include + #include "blockchain_db/blockchain_db.h" #include "cryptonote_protocol/blobdatatype.h" // for type blobdata namespace cryptonote { +struct bdb_txn_safe +{ + bdb_txn_safe() : m_txn(NULL) { } + ~bdb_txn_safe() + { + LOG_PRINT_L3("bdb_txn_safe: destructor"); + abort(); + } + + void commit(std::string message = "") + { + if (message.size() == 0) + { + message = "Failed to commit a transaction to the db"; + } + + if (txn->commit()) + { + m_txn = NULL; + LOG_PRINT_L0(message); + throw DB_ERROR(message.c_str()); + } + m_txn = NULL; + } + + void abort() + { + LOG_PRINT_L3("bdb_txn_safe: abort()"); + if(m_txn != NULL) + { + m_txn->abort(); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: bdb_txn_safe: abort() called, but m_txn is NULL"); + } + } + + operator DbTxn*() + { + return m_txn; + } + + operator DbTxn**() + { + return &m_txn; + } + + DbTxn* m_txn; +}; + class BlockchainBDB : public BlockchainDB { public: @@ -199,15 +253,35 @@ private: void check_open() const; + DbEnv* m_env; + + Db* m_blocks; + Db* m_block_heights; + Db* m_block_hashes; + Db* m_block_timestamps; + Db* m_block_sizes; + Db* m_block_diffs; + Db* m_block_coins; + + Db* m_txs; + Db* m_tx_unlocks; + Db* m_tx_heights; + Db* m_tx_outputs; + + Db* m_output_txs; + Db* m_output_indices; + Db* m_output_amounts; + Db* m_output_keys; + + Db* m_spent_keys; + bool m_open; uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; - txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn - txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainBDB + txn_safe m_write_txn; // may point to either a short-lived txn or a batch txn bool m_batch_transactions; // support for batch transactions - bool m_batch_active; // whether batch transaction is in progress }; } // namespace cryptonote From cade0da8f1e932a17886aa9893a580fa3e3289c7 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 16 Mar 2015 03:12:54 -0400 Subject: [PATCH 4/8] CMake wiring, minor cleanup, minor test addition Make Cmake things aware of BerkeleyDB and BlockchainBDB Make the BlockchainDB unit tests aware of BlockchainBDB --- CMakeLists.txt | 3 +++ external/CMakeLists.txt | 5 +++++ external/db_drivers/CMakeLists.txt | 24 +++++++++++++++++++++++- src/blockchain_db/CMakeLists.txt | 3 +++ tests/unit_tests/BlockchainDB.cpp | 3 ++- tests/unit_tests/CMakeLists.txt | 1 + 6 files changed, 37 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2916ff13..9e29dab5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -162,6 +162,9 @@ include_directories(external/rapidjson) # Final setup for liblmdb include_directories(${LMDB_INCLUDE}) +# Final setup for Berkeley DB +include_directories(${BDB_INCLUDE}) + if(MSVC) add_definitions("/bigobj /MP /W3 /GS- /D_CRT_SECURE_NO_WARNINGS /wd4996 /wd4345 /D_WIN32_WINNT=0x0600 /DWIN32_LEAN_AND_MEAN /DGTEST_HAS_TR1_TUPLE=0 /FIinline_c.h /D__SSE4_1__") # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Dinline=__inline") diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index f0d363e3..527c5b5a 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -103,3 +103,8 @@ set(LMDB_STATIC ${LMDB_STATIC} PARENT_SCOPE) set(LMDB_INCLUDE ${LMDB_INCLUDE} PARENT_SCOPE) set(LMDB_LIBRARY ${LMDB_LIBRARY} PARENT_SCOPE) set(LMDB_LIBRARY_DIRS ${LMDB_LIBRARY_DIRS} PARENT_SCOPE) + +set(BDB_STATIC ${BDB_STATIC} PARENT_SCOPE) +set(BDB_INCLUDE ${BDB_INCLUDE} PARENT_SCOPE) +set(BDB_LIBRARY ${BDB_LIBRARY} PARENT_SCOPE) +set(BDB_LIBRARY_DIRS ${BDB_LIBRARY_DIRS} PARENT_SCOPE) diff --git a/external/db_drivers/CMakeLists.txt b/external/db_drivers/CMakeLists.txt index 5993ed3a..0498eaaf 100644 --- a/external/db_drivers/CMakeLists.txt +++ b/external/db_drivers/CMakeLists.txt @@ -44,6 +44,28 @@ else() set(LMDB_LIBRARY ${LMDB_LIBRARIES} PARENT_SCOPE) set(LMDB_LIBRARY_DIRS "" PARENT_SCOPE) else() - die("Found liblmdb includes, but could not find liblmdb library. Please make sure you have installed liblmdb or liblmdb-dev or the equivalent") + die("Found liblmdb includes, but could not find liblmdb library. Please make sure you have installed liblmdb and liblmdb-dev or the equivalent") + endif() +endif() + +find_package(BerkeleyDB) + +if(NOT DB_LIBRARIES OR STATIC) + add_subdirectory(libdb) + message(STATUS "BerkeleyDB not found, building from src tree") + + set(BDB_STATIC true PARENT_SCOPE) + set(BDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/libdb" PARENT_SCOPE) + set(BDB_LIBRARY "db" PARENT_SCOPE) +else() + message(STATUS "Found BerkeleyDB include (db.h) in ${DB_INCLUDE_DIR}") + if(DB_LIBRARIES) + message(STATUS "Found BerkeleyDB shared library") + set(BDB_STATIC false PARENT_SCOPE) + set(BDB_INCLUDE ${DB_INCLUDE_DIR} PARENT_SCOPE) + set(BDB_LIBRARY ${DB_LIBRARIES} PARENT_SCOPE) + set(BDB_LIBRARY_DIRS "" PARENT_SCOPE) + else() + die("Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev or the equivalent") endif() endif() diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index 84b2d6a7..62b2eabe 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -29,6 +29,7 @@ set(blockchain_db_sources blockchain_db.cpp lmdb/db_lmdb.cpp + berkeleydb/db_bdb.cpp ) set(blockchain_db_headers) @@ -36,6 +37,7 @@ set(blockchain_db_headers) set(blockchain_db_private_headers blockchain_db.h lmdb/db_lmdb.h + berkeleydb/db_bdb.h ) bitmonero_private_headers(blockchain_db @@ -57,4 +59,5 @@ target_link_libraries(blockchain_db ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} ${LMDB_LIBRARY} + ${BDB_LIBRARY} ${EXTRA_LIBRARIES}) diff --git a/tests/unit_tests/BlockchainDB.cpp b/tests/unit_tests/BlockchainDB.cpp index 4d39d7da..bbe8582f 100644 --- a/tests/unit_tests/BlockchainDB.cpp +++ b/tests/unit_tests/BlockchainDB.cpp @@ -35,6 +35,7 @@ #include "blockchain_db/blockchain_db.h" #include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/berkeleydb/db_bdb.h" #include "cryptonote_core/cryptonote_format_utils.h" using namespace cryptonote; @@ -209,7 +210,7 @@ protected: using testing::Types; -typedef Types implementations; +typedef Types implementations; TYPED_TEST_CASE(BlockchainDBTest, implementations); diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index 54f25431..b8a22e31 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -57,6 +57,7 @@ add_executable(unit_tests target_link_libraries(unit_tests LINK_PRIVATE cryptonote_core + blockchain_db rpc wallet ${GTEST_MAIN_LIBRARIES} From 43477b7dac861a83ade364ebb5a5c3da8228b3e4 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 16 Mar 2015 09:14:51 -0400 Subject: [PATCH 5/8] BerkeleyDB Blockchain building, not working yet Everything except actually *using* BlockchainBDB is wired up, but the db itself is not yet working. Some error about user mem not large enough. I think I know what this error means, but I can't determine the cause. Notes: BerkeleyDB does not allow 0-indexing in its recno type databases, so block numbers *in the database* will be 1-indexed. Modifications to indexing have been made as needed. --- cmake/FindBerkeleyDB.cmake | 25 ++ external/db_drivers/CMakeLists.txt | 10 +- src/blockchain_db/CMakeLists.txt | 4 +- src/blockchain_db/berkeleydb/db_bdb.cpp | 367 ++++++++++++------------ src/blockchain_db/berkeleydb/db_bdb.h | 6 +- tests/unit_tests/BlockchainDB.cpp | 28 +- 6 files changed, 247 insertions(+), 193 deletions(-) create mode 100644 cmake/FindBerkeleyDB.cmake diff --git a/cmake/FindBerkeleyDB.cmake b/cmake/FindBerkeleyDB.cmake new file mode 100644 index 00000000..916d8f9c --- /dev/null +++ b/cmake/FindBerkeleyDB.cmake @@ -0,0 +1,25 @@ +# - Try to find Berkeley DB +# Once done this will define +# +# BERKELEY_DB_FOUND - system has Berkeley DB +# BERKELEY_DB_INCLUDE_DIR - the Berkeley DB include directory +# BERKELEY_DB_LIBRARIES - Link these to use Berkeley DB +# BERKELEY_DB_DEFINITIONS - Compiler switches required for using Berkeley DB + +# Copyright (c) 2006, Alexander Dymo, +# +# Redistribution and use is allowed according to the terms of the BSD license. +# For details see the accompanying COPYING-CMAKE-SCRIPTS file. + +find_path(BERKELEY_DB_INCLUDE_DIR db_cxx.h + /usr/include/db4 + /usr/local/include/db4 +) + +find_library(BERKELEY_DB_LIBRARIES NAMES db_cxx ) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Berkeley "Could not find Berkeley DB >= 4.1" BERKELEY_DB_INCLUDE_DIR BERKELEY_DB_LIBRARIES) +# show the BERKELEY_DB_INCLUDE_DIR and BERKELEY_DB_LIBRARIES variables only in the advanced view +mark_as_advanced(BERKELEY_DB_INCLUDE_DIR BERKELEY_DB_LIBRARIES ) + diff --git a/external/db_drivers/CMakeLists.txt b/external/db_drivers/CMakeLists.txt index 0498eaaf..f90f9f13 100644 --- a/external/db_drivers/CMakeLists.txt +++ b/external/db_drivers/CMakeLists.txt @@ -50,7 +50,7 @@ endif() find_package(BerkeleyDB) -if(NOT DB_LIBRARIES OR STATIC) +if(NOT BERKELEY_DB_LIBRARIES OR STATIC) add_subdirectory(libdb) message(STATUS "BerkeleyDB not found, building from src tree") @@ -58,12 +58,12 @@ if(NOT DB_LIBRARIES OR STATIC) set(BDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/libdb" PARENT_SCOPE) set(BDB_LIBRARY "db" PARENT_SCOPE) else() - message(STATUS "Found BerkeleyDB include (db.h) in ${DB_INCLUDE_DIR}") - if(DB_LIBRARIES) + message(STATUS "Found BerkeleyDB include (db.h) in ${BERKELEY_DB_INCLUDE_DIR}") + if(BERKELEY_DB_LIBRARIES) message(STATUS "Found BerkeleyDB shared library") set(BDB_STATIC false PARENT_SCOPE) - set(BDB_INCLUDE ${DB_INCLUDE_DIR} PARENT_SCOPE) - set(BDB_LIBRARY ${DB_LIBRARIES} PARENT_SCOPE) + set(BDB_INCLUDE ${BERKELEY_DB_INCLUDE_DIR} PARENT_SCOPE) + set(BDB_LIBRARY ${BERKELEY_DB_LIBRARIES} PARENT_SCOPE) set(BDB_LIBRARY_DIRS "" PARENT_SCOPE) else() die("Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev or the equivalent") diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index 62b2eabe..70b4c876 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -54,10 +54,10 @@ target_link_libraries(blockchain_db ${Boost_DATE_TIME_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} + ${LMDB_LIBRARY} + ${BDB_LIBRARY} LINK_PRIVATE ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} - ${LMDB_LIBRARY} - ${BDB_LIBRARY} ${EXTRA_LIBRARIES}) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 4b8065a0..6b510f51 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -69,6 +69,7 @@ struct bdb_cur operator Dbc*() { return m_cur; } operator Dbc**() { return &m_cur; } + Dbc* operator->() { return m_cur; } void close() { @@ -138,8 +139,9 @@ struct Dbt_copy: public Dbt Dbt_copy(const cryptonote::blobdata &bd) : m_data(new char[bd.size()]) { memcpy(m_data.get(), bd.data(), bd.size()); - set_data(&m_data); + set_data(m_data.get()); set_size(bd.size()); + set_flags(DB_DBT_USERMEM); } private: std::unique_ptr m_data; @@ -177,52 +179,52 @@ void BlockchainBDB::add_block( const block& blk check_open(); Dbt_copy val_h(blk_hash); - if (m_block_heights->exists(m_write_txn, &val_h, 0) == 0) + if (m_block_heights->exists(*m_write_txn, &val_h, 0) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); - if (m_height > 0) + if (m_height > 1) { Dbt_copy parent_key(blk.prev_id); - Dbt_copy parent_h; - if (m_block_heights->get(m_write_txn, &parent_key, &parent_h, 0)) + Dbt_copy parent_h; + if (m_block_heights->get(*m_write_txn, &parent_key, &parent_h, 0)) { LOG_PRINT_L3("m_height: " << m_height); LOG_PRINT_L3("parent_key: " << blk.prev_id); throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); } - uint64_t parent_height = parent_h; + uint32_t parent_height = parent_h; if (parent_height != m_height - 1) throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); } - Dbt_copy key(m_height); + Dbt_copy key(m_height); Dbt_copy blob(block_to_blob(blk)); - auto res = m_blocks->put(m_write_txn, &key, &blob, DB_APPEND); + auto res = m_blocks->put(*m_write_txn, &key, &blob, 0); if (res) - throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: "))); + throw0(DB_ERROR("Failed to add block blob to db transaction.")); Dbt_copy sz(block_size); - if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, DB_APPEND)) - throw0(DB_ERROR("Failed to add block size to db transaction")); + if (m_block_sizes->put(*m_write_txn, &key, &sz, 0)) + throw0(DB_ERROR("Failed to add block size to db transaction.")); Dbt_copy ts(blk.timestamp); - if (m_block_timestamps->put(m_write_txn, &key, &ts, DB_APPEND)) - throw0(DB_ERROR("Failed to add block timestamp to db transaction")); + if (m_block_timestamps->put(*m_write_txn, &key, &ts, 0)) + throw0(DB_ERROR("Failed to add block timestamp to db transaction.")); Dbt_copy diff(cumulative_difficulty); - if (m_block_diffs->put(m_write_txn, &key, &diff, DB_APPEND)) - throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction")); + if (m_block_diffs->put(*m_write_txn, &key, &diff, 0)) + throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction.")); Dbt_copy coinsgen(coins_generated); - if (m_block_coins->put(m_write_txn, &key, &coinsgen, DB_APPEND)) - throw0(DB_ERROR("Failed to add block total generated coins to db transaction")); + if (m_block_coins->put(*m_write_txn, &key, &coinsgen, 0)) + throw0(DB_ERROR("Failed to add block total generated coins to db transaction.")); - if (m_block_heights->put(m_write_txn, &val_h, &key, DB_APPEND)) - throw0(DB_ERROR("Failed to add block height by hash to db transaction")); + if (m_block_heights->put(*m_write_txn, &val_h, &key, 0)) + throw0(DB_ERROR("Failed to add block height by hash to db transaction.")); - if (m_block_hashes->put(m_write_txn, &key, &val_h, DB_APPEND)) - throw0(DB_ERROR("Failed to add block hash to db transaction")); + if (m_block_hashes->put(*m_write_txn, &key, &val_h, 0)) + throw0(DB_ERROR("Failed to add block hash to db transaction.")); } void BlockchainBDB::remove_block() @@ -230,33 +232,33 @@ void BlockchainBDB::remove_block() LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - if (m_height == 0) + if (m_height <= 1) throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); - Dbt_copy k(m_height - 1); - Dbt_copy h; - if (m_block_hashes->get(m_write_txn, &k, &h, 0)) + Dbt_copy k(m_height - 1); + Dbt_copy h; + if (m_block_hashes->get(*m_write_txn, &k, &h, 0)) throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); - if (m_blocks->del(m_write_txn, &k, 0)) + if (m_blocks->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block to db transaction")); - if (m_block_sizes->del(m_write_txn, &k, 0)) + if (m_block_sizes->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block size to db transaction")); - if (m_block_diffs->del(m_write_txn, &k, 0)) + if (m_block_diffs->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); - if (m_block_coins->del(m_write_txn, &k, 0)) + if (m_block_coins->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); - if (m_block_timestamps->del(m_write_txn, &k, 0)) + if (m_block_timestamps->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); - if (m_block_heights->del(m_write_txn, &h, 0)) + if (m_block_heights->del(*m_write_txn, &h, 0)) throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); - if (m_block_hashes->del(m_write_txn, &k, 0)) + if (m_block_hashes->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); } @@ -267,19 +269,19 @@ void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const tra Dbt_copy val_h(tx_hash); - if (m_txs->exists(m_write_txn, &val_h, 0) == 0) + if (m_txs->exists(*m_write_txn, &val_h, 0) == 0) throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); Dbt_copy blob(tx_to_blob(tx)); - if (m_txs->put(m_write_txn, &val_h, &blob, DB_APPEND)) + if (m_txs->put(*m_write_txn, &val_h, &blob, 0)) throw0(DB_ERROR("Failed to add tx blob to db transaction")); - Dbt_copy height(m_height); - if (m_tx_heights->put(m_write_txn, &val_h, &height, DB_APPEND)) + Dbt_copy height(m_height); + if (m_tx_heights->put(*m_write_txn, &val_h, &height, 0)) throw0(DB_ERROR("Failed to add tx block height to db transaction")); Dbt_copy unlock_time(tx.unlock_time); - if (m_tx_unlocks->put(m_write_txn, &val_h, &unlock_time, DB_APPEND)) + if (m_tx_unlocks->put(*m_write_txn, &val_h, &unlock_time, 0)) throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); } @@ -289,19 +291,19 @@ void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const t check_open(); Dbt_copy val_h(tx_hash); - if (m_txs->exists(m_write_txn, &val_h, 0)) + if (m_txs->exists(*m_write_txn, &val_h, 0)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); - if (m_txs->del(m_write_txn, &val_h, 0)) + if (m_txs->del(*m_write_txn, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx to db transaction")); - if (m_tx_unlocks->del(m_write_txn, &val_h, 0)) + if (m_tx_unlocks->del(*m_write_txn, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); - if (m_tx_heights->del(m_write_txn, &val_h, 0)) + if (m_tx_heights->del(*m_write_txn, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); remove_tx_outputs(tx_hash, tx); - if (m_tx_outputs->del(m_write_txn, &val_h, 0)) + if (m_tx_outputs->del(*m_write_txn, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); } @@ -311,26 +313,26 @@ void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_out LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - Dbt_copy k(m_num_outputs); + Dbt_copy k(m_num_outputs); Dbt_copy v(tx_hash); - if (m_output_txs->put(m_write_txn, &k, &v, DB_APPEND)) + if (m_output_txs->put(*m_write_txn, &k, &v, 0)) throw0(DB_ERROR("Failed to add output tx hash to db transaction")); - if (m_tx_outputs->put(m_write_txn, &v, &k, DB_APPEND)) + if (m_tx_outputs->put(*m_write_txn, &v, &k, 0)) throw0(DB_ERROR("Failed to add tx output index to db transaction")); Dbt_copy val_local_index(local_index); - if (m_output_indices->put(m_write_txn, &k, &val_local_index, DB_APPEND)) + if (m_output_indices->put(*m_write_txn, &k, &val_local_index, 0)) throw0(DB_ERROR("Failed to add tx output index to db transaction")); Dbt_copy val_amount(tx_output.amount); - if (auto result = m_output_amounts->put(m_write_txn, &val_amount, &k, DB_APPEND)) - throw0(DB_ERROR(std::string("Failed to add output amount to db transaction"))); + if (m_output_amounts->put(*m_write_txn, &val_amount, &k, 0)) + throw0(DB_ERROR("Failed to add output amount to db transaction.")); if (tx_output.target.type() == typeid(txout_to_key)) { Dbt_copy val_pubkey(boost::get(tx_output.target).key); - if (m_output_keys->put(m_write_txn, &k, &val_pubkey, DB_APPEND)) + if (m_output_keys->put(*m_write_txn, &k, &val_pubkey, 0)) throw0(DB_ERROR("Failed to add output pubkey to db transaction")); } @@ -341,12 +343,12 @@ void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transac { LOG_PRINT_L3("BlockchainBDB::" << __func__); - bdb_cur cur(m_write_txn, m_tx_outputs); + bdb_cur cur(*m_write_txn, m_tx_outputs); Dbt_copy k(tx_hash); - Dbt_copy v; + Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) { throw0(DB_ERROR("Attempting to remove a tx's outputs, but none found.")); @@ -358,7 +360,7 @@ void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transac else { db_recno_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + cur->count(&num_elems, 0); for (uint64_t i = 0; i < num_elems; ++i) { @@ -366,7 +368,7 @@ void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transac remove_output(v, tx_output.amount); if (i < num_elems - 1) { - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } } } @@ -386,9 +388,9 @@ void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amou LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - Dbt_copy k(out_index); + Dbt_copy k(out_index); - auto result = m_output_indices->del(m_write_txn, &k, 0); + auto result = m_output_indices->del(*m_write_txn, &k, 0); if (result == DB_NOTFOUND) { LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); @@ -398,7 +400,7 @@ void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amou throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); } - result = m_output_txs->del(m_write_txn, &k, 0); + result = m_output_txs->del(*m_write_txn, &k, 0); // if (result != 0 && result != DB_NOTFOUND) // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); if (result == DB_NOTFOUND) @@ -410,7 +412,7 @@ void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amou throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); } - result = m_output_keys->del(m_write_txn, &k, 0); + result = m_output_keys->del(*m_write_txn, &k, 0); if (result == DB_NOTFOUND) { LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); @@ -428,19 +430,19 @@ void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_cur cur(m_write_txn, m_output_amounts); + bdb_cur cur(*m_write_txn, m_output_amounts); Dbt_copy k(amount); - Dbt_copy v; + Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) throw0(DB_ERROR("DB error attempting to get an output")); db_recno_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + cur->count(&num_elems, 0); uint64_t amount_output_index = 0; uint64_t goi = 0; @@ -454,13 +456,13 @@ void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint found_index = true; break; } - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } if (found_index) { // found the amount output index // now delete it - result = (Dbc*)cur->del(0); + result = cur->del(0); if (result) throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast(amount_output_index)).c_str())); } @@ -478,12 +480,12 @@ void BlockchainBDB::add_spent_key(const crypto::key_image& k_image) check_open(); Dbt_copy val_key(k_image); - if (m_spent_keys->exists(m_write_txn, &val_key, 0) == 0) + if (m_spent_keys->exists(*m_write_txn, &val_key, 0) == 0) throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); Dbt_copy val('\0'); - if (auto result = m_spent_keys->put(m_write_txn, &val_key, &val, DB_APPEND)) - throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: "))); + if (m_spent_keys->put(*m_write_txn, &val_key, &val, 0)) + throw1(DB_ERROR("Error adding spent key image to db transaction.")); } void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) @@ -492,7 +494,7 @@ void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) check_open(); Dbt_copy k(k_image); - auto result = m_spent_keys->del(m_write_txn, &k, 0); + auto result = m_spent_keys->del(*m_write_txn, &k, 0); if (result != 0 && result != DB_NOTFOUND) throw1(DB_ERROR("Error adding removal of key image to db transaction")); } @@ -532,23 +534,23 @@ uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const ui bdb_cur cur(txn, m_output_amounts); Dbt_copy k(amount); - Dbt_copy v; + Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); - if (result == MDB_NOTFOUND) + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) throw0(DB_ERROR("DB error attempting to get an output")); db_recno_t num_elems; - (Dbc*)cur->count(&num_elems, 0); + cur->count(&num_elems, 0); if (num_elems <= index) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); for (uint64_t i = 0; i < index; ++i) { - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } uint64_t glob_index = v; @@ -587,7 +589,7 @@ BlockchainBDB::BlockchainBDB(bool batch_transactions) m_batch_transactions = batch_transactions; m_write_txn = nullptr; - m_height = 0; + m_height = 1; } void BlockchainBDB::open(const std::string& filename) @@ -629,25 +631,25 @@ void BlockchainBDB::open(const std::string& filename) m_env->txn_begin(NULL, txn, 0); // create Dbs in the environment - m_blocks = new Db(m_env); - m_block_heights = new Db(m_env); - m_block_hashes = new Db(m_env); - m_block_timestamps = new Db(m_env); - m_block_sizes = new Db(m_env); - m_block_diffs = new Db(m_env); - m_block_coins = new Db(m_env); + m_blocks = new Db(m_env, 0); + m_block_heights = new Db(m_env, 0); + m_block_hashes = new Db(m_env, 0); + m_block_timestamps = new Db(m_env, 0); + m_block_sizes = new Db(m_env, 0); + m_block_diffs = new Db(m_env, 0); + m_block_coins = new Db(m_env, 0); - m_txs = new Db(m_env); - m_tx_unlocks = new Db(m_env); - m_tx_heights = new Db(m_env); - m_tx_outputs = new Db(m_env); + m_txs = new Db(m_env, 0); + m_tx_unlocks = new Db(m_env, 0); + m_tx_heights = new Db(m_env, 0); + m_tx_outputs = new Db(m_env, 0); - m_output_txs = new Db(m_env); - m_output_indices = new Db(m_env); - m_output_amounts = new Db(m_env); - m_output_keys = new Db(m_env); + m_output_txs = new Db(m_env, 0); + m_output_indices = new Db(m_env, 0); + m_output_amounts = new Db(m_env, 0); + m_output_keys = new Db(m_env, 0); - m_spent_keys = new Db(m_env); + m_spent_keys = new Db(m_env, 0); // Tell DB about Dbs that need duplicate support // Note: no need to tell about sorting, @@ -656,24 +658,16 @@ void BlockchainBDB::open(const std::string& filename) m_output_amounts->set_flags(DB_DUP); // Tell DB about fixed-size values. - m_block_heights->set_re_len(sizeof(uint64_t)); m_block_hashes->set_re_len(sizeof(crypto::hash)); m_block_timestamps->set_re_len(sizeof(uint64_t)); m_block_sizes->set_re_len(sizeof(size_t)); // should really store block size as uint64_t... m_block_diffs->set_re_len(sizeof(difficulty_type)); m_block_coins->set_re_len(sizeof(uint64_t)); - m_tx_unlocks->set_re_len(sizeof(uint64_t)); - m_tx_heights->set_re_len(sizeof(uint64_t)); - m_tx_outputs->set_re_len(sizeof(uint64_t)); - - m_output_txs->set_re_len(sizeof(uint64_t)); + m_output_txs->set_re_len(sizeof(crypto::hash)); m_output_indices->set_re_len(sizeof(uint64_t)); - m_output_amounts->set_re_len(sizeof(uint64_t)); m_output_keys->set_re_len(sizeof(crypto::public_key)); - m_spent_keys->set_re_len(sizeof(crypto::key_image)); - //TODO: Find out if we need to do Db::set_flags(DB_RENUMBER) // for the RECNO databases. We shouldn't as we're only // inserting/removing from the end, but we'll see. @@ -685,15 +679,15 @@ void BlockchainBDB::open(const std::string& filename) m_blocks->open(txn, BDB_BLOCKS, NULL, DB_RECNO, DB_CREATE, 0); m_block_timestamps->open(txn, BDB_BLOCK_TIMESTAMPS, NULL, DB_RECNO, DB_CREATE, 0); - m_block_heights->open(txn, BDB_BLOCK_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_heights->open(txn, BDB_BLOCK_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0); m_block_hashes->open(txn, BDB_BLOCK_HASHES, NULL, DB_RECNO, DB_CREATE, 0); m_block_sizes->open(txn, BDB_BLOCK_SIZES, NULL, DB_RECNO, DB_CREATE, 0); m_block_diffs->open(txn, BDB_BLOCK_DIFFS, NULL, DB_RECNO, DB_CREATE, 0); m_block_coins->open(txn, BDB_BLOCK_COINS, NULL, DB_RECNO, DB_CREATE, 0); - m_txs->open(txn, BDB_TXS, NULL, DB_RECNO, DB_CREATE, 0); - m_tx_unlocks->open(txn, BDB_TX_UNLOCKS, NULL, DB_RECNO, DB_CREATE, 0); - m_tx_heights->open(txn, BDB_TX_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0); + m_txs->open(txn, BDB_TXS, NULL, DB_HASH, DB_CREATE, 0); + m_tx_unlocks->open(txn, BDB_TX_UNLOCKS, NULL, DB_HASH, DB_CREATE, 0); + m_tx_heights->open(txn, BDB_TX_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0); m_tx_outputs->open(txn, BDB_TX_OUTPUTS, NULL, DB_HASH, DB_CREATE, 0); m_output_txs->open(txn, BDB_OUTPUT_TXS, NULL, DB_RECNO, DB_CREATE, 0); @@ -701,7 +695,7 @@ void BlockchainBDB::open(const std::string& filename) m_output_amounts->open(txn, BDB_OUTPUT_AMOUNTS, NULL, DB_HASH, DB_CREATE, 0); m_output_keys->open(txn, BDB_OUTPUT_KEYS, NULL, DB_RECNO, DB_CREATE, 0); - m_spent_keys->open(txn, BDB_SPENT_KEYS, NULL, DB_RECNO, DB_CREATE, 0); + m_spent_keys->open(txn, BDB_SPENT_KEYS, NULL, DB_HASH, DB_CREATE, 0); DB_BTREE_STAT* stats; @@ -709,15 +703,15 @@ void BlockchainBDB::open(const std::string& filename) // to be returned. The flag should be set to 0 instead if this proves // to be the case. m_blocks->stat(txn, &stats, DB_FAST_STAT); - m_height = stats->bt_nkeys; + m_height = stats->bt_nkeys + 1; delete stats; // see above comment about DB_FAST_STAT m_output_indices->stat(txn, &stats, DB_FAST_STAT); - m_num_outputs = stats->bt_nkeys; + m_num_outputs = stats->bt_nkeys + 1; delete stats; - txn->commit(); + txn.commit(); } catch (const std::exception& e) { @@ -736,10 +730,11 @@ void BlockchainBDB::create(const std::string& filename) void BlockchainBDB::close() { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainBDB::" << __func__); this->sync(); // FIXME: not yet thread safe!!! Use with care. + m_open = false; m_env->close(DB_FORCESYNC); } @@ -750,29 +745,29 @@ void BlockchainBDB::sync() try { - m_blocks->sync(); - m_block_heights->sync(); - m_block_hashes->sync(); - m_block_timestamps->sync(); - m_block_sizes->sync(); - m_block_diffs->sync(); - m_block_coins->sync(); + m_blocks->sync(0); + m_block_heights->sync(0); + m_block_hashes->sync(0); + m_block_timestamps->sync(0); + m_block_sizes->sync(0); + m_block_diffs->sync(0); + m_block_coins->sync(0); - m_txs->sync(); - m_tx_unlocks->sync(); - m_tx_heights->sync(); - m_tx_outputs->sync(); + m_txs->sync(0); + m_tx_unlocks->sync(0); + m_tx_heights->sync(0); + m_tx_outputs->sync(0); - m_output_txs->sync(); - m_output_indices->sync(); - m_output_amounts->sync(); - m_output_keys->sync(); + m_output_txs->sync(0); + m_output_indices->sync(0); + m_output_amounts->sync(0); + m_output_keys->sync(0); - m_spent_keys->sync(); + m_spent_keys->sync(0); } catch (const std::exception& e) { - throw0(DB_ERROR(std::string("Failed to sync database: ").append(e.what()))); + throw0(DB_ERROR(std::string("Failed to sync database: ").append(e.what()).c_str())); } } @@ -790,56 +785,69 @@ std::vector BlockchainBDB::get_filenames() const std::vector filenames; char *fname, *dbname; + const char **pfname, **pdbname; - m_blocks->get_dbname(&fname, &dbname); + pfname = (const char **)&fname; + pdbname = (const char **)&dbname; + + m_blocks->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_heights->get_dbname(&fname, &dbname); + m_block_heights->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_hashes->get_dbname(&fname, &dbname); + m_block_hashes->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_timestamps->get_dbname(&fname, &dbname); + m_block_timestamps->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_sizes->get_dbname(&fname, &dbname); + m_block_sizes->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_diffs->get_dbname(&fname, &dbname); + m_block_diffs->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_coins->get_dbname(&fname, &dbname); + m_block_coins->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_txs->get_dbname(&fname, &dbname); + m_txs->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_tx_unlocks->get_dbname(&fname, &dbname); + m_tx_unlocks->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_tx_heights->get_dbname(&fname, &dbname); + m_tx_heights->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_tx_outputs->get_dbname(&fname, &dbname); + m_tx_outputs->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_output_txs->get_dbname(&fname, &dbname); + m_output_txs->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_output_indices->get_dbname(&fname, &dbname); + m_output_indices->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_output_amounts->get_dbname(&fname, &dbname); + m_output_amounts->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_output_keys->get_dbname(&fname, &dbname); + m_output_keys->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_spent_keys->get_dbname(&fname, &dbname); + m_spent_keys->get_dbname(pfname, pdbname); filenames.push_back(fname); - return filenames; + std::vector full_paths; + + for (auto& filename : filenames) + { + boost::filesystem::path p(m_folder); + p /= filename; + full_paths.push_back(p.string()); + } + + return full_paths; } std::string BlockchainBDB::get_db_name() const @@ -910,13 +918,14 @@ uint64_t BlockchainBDB::get_block_height(const crypto::hash& h) const Dbt_copy result; auto get_result = m_block_heights->get(txn, &key, &result, 0); - if (get_result == MDB_NOTFOUND) + if (get_result == DB_NOTFOUND) throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); txn.commit(); - return result; + + return (uint64_t)result - 1; } block_header BlockchainBDB::get_block_header(const crypto::hash& h) const @@ -937,10 +946,10 @@ block BlockchainBDB::get_block_from_height(const uint64_t& height) const if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_safe result; - auto get_result = mdb_get(txn, m_blocks, &key, &result); - if (get_result == MDB_NOTFOUND) + auto get_result = m_blocks->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast(height)).append(" failed -- block not in db").c_str())); } @@ -968,7 +977,7 @@ uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_copy result; auto get_result = m_block_timestamps->get(txn, &key, &result, 0); if (get_result == DB_NOTFOUND) @@ -988,7 +997,7 @@ uint64_t BlockchainBDB::get_top_block_timestamp() const check_open(); // if no blocks, return 0 - if (m_height == 0) + if (m_height <= 0) { return 0; } @@ -1005,7 +1014,7 @@ size_t BlockchainBDB::get_block_size(const uint64_t& height) const if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_copy result; auto get_result = m_block_sizes->get(txn, &key, &result, 0); if (get_result == DB_NOTFOUND) @@ -1028,7 +1037,7 @@ difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& h if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_copy result; auto get_result = m_block_diffs->get(txn, &key, &result, 0); if (get_result == DB_NOTFOUND) @@ -1068,7 +1077,7 @@ uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_copy result; auto get_result = m_block_coins->get(txn, &key, &result, 0); if (get_result == DB_NOTFOUND) @@ -1091,7 +1100,7 @@ crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) c if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_copy result; auto get_result = m_block_hashes->get(txn, &key, &result, 0); if (get_result == DB_NOTFOUND) @@ -1099,8 +1108,7 @@ crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) c throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); } else if (get_result) - throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). - append(mdb_strerror(get_result)).c_str())); + throw0(DB_ERROR("Error attempting to retrieve a block hash from the db.")); txn.commit(); return result; @@ -1165,7 +1173,7 @@ uint64_t BlockchainBDB::height() const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - return m_height; + return m_height - 1; } bool BlockchainBDB::tx_exists(const crypto::hash& h) const @@ -1302,7 +1310,7 @@ uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const txn.commit(); - return result; + return (uint64_t)result - 1; } //FIXME: make sure the random method used here is appropriate @@ -1331,7 +1339,7 @@ uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const Dbt_copy k(amount); Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) { return 0; @@ -1339,8 +1347,8 @@ uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const else if (result) throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); - size_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); txn.commit(); @@ -1358,7 +1366,7 @@ crypto::public_key BlockchainBDB::get_output_key(const uint64_t& amount, const u if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy k(glob_index); + Dbt_copy k(glob_index); Dbt_copy v; auto get_result = m_output_keys->get(txn, &k, &v, 0); if (get_result == DB_NOTFOUND) @@ -1394,7 +1402,7 @@ tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy k(index); + Dbt_copy k(index); Dbt_copy v; auto get_result = m_output_txs->get(txn, &k, &v, 0); @@ -1405,7 +1413,7 @@ tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& crypto::hash tx_hash = v; - Dbt_copy result; + Dbt_copy result; get_result = m_output_indices->get(txn, &k, &result, 0); if (get_result == DB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); @@ -1431,21 +1439,21 @@ tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, cons Dbt_copy k(amount); Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) throw0(DB_ERROR("DB error attempting to get an output")); - size_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); if (num_elems <= index) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); for (uint64_t i = 0; i < index; ++i) { - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } uint64_t glob_index = v; @@ -1471,19 +1479,19 @@ std::vector BlockchainBDB::get_tx_output_indices(const crypto::hash& h Dbt_copy k(h); Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); else if (result) throw0(DB_ERROR("DB error attempting to get an output")); - size_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); for (uint64_t i = 0; i < num_elems; ++i) { index_vec.push_back(v); - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } cur.close(); @@ -1522,14 +1530,14 @@ std::vector BlockchainBDB::get_tx_amount_output_indices(const crypto:: Dbt_copy k(amount); Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) throw0(DB_ERROR("DB error attempting to get an output")); - size_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); uint64_t amount_output_index = 0; uint64_t output_index = 0; @@ -1543,7 +1551,7 @@ std::vector BlockchainBDB::get_tx_amount_output_indices(const crypto:: found_index = true; break; } - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } if (found_index) { @@ -1643,16 +1651,17 @@ uint64_t BlockchainBDB::add_block( const block& blk txn.commit(); TIME_MEASURE_FINISH(time1); time_commit1 += time1; - } } - catch (...) + catch (const std::exception& e) { m_num_outputs = num_outputs; m_write_txn = NULL; - throw; + throw0(DB_ERROR(std::string("Error adding block: ").append(e.what()).c_str())); } - return ++m_height; + m_height++; + + return m_height - 1; } void BlockchainBDB::pop_block(block& blk, std::vector& txs) diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index b28afbf2..f804db51 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -49,7 +49,7 @@ struct bdb_txn_safe message = "Failed to commit a transaction to the db"; } - if (txn->commit()) + if (m_txn->commit(0)) { m_txn = NULL; LOG_PRINT_L0(message); @@ -103,6 +103,8 @@ public: virtual std::vector get_filenames() const; + virtual std::string get_db_name() const; + virtual bool lock(); virtual void unlock(); @@ -279,7 +281,7 @@ private: uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; - txn_safe m_write_txn; // may point to either a short-lived txn or a batch txn + bdb_txn_safe *m_write_txn; bool m_batch_transactions; // support for batch transactions }; diff --git a/tests/unit_tests/BlockchainDB.cpp b/tests/unit_tests/BlockchainDB.cpp index bbe8582f..b6e311da 100644 --- a/tests/unit_tests/BlockchainDB.cpp +++ b/tests/unit_tests/BlockchainDB.cpp @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include "gtest/gtest.h" @@ -88,6 +90,7 @@ bool compare_blocks(const block& a, const block& b) return hash_a == hash_b; } +/* void print_block(const block& blk, const std::string& prefix = "") { std::cerr << prefix << ": " << std::endl @@ -106,6 +109,7 @@ bool compare_txs(const transaction& a, const transaction& b) return ab == bb; } +*/ // convert hex string to string that has values based on that hex // thankfully should automatically ignore null-terminator. @@ -173,20 +177,29 @@ protected: } ~BlockchainDBTest() { - auto files = m_db->get_filenames(); delete m_db; - remove_files(files); + remove_files(); } BlockchainDB* m_db; std::string m_prefix; std::vector m_blocks; std::vector > m_txs; + std::vector m_filenames; - void remove_files(const std::vector& files) + void get_filenames() + { + m_filenames = m_db->get_filenames(); + for (auto& f : m_filenames) + { + std::cerr << "File created by test: " << f << std::endl; + } + } + + void remove_files() { // remove each file the db created, making sure it starts with fname. - for (auto& f : files) + for (auto& f : m_filenames) { if (boost::starts_with(f, m_prefix)) { @@ -199,7 +212,7 @@ protected: } // remove directory if it still exists - boost::filesystem::remove(m_prefix); + boost::filesystem::remove_all(m_prefix); } void set_prefix(const std::string& prefix) @@ -222,6 +235,7 @@ TYPED_TEST(BlockchainDBTest, OpenAndClose) // make sure open does not throw ASSERT_NO_THROW(this->m_db->open(fname)); + this->get_filenames(); // make sure open when already open DOES throw ASSERT_THROW(this->m_db->open(fname), DB_OPEN_FAILURE); @@ -232,9 +246,11 @@ TYPED_TEST(BlockchainDBTest, OpenAndClose) TYPED_TEST(BlockchainDBTest, AddBlock) { std::string fname(tmpnam(NULL)); + this->set_prefix(fname); // make sure open does not throw ASSERT_NO_THROW(this->m_db->open(fname)); + this->get_filenames(); // adding a block with no parent in the blockchain should throw. // note: this shouldn't be possible, but is a good (and cheap) failsafe. @@ -273,9 +289,11 @@ TYPED_TEST(BlockchainDBTest, AddBlock) TYPED_TEST(BlockchainDBTest, RetrieveBlockData) { std::string fname(tmpnam(NULL)); + this->set_prefix(fname); // make sure open does not throw ASSERT_NO_THROW(this->m_db->open(fname)); + this->get_filenames(); ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0])); From e146027acd3887f0cad9dea547cc74c608d5ec0b Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Tue, 17 Mar 2015 17:18:45 -0400 Subject: [PATCH 6/8] BlockchainBDB passes unit tests --- src/blockchain_db/berkeleydb/db_bdb.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 6b510f51..17208a78 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -122,6 +122,7 @@ struct Dbt_copy: public Dbt { set_data(&t_copy); set_size(sizeof(T)); + set_ulen(sizeof(T)); set_flags(DB_DBT_USERMEM); } @@ -141,6 +142,7 @@ struct Dbt_copy: public Dbt memcpy(m_data.get(), bd.data(), bd.size()); set_data(m_data.get()); set_size(bd.size()); + set_ulen(bd.size()); set_flags(DB_DBT_USERMEM); } private: @@ -152,6 +154,7 @@ struct Dbt_safe : public Dbt Dbt_safe() { set_data(NULL); + set_flags(DB_DBT_MALLOC); } ~Dbt_safe() { @@ -1656,7 +1659,7 @@ uint64_t BlockchainBDB::add_block( const block& blk { m_num_outputs = num_outputs; m_write_txn = NULL; - throw0(DB_ERROR(std::string("Error adding block: ").append(e.what()).c_str())); + throw; } m_height++; From ead7fad5526dad53c7885895bd5e52c890a49191 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Tue, 17 Mar 2015 22:12:09 -0400 Subject: [PATCH 7/8] BerkeleyDB implementation of BlockchainDB seems to be working! --- src/blockchain_db/berkeleydb/db_bdb.cpp | 34 ++++++++++++------------- src/blockchain_db/berkeleydb/db_bdb.h | 6 +++-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 17208a78..786c13b0 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -185,7 +185,7 @@ void BlockchainBDB::add_block( const block& blk if (m_block_heights->exists(*m_write_txn, &val_h, 0) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); - if (m_height > 1) + if (m_height > 0) { Dbt_copy parent_key(blk.prev_id); Dbt_copy parent_h; @@ -196,11 +196,11 @@ void BlockchainBDB::add_block( const block& blk throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); } uint32_t parent_height = parent_h; - if (parent_height != m_height - 1) + if (parent_height != m_height) throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); } - Dbt_copy key(m_height); + Dbt_copy key(m_height + 1); Dbt_copy blob(block_to_blob(blk)); auto res = m_blocks->put(*m_write_txn, &key, &blob, 0); @@ -235,10 +235,10 @@ void BlockchainBDB::remove_block() LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - if (m_height <= 1) + if (m_height == 0) throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); - Dbt_copy k(m_height - 1); + Dbt_copy k(m_height); Dbt_copy h; if (m_block_hashes->get(*m_write_txn, &k, &h, 0)) throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); @@ -279,7 +279,7 @@ void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const tra if (m_txs->put(*m_write_txn, &val_h, &blob, 0)) throw0(DB_ERROR("Failed to add tx blob to db transaction")); - Dbt_copy height(m_height); + Dbt_copy height(m_height + 1); if (m_tx_heights->put(*m_write_txn, &val_h, &height, 0)) throw0(DB_ERROR("Failed to add tx block height to db transaction")); @@ -316,7 +316,7 @@ void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_out LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - Dbt_copy k(m_num_outputs); + Dbt_copy k(m_num_outputs + 1); Dbt_copy v(tx_hash); if (m_output_txs->put(*m_write_txn, &k, &v, 0)) @@ -592,10 +592,10 @@ BlockchainBDB::BlockchainBDB(bool batch_transactions) m_batch_transactions = batch_transactions; m_write_txn = nullptr; - m_height = 1; + m_height = 0; } -void BlockchainBDB::open(const std::string& filename) +void BlockchainBDB::open(const std::string& filename, const int db_flags) { LOG_PRINT_L3("BlockchainBDB::" << __func__); @@ -706,12 +706,12 @@ void BlockchainBDB::open(const std::string& filename) // to be returned. The flag should be set to 0 instead if this proves // to be the case. m_blocks->stat(txn, &stats, DB_FAST_STAT); - m_height = stats->bt_nkeys + 1; + m_height = stats->bt_nkeys; delete stats; // see above comment about DB_FAST_STAT m_output_indices->stat(txn, &stats, DB_FAST_STAT); - m_num_outputs = stats->bt_nkeys + 1; + m_num_outputs = stats->bt_nkeys; delete stats; txn.commit(); @@ -1000,7 +1000,7 @@ uint64_t BlockchainBDB::get_top_block_timestamp() const check_open(); // if no blocks, return 0 - if (m_height <= 0) + if (m_height == 0) { return 0; } @@ -1149,7 +1149,7 @@ crypto::hash BlockchainBDB::top_block_hash() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - if (m_height != 0) + if (m_height > 0) { return get_block_hash_from_height(m_height - 1); } @@ -1162,7 +1162,7 @@ block BlockchainBDB::get_top_block() const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - if (m_height != 0) + if (m_height > 0) { return get_block_from_height(m_height - 1); } @@ -1176,7 +1176,7 @@ uint64_t BlockchainBDB::height() const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - return m_height - 1; + return m_height; } bool BlockchainBDB::tx_exists(const crypto::hash& h) const @@ -1662,9 +1662,7 @@ uint64_t BlockchainBDB::add_block( const block& blk throw; } - m_height++; - - return m_height - 1; + return ++m_height; } void BlockchainBDB::pop_block(block& blk, std::vector& txs) diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index f804db51..b68ed287 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -39,7 +39,9 @@ struct bdb_txn_safe ~bdb_txn_safe() { LOG_PRINT_L3("bdb_txn_safe: destructor"); - abort(); + + if (m_txn != NULL) + abort(); } void commit(std::string message = "") @@ -91,7 +93,7 @@ public: BlockchainBDB(bool batch_transactions=false); ~BlockchainBDB(); - virtual void open(const std::string& filename); + virtual void open(const std::string& filename, const int db_flags); virtual void create(const std::string& filename); From 7b14d4a17f739c383322312f1a597f264c074c6e Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Wed, 25 Mar 2015 11:41:30 -0400 Subject: [PATCH 8/8] Steps toward multiple dbs available -- working There will need to be some more refactoring for these changes to be considered complete/correct, but for now it's working. new daemon cli argument "--db-type", works for LMDB and BerkeleyDB. A good deal of refactoring is also present in this commit, namely Blockchain no longer instantiates BlockchainDB, but rather is passed a pointer to an already-instantiated BlockchainDB on init(). --- src/blockchain_converter/fake_core.h | 26 ++++++++++++++-- src/blockchain_db/berkeleydb/db_bdb.cpp | 7 ----- src/blockchain_db/berkeleydb/db_bdb.h | 3 -- src/blockchain_db/blockchain_db.cpp | 5 +++ src/blockchain_db/blockchain_db.h | 6 ++-- src/blockchain_db/db_types.h | 40 ++++++++++++++++++++++++ src/blockchain_db/lmdb/db_lmdb.cpp | 7 ----- src/blockchain_db/lmdb/db_lmdb.h | 4 +-- src/cryptonote_core/blockchain.cpp | 41 ++++++------------------- src/cryptonote_core/blockchain.h | 4 +-- src/cryptonote_core/cryptonote_core.cpp | 41 +++++++++++++++++++++++++ src/daemon/command_line_args.h | 5 +++ src/daemon/main.cpp | 15 +++++++++ 13 files changed, 146 insertions(+), 58 deletions(-) create mode 100644 src/blockchain_db/db_types.h diff --git a/src/blockchain_converter/fake_core.h b/src/blockchain_converter/fake_core.h index 175cb466..f82b05d0 100644 --- a/src/blockchain_converter/fake_core.h +++ b/src/blockchain_converter/fake_core.h @@ -29,8 +29,10 @@ #pragma once #include -#include "cryptonote_core/blockchain.h" // BlockchainDB and LMDB +#include "cryptonote_core/blockchain.h" // BlockchainDB #include "cryptonote_core/blockchain_storage.h" // in-memory DB +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" using namespace cryptonote; @@ -53,7 +55,27 @@ struct fake_core_lmdb #endif { m_pool.init(path.string()); - m_storage.init(path.string(), use_testnet, mdb_flags); + + BlockchainDB* db = new BlockchainLMDB(); + + boost::filesystem::path folder(path); + + folder /= db->get_db_name(); + + LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); + + const std::string filename = folder.string(); + try + { + db->open(filename, mdb_flags); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + throw; + } + + m_storage.init(db, use_testnet); if (do_batch) m_storage.get_db().set_batch_transactions(do_batch); support_batch = true; diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 786c13b0..4b254500 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -724,13 +724,6 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags) m_open = true; } -// unused for now, create will happen on open if doesn't exist -void BlockchainBDB::create(const std::string& filename) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed."); -} - void BlockchainBDB::close() { LOG_PRINT_L3("BlockchainBDB::" << __func__); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index b68ed287..d4eb5434 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -95,8 +95,6 @@ public: virtual void open(const std::string& filename, const int db_flags); - virtual void create(const std::string& filename); - virtual void close(); virtual void sync(); @@ -279,7 +277,6 @@ private: Db* m_spent_keys; - bool m_open; uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 8b08d380..d648be44 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -129,6 +129,11 @@ void BlockchainDB::pop_block(block& blk, std::vector& txs) } } +bool BlockchainDB::is_open() +{ + return m_open; +} + void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) { transaction tx = get_tx(tx_hash); diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 7b6b55a4..04d9c538 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -62,6 +62,7 @@ * * General: * open() + * is_open() * close() * sync() * reset() @@ -328,8 +329,8 @@ public: // open the db at location , or create it if there isn't one. virtual void open(const std::string& filename, const int db_flags = 0) = 0; - // make sure implementation has a create function as well - virtual void create(const std::string& filename) = 0; + // returns true of the db is open/ready, else false + bool is_open(); // close and sync the db virtual void close() = 0; @@ -482,6 +483,7 @@ public: // returns true if key image is present in spent key images storage virtual bool has_key_image(const crypto::key_image& img) const = 0; + bool m_open; }; // class BlockchainDB diff --git a/src/blockchain_db/db_types.h b/src/blockchain_db/db_types.h new file mode 100644 index 00000000..b13007df --- /dev/null +++ b/src/blockchain_db/db_types.h @@ -0,0 +1,40 @@ +// Copyright (c) 2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +#pragma once + +namespace cryptonote +{ + + const std::unordered_set blockchain_db_types = + { "lmdb" + , "berkeley" + }; + +} // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 524a1c26..8e09dfab 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -733,13 +733,6 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // from here, init should be finished } -// unused for now, create will happen on open if doesn't exist -void BlockchainLMDB::create(const std::string& filename) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed."); -} - void BlockchainLMDB::close() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index c3c7ce43..8f1e07e0 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -24,6 +24,7 @@ // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once #include "blockchain_db/blockchain_db.h" #include "cryptonote_protocol/blobdatatype.h" // for type blobdata @@ -116,8 +117,6 @@ public: virtual void open(const std::string& filename, const int mdb_flags=0); - virtual void create(const std::string& filename); - virtual void close(); virtual void sync(); @@ -302,7 +301,6 @@ private: MDB_dbi m_spent_keys; - bool m_open; uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 57934b3f..0261a561 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -39,7 +39,6 @@ #include "tx_pool.h" #include "blockchain.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/lmdb/db_lmdb.h" #include "cryptonote_format_utils.h" #include "cryptonote_boost_serialization.h" #include "cryptonote_config.h" @@ -65,8 +64,7 @@ using epee::string_tools::pod_to_hex; DISABLE_VS_WARNINGS(4267) //------------------------------------------------------------------ -// TODO: initialize m_db with a concrete implementation of BlockchainDB -Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_testnet(false), m_enforce_dns_checkpoints(false) +Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -226,43 +224,24 @@ uint64_t Blockchain::get_current_blockchain_height() const //------------------------------------------------------------------ //FIXME: possibly move this into the constructor, to avoid accidentally // dereferencing a null BlockchainDB pointer -bool Blockchain::init(const std::string& config_folder, const bool testnet, const int db_flags) +bool Blockchain::init(BlockchainDB* db, const bool testnet) { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - // TODO: make this configurable - m_db = new BlockchainLMDB(); - - m_config_folder = config_folder; - m_testnet = testnet; - - boost::filesystem::path folder(m_config_folder); - - folder /= m_db->get_db_name(); - - LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); - - const std::string filename = folder.string(); - try + if (db == nullptr) { - m_db->open(filename, db_flags); + LOG_ERROR("Attempted to init Blockchain with null DB"); + return false; } - catch (const DB_OPEN_FAILURE& e) + if (!db->is_open()) { - LOG_PRINT_L0("No blockchain file found, attempting to create one."); - try - { - m_db->create(filename); - } - catch (const DB_CREATE_FAILURE& db_create_error) - { - LOG_PRINT_L0("Unable to create BlockchainDB! This is not good..."); - //TODO: make sure whatever calls this handles the return value properly - return false; - } + LOG_ERROR("Attempted to init Blockchain with unopened DB"); + return false; } + m_db = db; + // if the blockchain is new, add the genesis block // this feels kinda kludgy to do it this way, but can be looked at later. // TODO: add function to create and store genesis block, diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index dc98a56a..bc13901d 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -81,7 +81,7 @@ namespace cryptonote Blockchain(tx_memory_pool& tx_pool); - bool init(const std::string& config_folder, const bool testnet = false, const int db_flags = 0); + bool init(BlockchainDB* db, const bool testnet = false); bool deinit(); void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } @@ -180,11 +180,9 @@ namespace cryptonote outputs_container m_outputs; - std::string m_config_folder; checkpoints m_checkpoints; std::atomic m_is_in_checkpoint_zone; std::atomic m_is_blockchain_storing; - bool m_testnet; bool m_enforce_dns_checkpoints; bool switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain); diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index f9b2b19f..7864b55c 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -44,6 +44,9 @@ using namespace epee; #include #include "daemon/command_line_args.h" #include "cryptonote_core/checkpoints_create.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/berkeleydb/db_bdb.h" DISABLE_VS_WARNINGS(4355) @@ -194,7 +197,45 @@ namespace cryptonote r = m_mempool.init(m_config_folder); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); +#if BLOCKCHAIN_DB == DB_LMDB + std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type); + + BlockchainDB* db = nullptr; + if (db_type == "lmdb") + { + db = new BlockchainLMDB(); + } + else if (db_type == "berkeley") + { + db = new BlockchainBDB(); + } + else + { + LOG_ERROR("Attempted to use non-existant database type"); + return false; + } + + boost::filesystem::path folder(m_config_folder); + + folder /= db->get_db_name(); + + LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); + + const std::string filename = folder.string(); + try + { + db->open(filename); + } + catch (const DB_ERROR& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return false; + } + + r = m_blockchain_storage.init(db, m_testnet); +#else r = m_blockchain_storage.init(m_config_folder, m_testnet); +#endif CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage"); // load json & DNS checkpoints, and verify them diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index bcf59912..2bd91847 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -70,6 +70,11 @@ namespace daemon_args , "checkpoints from DNS server will be enforced" , false }; + const command_line::arg_descriptor arg_db_type = { + "db-type" + , "Specify database type" + , "lmdb" + }; } // namespace daemon_args diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 5d8baf49..3bad7037 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -42,6 +42,7 @@ #include "rpc/core_rpc_server.h" #include #include "daemon/command_line_args.h" +#include "blockchain_db/db_types.h" namespace po = boost::program_options; namespace bf = boost::filesystem; @@ -78,6 +79,7 @@ int main(int argc, char const * argv[]) command_line::add_arg(core_settings, daemon_args::arg_log_level); command_line::add_arg(core_settings, daemon_args::arg_testnet_on); command_line::add_arg(core_settings, daemon_args::arg_dns_checkpoints); + command_line::add_arg(core_settings, daemon_args::arg_db_type); daemonizer::init_options(hidden_options, visible_options); daemonize::t_executor::init_options(core_settings); @@ -128,6 +130,19 @@ int main(int argc, char const * argv[]) return 0; } + std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type); + + // verify that blockchaindb type is valid + if(cryptonote::blockchain_db_types.count(db_type) == 0) + { + std::cout << "Invalid database type (" << db_type << "), available types are:" << std::endl; + for (const auto& type : cryptonote::blockchain_db_types) + { + std::cout << "\t" << type << std::endl; + } + return 0; + } + bool testnet_mode = command_line::get_arg(vm, daemon_args::arg_testnet_on); auto data_dir_arg = testnet_mode ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;