Merge BerkeleyDB blockchain db implementation

This commit is contained in:
Thomas Winget 2015-03-29 09:03:30 -04:00
commit e7391a4113
No known key found for this signature in database
GPG key ID: 58131A160789E630
19 changed files with 2250 additions and 102 deletions

View file

@ -177,6 +177,9 @@ include_directories(external/rapidjson)
# Final setup for liblmdb
include_directories(${LMDB_INCLUDE})
# Final setup for Berkeley DB
include_directories(${BDB_INCLUDE})
if(MSVC)
add_definitions("/bigobj /MP /W3 /GS- /D_CRT_SECURE_NO_WARNINGS /wd4996 /wd4345 /D_WIN32_WINNT=0x0600 /DWIN32_LEAN_AND_MEAN /DGTEST_HAS_TR1_TUPLE=0 /FIinline_c.h /D__SSE4_1__")
# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Dinline=__inline")

View file

@ -0,0 +1,25 @@
# - Try to find Berkeley DB
# Once done this will define
#
# BERKELEY_DB_FOUND - system has Berkeley DB
# BERKELEY_DB_INCLUDE_DIR - the Berkeley DB include directory
# BERKELEY_DB_LIBRARIES - Link these to use Berkeley DB
# BERKELEY_DB_DEFINITIONS - Compiler switches required for using Berkeley DB
# Copyright (c) 2006, Alexander Dymo, <adymo@kdevelop.org>
#
# Redistribution and use is allowed according to the terms of the BSD license.
# For details see the accompanying COPYING-CMAKE-SCRIPTS file.
find_path(BERKELEY_DB_INCLUDE_DIR db_cxx.h
/usr/include/db4
/usr/local/include/db4
)
find_library(BERKELEY_DB_LIBRARIES NAMES db_cxx )
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Berkeley "Could not find Berkeley DB >= 4.1" BERKELEY_DB_INCLUDE_DIR BERKELEY_DB_LIBRARIES)
# show the BERKELEY_DB_INCLUDE_DIR and BERKELEY_DB_LIBRARIES variables only in the advanced view
mark_as_advanced(BERKELEY_DB_INCLUDE_DIR BERKELEY_DB_LIBRARIES )

View file

@ -33,3 +33,25 @@ add_subdirectory(liblmdb${ARCH_WIDTH})
set(LMDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/liblmdb${ARCH_WIDTH}" CACHE STRING "LMDB Include path")
set(LMDB_LIBRARY "lmdb" CACHE STRING "LMDB Library name")
find_package(BerkeleyDB)
if(NOT BERKELEY_DB_LIBRARIES OR STATIC)
add_subdirectory(libdb)
message(STATUS "BerkeleyDB not found, building from src tree")
set(BDB_STATIC true CACHE BOOL "BDB Static flag")
set(BDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/libdb" CACHE STRING "BDB include path")
set(BDB_LIBRARY "db" CACHE STRING "BDB library name")
else()
message(STATUS "Found BerkeleyDB include (db.h) in ${BERKELEY_DB_INCLUDE_DIR}")
if(BERKELEY_DB_LIBRARIES)
message(STATUS "Found BerkeleyDB shared library")
set(BDB_STATIC false CACHE BOOL "BDB Static flag")
set(BDB_INCLUDE ${BERKELEY_DB_INCLUDE_DIR} CACHE STRING "BDB include path")
set(BDB_LIBRARY ${BERKELEY_DB_LIBRARIES} CACHE STRING "BDB library name")
set(BDB_LIBRARY_DIRS "" CACHE STRING "BDB Library dirs")
else()
die("Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev or the equivalent")
endif()
endif()

View file

@ -29,8 +29,10 @@
#pragma once
#include <boost/filesystem.hpp>
#include "cryptonote_core/blockchain.h" // BlockchainDB and LMDB
#include "cryptonote_core/blockchain.h" // BlockchainDB
#include "cryptonote_core/blockchain_storage.h" // in-memory DB
#include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h"
using namespace cryptonote;
@ -53,7 +55,27 @@ struct fake_core_lmdb
#endif
{
m_pool.init(path.string());
m_storage.init(path.string(), use_testnet, mdb_flags);
BlockchainDB* db = new BlockchainLMDB();
boost::filesystem::path folder(path);
folder /= db->get_db_name();
LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ...");
const std::string filename = folder.string();
try
{
db->open(filename, mdb_flags);
}
catch (const std::exception& e)
{
LOG_PRINT_L0("Error opening database: " << e.what());
throw;
}
m_storage.init(db, use_testnet);
if (do_batch)
m_storage.get_db().set_batch_transactions(do_batch);
support_batch = true;

View file

@ -29,6 +29,7 @@
set(blockchain_db_sources
blockchain_db.cpp
lmdb/db_lmdb.cpp
berkeleydb/db_bdb.cpp
)
set(blockchain_db_headers)
@ -36,6 +37,7 @@ set(blockchain_db_headers)
set(blockchain_db_private_headers
blockchain_db.h
lmdb/db_lmdb.h
berkeleydb/db_bdb.h
)
bitmonero_private_headers(blockchain_db
@ -52,9 +54,10 @@ target_link_libraries(blockchain_db
${Boost_DATE_TIME_LIBRARY}
${Boost_PROGRAM_OPTIONS_LIBRARY}
${Boost_SERIALIZATION_LIBRARY}
${LMDB_LIBRARY}
${BDB_LIBRARY}
LINK_PRIVATE
${Boost_FILESYSTEM_LIBRARY}
${Boost_SYSTEM_LIBRARY}
${Boost_THREAD_LIBRARY}
${LMDB_LIBRARY}
${EXTRA_LIBRARIES})

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,288 @@
// Copyright (c) 2014, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <db_cxx.h>
#include "blockchain_db/blockchain_db.h"
#include "cryptonote_protocol/blobdatatype.h" // for type blobdata
namespace cryptonote
{
struct bdb_txn_safe
{
bdb_txn_safe() : m_txn(NULL) { }
~bdb_txn_safe()
{
LOG_PRINT_L3("bdb_txn_safe: destructor");
if (m_txn != NULL)
abort();
}
void commit(std::string message = "")
{
if (message.size() == 0)
{
message = "Failed to commit a transaction to the db";
}
if (m_txn->commit(0))
{
m_txn = NULL;
LOG_PRINT_L0(message);
throw DB_ERROR(message.c_str());
}
m_txn = NULL;
}
void abort()
{
LOG_PRINT_L3("bdb_txn_safe: abort()");
if(m_txn != NULL)
{
m_txn->abort();
m_txn = NULL;
}
else
{
LOG_PRINT_L0("WARNING: bdb_txn_safe: abort() called, but m_txn is NULL");
}
}
operator DbTxn*()
{
return m_txn;
}
operator DbTxn**()
{
return &m_txn;
}
DbTxn* m_txn;
};
class BlockchainBDB : public BlockchainDB
{
public:
BlockchainBDB(bool batch_transactions=false);
~BlockchainBDB();
virtual void open(const std::string& filename, const int db_flags);
virtual void close();
virtual void sync();
virtual void reset();
virtual std::vector<std::string> get_filenames() const;
virtual std::string get_db_name() const;
virtual bool lock();
virtual void unlock();
virtual bool block_exists(const crypto::hash& h) const;
virtual block get_block(const crypto::hash& h) const;
virtual uint64_t get_block_height(const crypto::hash& h) const;
virtual block_header get_block_header(const crypto::hash& h) const;
virtual block get_block_from_height(const uint64_t& height) const;
virtual uint64_t get_block_timestamp(const uint64_t& height) const;
virtual uint64_t get_top_block_timestamp() const;
virtual size_t get_block_size(const uint64_t& height) const;
virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const;
virtual difficulty_type get_block_difficulty(const uint64_t& height) const;
virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const;
virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const;
virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const;
virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const;
virtual crypto::hash top_block_hash() const;
virtual block get_top_block() const;
virtual uint64_t height() const;
virtual bool tx_exists(const crypto::hash& h) const;
virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const;
virtual transaction get_tx(const crypto::hash& h) const;
virtual uint64_t get_tx_count() const;
virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const;
virtual uint64_t get_tx_block_height(const crypto::hash& h) const;
virtual uint64_t get_random_output(const uint64_t& amount) const;
virtual uint64_t get_num_outputs(const uint64_t& amount) const;
virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const;
virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const;
/**
* @brief get an output from its global index
*
* @param index global index of the output desired
*
* @return the output associated with the index.
* Will throw OUTPUT_DNE if not output has that global index.
* Will throw DB_ERROR if there is a non-specific LMDB error in fetching
*/
tx_out get_output(const uint64_t& index) const;
virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const;
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const;
virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const;
virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const;
virtual bool has_key_image(const crypto::key_image& img) const;
virtual uint64_t add_block( const block& blk
, const size_t& block_size
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const std::vector<transaction>& txs
);
virtual void set_batch_transactions(bool batch_transactions);
virtual void batch_start();
virtual void batch_commit();
virtual void batch_stop();
virtual void batch_abort();
virtual void pop_block(block& blk, std::vector<transaction>& txs);
private:
virtual void add_block( const block& blk
, const size_t& block_size
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const crypto::hash& block_hash
);
virtual void remove_block();
virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash);
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index);
virtual void remove_output(const tx_out& tx_output);
void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx);
void remove_output(const uint64_t& out_index, const uint64_t amount);
void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index);
virtual void add_spent_key(const crypto::key_image& k_image);
virtual void remove_spent_key(const crypto::key_image& k_image);
/**
* @brief convert a tx output to a blob for storage
*
* @param output the output to convert
*
* @return the resultant blob
*/
blobdata output_to_blob(const tx_out& output);
/**
* @brief convert a tx output blob to a tx output
*
* @param blob the blob to convert
*
* @return the resultant tx output
*/
tx_out output_from_blob(const blobdata& blob) const;
/**
* @brief get the global index of the index-th output of the given amount
*
* @param amount the output amount
* @param index the index into the set of outputs of that amount
*
* @return the global index of the desired output
*/
uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const;
void check_open() const;
DbEnv* m_env;
Db* m_blocks;
Db* m_block_heights;
Db* m_block_hashes;
Db* m_block_timestamps;
Db* m_block_sizes;
Db* m_block_diffs;
Db* m_block_coins;
Db* m_txs;
Db* m_tx_unlocks;
Db* m_tx_heights;
Db* m_tx_outputs;
Db* m_output_txs;
Db* m_output_indices;
Db* m_output_amounts;
Db* m_output_keys;
Db* m_spent_keys;
uint64_t m_height;
uint64_t m_num_outputs;
std::string m_folder;
bdb_txn_safe *m_write_txn;
bool m_batch_transactions; // support for batch transactions
};
} // namespace cryptonote

View file

@ -129,6 +129,11 @@ void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs)
}
}
bool BlockchainDB::is_open()
{
return m_open;
}
void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
{
transaction tx = get_tx(tx_hash);

View file

@ -62,6 +62,7 @@
*
* General:
* open()
* is_open()
* close()
* sync()
* reset()
@ -328,8 +329,8 @@ public:
// open the db at location <filename>, or create it if there isn't one.
virtual void open(const std::string& filename, const int db_flags = 0) = 0;
// make sure implementation has a create function as well
virtual void create(const std::string& filename) = 0;
// returns true of the db is open/ready, else false
bool is_open();
// close and sync the db
virtual void close() = 0;
@ -482,6 +483,7 @@ public:
// returns true if key image <img> is present in spent key images storage
virtual bool has_key_image(const crypto::key_image& img) const = 0;
bool m_open;
}; // class BlockchainDB

View file

@ -0,0 +1,40 @@
// Copyright (c) 2015, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
#pragma once
namespace cryptonote
{
const std::unordered_set<std::string> blockchain_db_types =
{ "lmdb"
, "berkeley"
};
} // namespace cryptonote

View file

@ -567,7 +567,7 @@ uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const u
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -679,7 +679,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str()));
// get a read/write MDB_txn
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, 0, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -733,13 +733,6 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
// from here, init should be finished
}
// unused for now, create will happen on open if doesn't exist
void BlockchainLMDB::create(const std::string& filename)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed.");
}
void BlockchainLMDB::close()
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@ -816,7 +809,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -849,7 +842,7 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -879,7 +872,7 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -910,8 +903,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
txn_safe* txn_ptr = &txn;
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
@ -953,8 +946,8 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
txn_safe* txn_ptr = &txn;
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
@ -983,8 +976,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t&
LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height);
check_open();
txn_safe txn;
txn_safe* txn_ptr = &txn;
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
@ -1029,8 +1022,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
txn_safe* txn_ptr = &txn;
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
@ -1059,8 +1052,8 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height)
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
txn_safe* txn_ptr = &txn;
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
@ -1152,8 +1145,8 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
txn_safe* txn_ptr = &txn;
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
@ -1187,7 +1180,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -1207,8 +1200,8 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
txn_safe* txn_ptr = &txn;
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
@ -1242,7 +1235,7 @@ uint64_t BlockchainLMDB::get_tx_count() const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -1274,8 +1267,8 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
txn_safe* txn_ptr = &txn;
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
@ -1318,7 +1311,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -1349,7 +1342,7 @@ crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const
uint64_t glob_index = get_output_global_index(amount, index);
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -1369,7 +1362,7 @@ tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index)
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -1414,7 +1407,7 @@ tx_out BlockchainLMDB::get_output(const uint64_t& index) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -1438,8 +1431,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t&
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
txn_safe* txn_ptr = &txn;
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
@ -1474,8 +1467,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
txn_safe* txn_ptr = &txn;
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
@ -1524,7 +1517,7 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_output_indices(const crypto::hash&
check_open();
std::vector<uint64_t> index_vec;
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -1569,7 +1562,7 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const crypto:
transaction tx = get_tx(h);
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -1640,7 +1633,7 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
@ -1751,7 +1744,7 @@ uint64_t BlockchainLMDB::add_block( const block& blk
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (! m_batch_active)
{
if (mdb_txn_begin(m_env, NULL, 0, txn))
@ -1789,7 +1782,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
txn_safe txn;
mdb_txn_safe txn;
if (! m_batch_active)
{
if (mdb_txn_begin(m_env, NULL, 0, txn))

View file

@ -24,6 +24,7 @@
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include "blockchain_db/blockchain_db.h"
#include "cryptonote_protocol/blobdatatype.h" // for type blobdata
@ -33,17 +34,17 @@
namespace cryptonote
{
struct txn_safe
struct mdb_txn_safe
{
txn_safe() : m_txn(NULL) { }
~txn_safe()
mdb_txn_safe() : m_txn(NULL) { }
~mdb_txn_safe()
{
LOG_PRINT_L3("txn_safe: destructor");
LOG_PRINT_L3("mdb_txn_safe: destructor");
if (m_txn != NULL)
{
if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
{
LOG_PRINT_L0("WARNING: txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
}
else
{
@ -53,7 +54,7 @@ struct txn_safe
//
// NOTE: not sure if this is ever reached for a non-batch write
// transaction, but it's probably not ideal if it did.
LOG_PRINT_L3("txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
}
mdb_txn_abort(m_txn);
}
@ -77,11 +78,11 @@ struct txn_safe
// This should only be needed for batch transaction which must be ensured to
// be aborted before mdb_env_close, not after. So we can't rely on
// BlockchainLMDB destructor to call txn_safe destructor, as that's too late
// BlockchainLMDB destructor to call mdb_txn_safe destructor, as that's too late
// to properly abort, since mdb_env_close would have been called earlier.
void abort()
{
LOG_PRINT_L3("txn_safe: abort()");
LOG_PRINT_L3("mdb_txn_safe: abort()");
if(m_txn != NULL)
{
mdb_txn_abort(m_txn);
@ -89,7 +90,7 @@ struct txn_safe
}
else
{
LOG_PRINT_L0("WARNING: txn_safe: abort() called, but m_txn is NULL");
LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL");
}
}
@ -116,8 +117,6 @@ public:
virtual void open(const std::string& filename, const int mdb_flags=0);
virtual void create(const std::string& filename);
virtual void close();
virtual void sync();
@ -302,12 +301,11 @@ private:
MDB_dbi m_spent_keys;
bool m_open;
uint64_t m_height;
uint64_t m_num_outputs;
std::string m_folder;
txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
mdb_txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
bool m_batch_transactions; // support for batch transactions
bool m_batch_active; // whether batch transaction is in progress

View file

@ -39,7 +39,6 @@
#include "tx_pool.h"
#include "blockchain.h"
#include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h"
#include "cryptonote_format_utils.h"
#include "cryptonote_boost_serialization.h"
#include "cryptonote_config.h"
@ -65,8 +64,7 @@ using epee::string_tools::pod_to_hex;
DISABLE_VS_WARNINGS(4267)
//------------------------------------------------------------------
// TODO: initialize m_db with a concrete implementation of BlockchainDB
Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_testnet(false), m_enforce_dns_checkpoints(false)
Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false)
{
LOG_PRINT_L3("Blockchain::" << __func__);
}
@ -226,43 +224,24 @@ uint64_t Blockchain::get_current_blockchain_height() const
//------------------------------------------------------------------
//FIXME: possibly move this into the constructor, to avoid accidentally
// dereferencing a null BlockchainDB pointer
bool Blockchain::init(const std::string& config_folder, const bool testnet, const int db_flags)
bool Blockchain::init(BlockchainDB* db, const bool testnet)
{
LOG_PRINT_L3("Blockchain::" << __func__);
CRITICAL_REGION_LOCAL(m_blockchain_lock);
// TODO: make this configurable
m_db = new BlockchainLMDB();
m_config_folder = config_folder;
m_testnet = testnet;
boost::filesystem::path folder(m_config_folder);
folder /= m_db->get_db_name();
LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ...");
const std::string filename = folder.string();
try
if (db == nullptr)
{
m_db->open(filename, db_flags);
}
catch (const DB_OPEN_FAILURE& e)
{
LOG_PRINT_L0("No blockchain file found, attempting to create one.");
try
{
m_db->create(filename);
}
catch (const DB_CREATE_FAILURE& db_create_error)
{
LOG_PRINT_L0("Unable to create BlockchainDB! This is not good...");
//TODO: make sure whatever calls this handles the return value properly
LOG_ERROR("Attempted to init Blockchain with null DB");
return false;
}
if (!db->is_open())
{
LOG_ERROR("Attempted to init Blockchain with unopened DB");
return false;
}
m_db = db;
// if the blockchain is new, add the genesis block
// this feels kinda kludgy to do it this way, but can be looked at later.
// TODO: add function to create and store genesis block,

View file

@ -81,7 +81,7 @@ namespace cryptonote
Blockchain(tx_memory_pool& tx_pool);
bool init(const std::string& config_folder, const bool testnet = false, const int db_flags = 0);
bool init(BlockchainDB* db, const bool testnet = false);
bool deinit();
void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; }
@ -180,11 +180,9 @@ namespace cryptonote
outputs_container m_outputs;
std::string m_config_folder;
checkpoints m_checkpoints;
std::atomic<bool> m_is_in_checkpoint_zone;
std::atomic<bool> m_is_blockchain_storing;
bool m_testnet;
bool m_enforce_dns_checkpoints;
bool switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::iterator>& alt_chain, bool discard_disconnected_chain);

View file

@ -44,6 +44,9 @@ using namespace epee;
#include <csignal>
#include "daemon/command_line_args.h"
#include "cryptonote_core/checkpoints_create.h"
#include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h"
#include "blockchain_db/berkeleydb/db_bdb.h"
DISABLE_VS_WARNINGS(4355)
@ -194,7 +197,45 @@ namespace cryptonote
r = m_mempool.init(m_config_folder);
CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool");
#if BLOCKCHAIN_DB == DB_LMDB
std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type);
BlockchainDB* db = nullptr;
if (db_type == "lmdb")
{
db = new BlockchainLMDB();
}
else if (db_type == "berkeley")
{
db = new BlockchainBDB();
}
else
{
LOG_ERROR("Attempted to use non-existant database type");
return false;
}
boost::filesystem::path folder(m_config_folder);
folder /= db->get_db_name();
LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ...");
const std::string filename = folder.string();
try
{
db->open(filename);
}
catch (const DB_ERROR& e)
{
LOG_PRINT_L0("Error opening database: " << e.what());
return false;
}
r = m_blockchain_storage.init(db, m_testnet);
#else
r = m_blockchain_storage.init(m_config_folder, m_testnet);
#endif
CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage");
// load json & DNS checkpoints, and verify them

View file

@ -70,6 +70,11 @@ namespace daemon_args
, "checkpoints from DNS server will be enforced"
, false
};
const command_line::arg_descriptor<std::string> arg_db_type = {
"db-type"
, "Specify database type"
, "lmdb"
};
} // namespace daemon_args

View file

@ -42,6 +42,7 @@
#include "rpc/core_rpc_server.h"
#include <boost/program_options.hpp>
#include "daemon/command_line_args.h"
#include "blockchain_db/db_types.h"
namespace po = boost::program_options;
namespace bf = boost::filesystem;
@ -78,6 +79,7 @@ int main(int argc, char const * argv[])
command_line::add_arg(core_settings, daemon_args::arg_log_level);
command_line::add_arg(core_settings, daemon_args::arg_testnet_on);
command_line::add_arg(core_settings, daemon_args::arg_dns_checkpoints);
command_line::add_arg(core_settings, daemon_args::arg_db_type);
daemonizer::init_options(hidden_options, visible_options);
daemonize::t_executor::init_options(core_settings);
@ -128,6 +130,19 @@ int main(int argc, char const * argv[])
return 0;
}
std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type);
// verify that blockchaindb type is valid
if(cryptonote::blockchain_db_types.count(db_type) == 0)
{
std::cout << "Invalid database type (" << db_type << "), available types are:" << std::endl;
for (const auto& type : cryptonote::blockchain_db_types)
{
std::cout << "\t" << type << std::endl;
}
return 0;
}
bool testnet_mode = command_line::get_arg(vm, daemon_args::arg_testnet_on);
auto data_dir_arg = testnet_mode ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;

View file

@ -30,11 +30,14 @@
#include <boost/algorithm/string/predicate.hpp>
#include <cstdio>
#include <iostream>
#include <chrono>
#include <thread>
#include "gtest/gtest.h"
#include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h"
#include "blockchain_db/berkeleydb/db_bdb.h"
#include "cryptonote_core/cryptonote_format_utils.h"
using namespace cryptonote;
@ -87,6 +90,7 @@ bool compare_blocks(const block& a, const block& b)
return hash_a == hash_b;
}
/*
void print_block(const block& blk, const std::string& prefix = "")
{
std::cerr << prefix << ": " << std::endl
@ -105,6 +109,7 @@ bool compare_txs(const transaction& a, const transaction& b)
return ab == bb;
}
*/
// convert hex string to string that has values based on that hex
// thankfully should automatically ignore null-terminator.
@ -172,20 +177,29 @@ protected:
}
~BlockchainDBTest() {
auto files = m_db->get_filenames();
delete m_db;
remove_files(files);
remove_files();
}
BlockchainDB* m_db;
std::string m_prefix;
std::vector<block> m_blocks;
std::vector<std::vector<transaction> > m_txs;
std::vector<std::string> m_filenames;
void remove_files(const std::vector<std::string>& files)
void get_filenames()
{
m_filenames = m_db->get_filenames();
for (auto& f : m_filenames)
{
std::cerr << "File created by test: " << f << std::endl;
}
}
void remove_files()
{
// remove each file the db created, making sure it starts with fname.
for (auto& f : files)
for (auto& f : m_filenames)
{
if (boost::starts_with(f, m_prefix))
{
@ -198,7 +212,7 @@ protected:
}
// remove directory if it still exists
boost::filesystem::remove(m_prefix);
boost::filesystem::remove_all(m_prefix);
}
void set_prefix(const std::string& prefix)
@ -209,7 +223,7 @@ protected:
using testing::Types;
typedef Types<BlockchainLMDB> implementations;
typedef Types<BlockchainLMDB, BlockchainBDB> implementations;
TYPED_TEST_CASE(BlockchainDBTest, implementations);
@ -221,6 +235,7 @@ TYPED_TEST(BlockchainDBTest, OpenAndClose)
// make sure open does not throw
ASSERT_NO_THROW(this->m_db->open(fname));
this->get_filenames();
// make sure open when already open DOES throw
ASSERT_THROW(this->m_db->open(fname), DB_OPEN_FAILURE);
@ -231,9 +246,11 @@ TYPED_TEST(BlockchainDBTest, OpenAndClose)
TYPED_TEST(BlockchainDBTest, AddBlock)
{
std::string fname(tmpnam(NULL));
this->set_prefix(fname);
// make sure open does not throw
ASSERT_NO_THROW(this->m_db->open(fname));
this->get_filenames();
// adding a block with no parent in the blockchain should throw.
// note: this shouldn't be possible, but is a good (and cheap) failsafe.
@ -272,9 +289,11 @@ TYPED_TEST(BlockchainDBTest, AddBlock)
TYPED_TEST(BlockchainDBTest, RetrieveBlockData)
{
std::string fname(tmpnam(NULL));
this->set_prefix(fname);
// make sure open does not throw
ASSERT_NO_THROW(this->m_db->open(fname));
this->get_filenames();
ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0]));

View file

@ -57,6 +57,7 @@ add_executable(unit_tests
target_link_libraries(unit_tests
LINK_PRIVATE
cryptonote_core
blockchain_db
rpc
wallet
${GTEST_MAIN_LIBRARIES}