From 15ee0bef4b07dd177237380130b7c9accb3a215b Mon Sep 17 00:00:00 2001 From: warptangent Date: Mon, 8 Feb 2016 04:57:16 -0800 Subject: [PATCH 1/8] BlockchainLMDB: extract txn macros used during block add/remove --- src/blockchain_db/lmdb/db_lmdb.cpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 5982b9d9..65003c61 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1265,6 +1265,32 @@ void BlockchainLMDB::unlock() auto_txn.commit(); \ } while(0) + +// The below two macros are for DB access within block add/remove, whether +// regular batch txn is in use or not. m_write_txn is used as a batch txn, even +// if it's only within block add/remove. +// +// DB access functions that may be called both within block add/remove and +// without should use these. If the function will be called ONLY within block +// add/remove, m_write_txn alone may be used instead of these macros. + +#define TXN_BLOCK_PREFIX(flags); \ + mdb_txn_safe auto_txn; \ + mdb_txn_safe* txn_ptr = &auto_txn; \ + if (m_batch_active || m_write_txn) \ + txn_ptr = m_write_txn; \ + else \ + { \ + if (auto mdb_res = mdb_txn_begin(m_env, NULL, flags, auto_txn)) \ + throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \ + } \ + +#define TXN_BLOCK_POSTFIX_SUCCESS() \ + do { \ + if (! m_batch_active && ! m_write_txn) \ + auto_txn.commit(); \ + } while(0) + bool BlockchainLMDB::block_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); From f47d5b0fe37adb4d015c3c6199b4d3b58563b1c1 Mon Sep 17 00:00:00 2001 From: warptangent Date: Mon, 8 Feb 2016 04:58:08 -0800 Subject: [PATCH 2/8] BlockchainLMDB: Allow two HardFork functions to update DB during block add Note that this doesn't yet cause them to be called during block add. --- src/blockchain_db/lmdb/db_lmdb.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 65003c61..401983f1 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2507,14 +2507,14 @@ void BlockchainLMDB::set_hard_fork_starting_height(uint8_t version, uint64_t hei LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - TXN_PREFIX(0); + TXN_BLOCK_PREFIX(0); MDB_val_copy val_key(version); MDB_val_copy val_value(height); if (auto result = mdb_put(*txn_ptr, m_hf_starting_heights, &val_key, &val_value, 0)) throw1(DB_ERROR(std::string("Error adding hard fork starting height to db transaction: ").append(mdb_strerror(result)).c_str())); - TXN_POSTFIX_SUCCESS(); + TXN_BLOCK_POSTFIX_SUCCESS(); } uint64_t BlockchainLMDB::get_hard_fork_starting_height(uint8_t version) const @@ -2542,14 +2542,14 @@ void BlockchainLMDB::set_hard_fork_version(uint64_t height, uint8_t version) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - TXN_PREFIX(0); + TXN_BLOCK_PREFIX(0); MDB_val_copy val_key(height); MDB_val_copy val_value(version); if (auto result = mdb_put(*txn_ptr, m_hf_versions, &val_key, &val_value, 0)) throw1(DB_ERROR(std::string("Error adding hard fork version to db transaction: ").append(mdb_strerror(result)).c_str())); - TXN_POSTFIX_SUCCESS(); + TXN_BLOCK_POSTFIX_SUCCESS(); } uint8_t BlockchainLMDB::get_hard_fork_version(uint64_t height) const From f3a60000946c86a42abbac3b8c4d9e6865b9cefb Mon Sep 17 00:00:00 2001 From: warptangent Date: Mon, 8 Feb 2016 08:32:19 -0800 Subject: [PATCH 3/8] BlockchainDB/LMDB/BDB: Extract DB txn functions for block add/remove --- src/blockchain_db/berkeleydb/db_bdb.cpp | 15 ++++++++++ src/blockchain_db/berkeleydb/db_bdb.h | 4 +++ src/blockchain_db/blockchain_db.h | 4 +++ src/blockchain_db/lmdb/db_lmdb.cpp | 38 +++++++++++++++++++++++++ src/blockchain_db/lmdb/db_lmdb.h | 4 +++ 5 files changed, 65 insertions(+) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 2c18ae5d..f572ddf9 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -1831,6 +1831,21 @@ void BlockchainBDB::set_batch_transactions(bool batch_transactions) LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } +void BlockchainBDB::block_txn_start() +{ + // TODO +} + +void BlockchainBDB::block_txn_stop() +{ + // TODO +} + +void BlockchainBDB::block_txn_abort() +{ + // TODO +} + uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector& txs) { LOG_PRINT_L3("BlockchainBDB::" << __func__); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index 6db91aa1..42119da9 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -328,6 +328,10 @@ public: virtual void batch_stop(); virtual void batch_abort(); + virtual void block_txn_start(); + virtual void block_txn_stop(); + virtual void block_txn_abort(); + virtual void pop_block(block& blk, std::vector& txs); #if defined(BDB_BULK_CAN_THREAD) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 4a140e5f..31d5e264 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -368,6 +368,10 @@ public: virtual void batch_stop() = 0; virtual void set_batch_transactions(bool) = 0; + virtual void block_txn_start() = 0; + virtual void block_txn_stop() = 0; + virtual void block_txn_abort() = 0; + // adds a block with the given metadata to the top of the blockchain, returns the new height // NOTE: subclass implementations of this (or the functions it calls) need // to handle undoing any partially-added blocks in the event of a failure. diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 401983f1..de82357b 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2190,6 +2190,44 @@ void BlockchainLMDB::set_batch_transactions(bool batch_transactions) LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } +void BlockchainLMDB::block_txn_start() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_active && m_write_txn) + throw0(DB_ERROR((std::string("Attempted to start new write txn when write txn already exists in ")+__FUNCTION__).c_str())); + if (! m_batch_active) + { + m_write_txn = new mdb_txn_safe(); + if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, *m_write_txn)) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str())); + } +} + +void BlockchainLMDB::block_txn_stop() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_active) + { + TIME_MEASURE_START(time1); + m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + + delete m_write_txn; + m_write_txn = NULL; + } +} + +void BlockchainLMDB::block_txn_abort() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_active) + { + delete m_write_txn; + m_write_txn = NULL; + } +} + uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector& txs) { diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index eb0704ab..e88bcd01 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -194,6 +194,10 @@ public: virtual void batch_stop(); virtual void batch_abort(); + virtual void block_txn_start(); + virtual void block_txn_stop(); + virtual void block_txn_abort(); + virtual void pop_block(block& blk, std::vector& txs); virtual bool can_thread_bulk_indices() const { return true; } From fd46c96dce043cbf9f018f59ac9cedb12f6a4286 Mon Sep 17 00:00:00 2001 From: warptangent Date: Mon, 8 Feb 2016 08:32:36 -0800 Subject: [PATCH 4/8] BlockchainDB/LMDB: Refactor block-scope DB txn handling for add block Move block-scope txn start and stop from BlockchainLMDB to BlockchainDB. --- src/blockchain_db/blockchain_db.cpp | 10 +++++++++- src/blockchain_db/lmdb/db_lmdb.cpp | 20 +------------------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 3737dfc4..b504f946 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -99,6 +99,8 @@ uint64_t BlockchainDB::add_block( const block& blk , const std::vector& txs ) { + block_txn_start(); + TIME_MEASURE_START(time1); crypto::hash blk_hash = get_block_hash(blk); TIME_MEASURE_FINISH(time1); @@ -125,9 +127,15 @@ uint64_t BlockchainDB::add_block( const block& blk TIME_MEASURE_FINISH(time1); time_add_transaction += time1; + // DB's new height based on this added block is only incremented after this + // function returns, so height() here returns the new previous height. + uint64_t prev_height = height(); + + block_txn_stop(); + ++num_calls; - return height(); + return prev_height; } void BlockchainDB::pop_block(block& blk, std::vector& txs) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index de82357b..8bd64e70 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2244,33 +2244,15 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c } } - mdb_txn_safe txn; - if (! m_batch_active) - { - if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str())); - m_write_txn = &txn; - } - uint64_t num_outputs = m_num_outputs; try { BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); - if (! m_batch_active) - { - m_write_txn = NULL; - - TIME_MEASURE_START(time1); - txn.commit(); - TIME_MEASURE_FINISH(time1); - time_commit1 += time1; - } } catch (...) { m_num_outputs = num_outputs; - if (! m_batch_active) - m_write_txn = NULL; + block_txn_abort(); throw; } From 3800875406fecab5123564e58ddb698bce550441 Mon Sep 17 00:00:00 2001 From: warptangent Date: Mon, 8 Feb 2016 07:51:57 -0800 Subject: [PATCH 5/8] Make HardFork object available to BlockchainDB and derived DB implementations This will later allow the HardFork object's DB update functions to be called when the DB transaction that persists across block add/remove is open. --- src/blockchain_db/berkeleydb/db_bdb.cpp | 2 ++ src/blockchain_db/blockchain_db.cpp | 5 +++++ src/blockchain_db/blockchain_db.h | 7 +++++++ src/blockchain_db/lmdb/db_lmdb.cpp | 2 ++ src/cryptonote_core/blockchain.cpp | 2 ++ 5 files changed, 18 insertions(+) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index f572ddf9..cdbca52f 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -781,6 +781,8 @@ BlockchainBDB::BlockchainBDB(bool batch_transactions) : m_batch_transactions = batch_transactions; m_write_txn = nullptr; m_height = 0; + + m_hardfork = nullptr; } void BlockchainBDB::open(const std::string& filename, const int db_flags) diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index b504f946..4b05ddf2 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -138,6 +138,11 @@ uint64_t BlockchainDB::add_block( const block& blk return prev_height; } +void BlockchainDB::set_hard_fork(HardFork*& hf) +{ + m_hardfork = hf; +} + void BlockchainDB::pop_block(block& blk, std::vector& txs) { blk = get_top_block(); diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 31d5e264..5926f34a 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -28,12 +28,15 @@ #ifndef BLOCKCHAIN_DB_H #define BLOCKCHAIN_DB_H +#pragma once + #include #include #include #include "crypto/hash.h" #include "cryptonote_core/cryptonote_basic.h" #include "cryptonote_core/difficulty.h" +#include "cryptonote_core/hardfork.h" /* DB Driver Interface * @@ -322,6 +325,8 @@ protected: uint64_t time_commit1 = 0; bool m_auto_remove_logs = true; + HardFork* m_hardfork; + public: // virtual dtor @@ -372,6 +377,8 @@ public: virtual void block_txn_stop() = 0; virtual void block_txn_abort() = 0; + virtual void set_hard_fork(HardFork*& hf); + // adds a block with the given metadata to the top of the blockchain, returns the new height // NOTE: subclass implementations of this (or the functions it calls) need // to handle undoing any partially-added blocks in the event of a failure. diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 8bd64e70..28e6f552 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -945,6 +945,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) m_write_batch_txn = nullptr; m_batch_active = false; m_height = 0; + + m_hardfork = nullptr; } void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index e43875bd..94ef4b89 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -290,6 +290,8 @@ bool Blockchain::init(BlockchainDB* db, const bool testnet, const bool fakechain } m_hardfork->init(); + m_db->set_hard_fork(m_hardfork); + // if the blockchain is new, add the genesis block // this feels kinda kludgy to do it this way, but can be looked at later. // TODO: add function to create and store genesis block, From e02577f594ecd0a619ef3efaf65de90df9c20289 Mon Sep 17 00:00:00 2001 From: warptangent Date: Mon, 8 Feb 2016 08:09:07 -0800 Subject: [PATCH 6/8] Move HardFork DB update to BlockchainDB::add_block() Ensures the database is consistent. Also simplifes blockchain_import in that verify mode off has less to work around. --- src/blockchain_db/blockchain_db.cpp | 1 + src/blockchain_utilities/blockchain_import.cpp | 3 --- src/cryptonote_core/blockchain.cpp | 3 --- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 4b05ddf2..270b5399 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -130,6 +130,7 @@ uint64_t BlockchainDB::add_block( const block& blk // DB's new height based on this added block is only incremented after this // function returns, so height() here returns the new previous height. uint64_t prev_height = height(); + m_hardfork->add(blk, prev_height); block_txn_stop(); diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 7eb493b6..447d3a21 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -469,9 +469,6 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path, try { simple_core.add_block(b, block_size, cumulative_difficulty, coins_generated, txs); - #if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) - simple_core.m_hardfork->add(b, h-1); - #endif } catch (const std::exception& e) { diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 94ef4b89..6fe99826 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2692,9 +2692,6 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& TIME_MEASURE_FINISH(addblock); - // this will not fail since check succeeded above - m_hardfork->add(bl, new_height - 1); - // do this after updating the hard fork state since the size limit may change due to fork update_next_cumulative_size_limit(); From b368e29f67a856872fa7ae01b918d66d76545929 Mon Sep 17 00:00:00 2001 From: warptangent Date: Mon, 8 Feb 2016 08:12:11 -0800 Subject: [PATCH 7/8] blockchain_import: Get hard fork version from HardFork Replace temporary assignment that used hardcoded values. --- src/blockchain_utilities/blockchain_import.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 447d3a21..d3045a22 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -400,11 +400,8 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path, // size_t blob_size = 0; // get_transaction_hash(tx, hsh, blob_size); - // we'd need to get the starting heights from the daemon - // to be correct once voting kicks in - uint64_t v2height = opt_testnet ? 624634 : 1009827; - uint8_t version = h < v2height ? 1 : 2; + uint8_t version = simple_core.m_storage.get_current_hard_fork_version(); tx_verification_context tvc = AUTO_VAL_INIT(tvc); bool r = true; r = simple_core.m_pool.add_tx(tx, tvc, true, true, version); From e298b14a55c7f5792481a488104e6358127b6aa3 Mon Sep 17 00:00:00 2001 From: warptangent Date: Mon, 8 Feb 2016 08:19:24 -0800 Subject: [PATCH 8/8] Blockchain: Update comments on removing block --- src/cryptonote_core/blockchain.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 6fe99826..f9247ae2 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -461,6 +461,11 @@ block Blockchain::pop_block_from_blockchain() { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + // FIXME: HardFork + // Besides the below, popping a block should also remove the last entry + // in hf_versions. + // + // FIXME: HardFork // This is not quite correct, as we really want to add the txes // to the pool based on the version determined after all blocks // are popped.