From 5f397e441275d0e282a371bcdeed963c9a9b69f1 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 25 Oct 2015 10:45:25 +0000 Subject: [PATCH] Add functions to iterate through blocks, txes, outputs, key images --- src/blockchain_db/berkeleydb/db_bdb.cpp | 120 ++++++++++++++++ src/blockchain_db/berkeleydb/db_bdb.h | 5 + src/blockchain_db/blockchain_db.h | 5 + src/blockchain_db/lmdb/db_lmdb.cpp | 159 +++++++++++++++++++++ src/blockchain_db/lmdb/db_lmdb.h | 5 + src/cryptonote_core/blockchain.cpp | 20 +++ src/cryptonote_core/blockchain.h | 5 + src/cryptonote_core/blockchain_storage.cpp | 38 +++++ src/cryptonote_core/blockchain_storage.h | 5 + 9 files changed, 362 insertions(+) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 7648d657..6560ce5c 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -533,6 +533,126 @@ void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) throw1(DB_ERROR("Error adding removal of key image to db transaction")); } +bool BlockchainBDB::for_all_key_images(std::function f) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_cur cur(DB_DEFAULT_TX, m_spent_keys); + + Dbt_copy k; + Dbt_copy v; + bool ret = true; + int result; + while ((result = cur->get(&k, &v, DB_NEXT)) == 0) + { + if (!f(k)) + { + ret = false; + break; + } + } + if (result != DB_NOTFOUND) + ret = false; + + cur.close(); + return ret; +} + +bool BlockchainBDB::for_all_blocks(std::function f) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_cur cur(DB_DEFAULT_TX, m_blocks); + + Dbt_copy k; + Dbt_safe v; + bool ret = true; + int result; + while ((result = cur->get(&k, &v, DB_NEXT)) == 0) + { + uint64_t height = k - 1; + blobdata bd; + bd.assign(reinterpret_cast(v.get_data()), v.get_size()); + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + crypto::hash hash; + if (!get_block_hash(b, hash)) + throw0(DB_ERROR("Failed to get block hash from blob retrieved from the db")); + if (!f(height, hash, b)) + { + ret = false; + break; + } + } + if (result != DB_NOTFOUND) + ret = false; + + cur.close(); + return ret; +} + +bool BlockchainBDB::for_all_transactions(std::function f) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_cur cur(DB_DEFAULT_TX, m_txs); + + Dbt_copy k; + Dbt_safe v; + bool ret = true; + int result; + while ((result = cur->get(&k, &v, DB_NEXT)) == 0) + { + blobdata bd; + bd.assign(reinterpret_cast(v.get_data()), v.get_size()); + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + if (!f(k, tx)) + { + ret = false; + break; + } + } + if (result != DB_NOTFOUND) + ret = false; + + cur.close(); + return ret; +} + +bool BlockchainBDB::for_all_outputs(std::function f) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_cur cur(DB_DEFAULT_TX, m_output_amounts); + + Dbt_copy k; + Dbt_copy v; + bool ret = true; + int result; + while ((result = cur->get(&k, &v, DB_NEXT)) == 0) + { + uint32_t global_index = v - 1; + tx_out_index toi = get_output_tx_and_index_from_global(global_index); + if (!f(k, toi.first, toi.second)) + { + ret = false; + break; + } + } + if (result != DB_NOTFOUND) + ret = false; + + cur.close(); + return ret; +} + blobdata BlockchainBDB::output_to_blob(const tx_out& output) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index 54edcf0a..b51d745d 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -375,6 +375,11 @@ private: void get_output_global_indices(const uint64_t& amount, const std::vector &offsets, std::vector &global_indices); + virtual bool for_all_key_images(std::function) const; + virtual bool for_all_blocks(std::function) const; + virtual bool for_all_transactions(std::function) const; + virtual bool for_all_outputs(std::function f) const; + // Hard fork related storage virtual void set_hard_fork_starting_height(uint8_t version, uint64_t height); virtual uint64_t get_hard_fork_starting_height(uint8_t version) const; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 24bf4024..1666e57c 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -493,6 +493,11 @@ public: // returns true if key image is present in spent key images storage virtual bool has_key_image(const crypto::key_image& img) const = 0; + virtual bool for_all_key_images(std::function) const = 0; + virtual bool for_all_blocks(std::function) const = 0; + virtual bool for_all_transactions(std::function) const = 0; + virtual bool for_all_outputs(std::function f) const = 0; + // Hard fork related storage virtual void set_hard_fork_starting_height(uint8_t version, uint64_t height) = 0; virtual uint64_t get_hard_fork_starting_height(uint8_t version) const = 0; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index fb853a7f..1e7078c6 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1968,6 +1968,165 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const return false; } +bool BlockchainLMDB::for_all_key_images(std::function f) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val k; + MDB_val v; + bool ret = true; + + lmdb_cur cur(txn, m_spent_keys); + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret < 0) + throw0(DB_ERROR("Failed to enumerate key images")); + const crypto::key_image k_image = *(crypto::key_image*)k.mv_data; + if (!f(k_image)) { + ret = false; + break; + } + } + + cur.close(); + txn.commit(); + + return ret; +} + +bool BlockchainLMDB::for_all_blocks(std::function f) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val k; + MDB_val v; + bool ret = true; + + lmdb_cur cur(txn, m_blocks); + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR("Failed to enumerate blocks")); + uint64_t height = *(uint64_t*)k.mv_data; + blobdata bd; + bd.assign(reinterpret_cast(v.mv_data), v.mv_size); + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + crypto::hash hash; + if (!get_block_hash(b, hash)) + throw0(DB_ERROR("Failed to get block hash from blob retrieved from the db")); + if (!f(height, hash, b)) { + ret = false; + break; + } + } + + cur.close(); + txn.commit(); + + return ret; +} + +bool BlockchainLMDB::for_all_transactions(std::function f) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val k; + MDB_val v; + bool ret = true; + + lmdb_cur cur(txn, m_txs); + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR("Failed to enumerate transactions")); + const crypto::hash hash = *(crypto::hash*)k.mv_data; + blobdata bd; + bd.assign(reinterpret_cast(v.mv_data), v.mv_size); + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + if (!f(hash, tx)) { + ret = false; + break; + } + } + + cur.close(); + txn.commit(); + + return ret; +} + +bool BlockchainLMDB::for_all_outputs(std::function f) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val k; + MDB_val v; + bool ret = true; + + lmdb_cur cur(txn, m_output_amounts); + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR("Failed to enumerate outputs")); + uint64_t amount = *(uint64_t*)k.mv_data; + uint64_t global_index = *(uint64_t*)v.mv_data; + tx_out_index toi = get_output_tx_and_index_from_global(global_index); + if (!f(amount, toi.first, toi.second)) { + ret = false; + break; + } + } + + cur.close(); + txn.commit(); + + return ret; +} + // batch_num_blocks: (optional) Used to check if resize needed before batch transaction starts. void BlockchainLMDB::batch_start(uint64_t batch_num_blocks) { diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 38095429..6125ef2e 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -189,6 +189,11 @@ public: virtual bool has_key_image(const crypto::key_image& img) const; + virtual bool for_all_key_images(std::function) const; + virtual bool for_all_blocks(std::function) const; + virtual bool for_all_transactions(std::function) const; + virtual bool for_all_outputs(std::function f) const; + virtual uint64_t add_block( const block& blk , const size_t& block_size , const difficulty_type& cumulative_difficulty diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index c38b5884..08587491 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -3188,3 +3188,23 @@ bool Blockchain::get_hard_fork_voting_info(uint8_t version, uint32_t &window, ui { return m_hardfork->get_voting_info(version, window, votes, threshold, voting); } + +bool Blockchain::for_all_key_images(std::function f) const +{ + return m_db->for_all_key_images(f); +} + +bool Blockchain::for_all_blocks(std::function f) const +{ + return m_db->for_all_blocks(f); +} + +bool Blockchain::for_all_transactions(std::function f) const +{ + return m_db->for_all_transactions(f); +} + +bool Blockchain::for_all_outputs(std::function f) const +{ + return m_db->for_all_outputs(f);; +} diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 3a663a34..21bbfb44 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -165,6 +165,11 @@ namespace cryptonote uint8_t get_ideal_hard_fork_version() const { return m_hardfork->get_ideal_version(); } bool get_hard_fork_voting_info(uint8_t version, uint32_t &window, uint32_t &votes, uint32_t &threshold, uint8_t &voting) const; + bool for_all_key_images(std::function) const; + bool for_all_blocks(std::function) const; + bool for_all_transactions(std::function) const; + bool for_all_outputs(std::function) const; + BlockchainDB& get_db() { return *m_db; diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index 4a4d348b..16aaf707 100644 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -1890,3 +1890,41 @@ void blockchain_storage::set_enforce_dns_checkpoints(bool enforce_checkpoints) { m_enforce_dns_checkpoints = enforce_checkpoints; } +//------------------------------------------------------------------ +bool blockchain_storage::for_all_key_images(std::function f) const +{ + for (key_images_container::const_iterator i = m_spent_keys.begin(); i != m_spent_keys.end(); ++i) { + if (!f(*i)) + return false; + } + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::for_all_blocks(std::function f) const +{ + for (blocks_container::const_iterator i = m_blocks.begin(); i != m_blocks.end(); ++i) { + if (!f(i->height, i->bl)) + return false; + } + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::for_all_transactions(std::function f) const +{ + for (transactions_container::const_iterator i = m_transactions.begin(); i != m_transactions.end(); ++i) { + if (!f(i->second.tx)) + return false; + } + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::for_all_outputs(std::function f) const +{ + for (outputs_container::const_iterator i = m_outputs.begin(); i != m_outputs.end(); ++i) { + for (size_t n = 0; n < i->second.size(); ++n) { + if (!f(i->first, i->second[n].first, i->second[n].second)) + return false; + } + } + return true; +} diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index 2a4cfcd4..4a4fc14c 100644 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -185,6 +185,11 @@ namespace cryptonote difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return m_blocks[height].cumulative_difficulty; } uint64_t get_block_coins_generated(uint64_t height) const { return m_blocks[height].already_generated_coins; } + bool for_all_key_images(std::function) const; + bool for_all_blocks(std::function) const; + bool for_all_transactions(std::function) const; + bool for_all_outputs(std::function) const; + // use for testing only bool debug_pop_block_from_blockchain() { return pop_block_from_blockchain(); }