diff --git a/README.md b/README.md index 4fa21d65..3311c4ec 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,15 @@ See README.i18n While Monero isn't made to integrate with Tor, it can be used wrapped with torsocks, if you add --p2p-bind-ip 127.0.0.1 to the bitmonerod command line. You also want to set DNS requests to go over TCP, so they'll be routed through Tor, by setting DNS_PUBLIC=tcp. You may also disable IGD (UPnP port forwarding negotiation), which is pointless with Tor. To allow local connections from the wallet, add TORSOCKS_ALLOW_INBOUND=1. Example: -DNS_PUBLIC=tcp TORSOCKS_ALLOW_INBOUND=1 torsocks bitmonerod --p2p-bind-ip 127.0.0.1 --no-igd +`DNS_PUBLIC=tcp TORSOCKS_ALLOW_INBOUND=1 torsocks bitmonerod --p2p-bind-ip 127.0.0.1 --no-igd` + +TAILS ships with a very restrictive set of firewall rules. Therefore, you need to add a rule to allow this connection too, in addition to telling torsocks to allow inbound connections. Full example: + +`sudo iptables -I OUTPUT 2 -p tcp -d 127.0.0.1 -m tcp --dport 18081 -j ACCEPT` + +`DNS_PUBLIC=tcp TORSOCKS_ALLOW_INBOUND=1 torsocks ./bitmonerod --p2p-bind-ip 127.0.0.1 --no-igd --rpc-bind-ip 127.0.0.1 --data-dir /home/amnesia/Persistent/your/directory/to/the/blockchain` + +`./simplewallet` ## Using readline diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 7cb1e61a..82760dff 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -1441,8 +1441,16 @@ POP_WARNINGS #define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message) do{if(!(expr)) {LOG_ERROR(message); return fail_ret_val;};}while(0) #endif +#ifndef CHECK_AND_NO_ASSERT_MES_L +#define CHECK_AND_NO_ASSERT_MES_L(expr, fail_ret_val, l, message) do{if(!(expr)) {LOG_PRINT_L##l(message); /*LOCAL_ASSERT(expr);*/ return fail_ret_val;};}while(0) +#endif + #ifndef CHECK_AND_NO_ASSERT_MES -#define CHECK_AND_NO_ASSERT_MES(expr, fail_ret_val, message) do{if(!(expr)) {LOG_PRINT_L0(message); /*LOCAL_ASSERT(expr);*/ return fail_ret_val;};}while(0) +#define CHECK_AND_NO_ASSERT_MES(expr, fail_ret_val, message) CHECK_AND_NO_ASSERT_MES_L(expr, fail_ret_val, 0, message) +#endif + +#ifndef CHECK_AND_NO_ASSERT_MES_L1 +#define CHECK_AND_NO_ASSERT_MES_L1(expr, fail_ret_val, message) CHECK_AND_NO_ASSERT_MES_L(expr, fail_ret_val, 1, message) #endif diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index a822cce3..3a7d5333 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -168,8 +168,8 @@ response_info.m_header_info.m_content_type = " application/json"; \ LOG_PRINT( query_info.m_URI << "[" << method_name << "] processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms", LOG_LEVEL_2); -#define MAP_JON_RPC_WE(method_name, callback_f, command_type) \ - else if(callback_name == method_name) \ +#define MAP_JON_RPC_WE_IF(method_name, callback_f, command_type, cond) \ + else if((callback_name == method_name) && (cond)) \ { \ PREPARE_OBJECTS_FROM_JSON(command_type) \ epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \ @@ -184,6 +184,8 @@ return true;\ } +#define MAP_JON_RPC_WE(method_name, callback_f, command_type) MAP_JON_RPC_WE_IF(method_name, callback_f, command_type, true) + #define MAP_JON_RPC_WERI(method_name, callback_f, command_type) \ else if(callback_name == method_name) \ { \ diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index a366b402..2d8e458f 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -6771,8 +6771,8 @@ set1: if (op == MDB_GET_BOTH || rc > 0) return MDB_NOTFOUND; rc = 0; - *data = olddata; } + *data = olddata; } else { if (mc->mc_xcursor) @@ -10501,8 +10501,11 @@ mdb_drop0(MDB_cursor *mc, int subs) /* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves. * This also avoids any P_LEAF2 pages, which have no nodes. + * Also if the DB doesn't have sub-DBs and has no overflow + * pages, omit scanning leaves. */ - if (mc->mc_flags & C_SUB) + if ((mc->mc_flags & C_SUB) || + (!subs && !mc->mc_db->md_overflow_pages)) mdb_cursor_pop(mc); mdb_cursor_copy(mc, &mx); @@ -10529,6 +10532,9 @@ mdb_drop0(MDB_cursor *mc, int subs) pg, omp->mp_pages); if (rc) goto done; + mc->mc_db->md_overflow_pages -= omp->mp_pages; + if (!mc->mc_db->md_overflow_pages && !subs) + break; } else if (subs && (ni->mn_flags & F_SUBDATA)) { mdb_xcursor_init1(mc, ni); rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); @@ -10536,6 +10542,8 @@ mdb_drop0(MDB_cursor *mc, int subs) goto done; } } + if (!subs && !mc->mc_db->md_overflow_pages) + goto pop; } else { if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0) goto done; @@ -10557,6 +10565,7 @@ mdb_drop0(MDB_cursor *mc, int subs) /* no more siblings, go back to beginning * of previous level. */ +pop: mdb_cursor_pop(mc); mc->mc_ki[0] = 0; for (i=1; imc_snum; i++) { diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index a66f4a40..68f635d1 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -82,14 +82,17 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } } - add_transaction_data(blk_hash, tx, tx_hash); + uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash); + + std::vector amount_output_indices; // iterate tx.vout using indices instead of C++11 foreach syntax because // we need the index for (uint64_t i = 0; i < tx.vout.size(); ++i) { - add_output(tx_hash, tx.vout[i], i, tx.unlock_time); + amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time)); } + add_tx_amount_output_indices(tx_id, amount_output_indices); } uint64_t BlockchainDB::add_block( const block& blk diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 3585bd06..1445dd13 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -59,6 +59,39 @@ * Unspent transaction outputs are duplicated to quickly gather random * outputs to use for mixins * + * Indices and Identifiers: + * The word "index" is used ambiguously throughout this code. It is + * particularly confusing when talking about the output or transaction + * tables since their indexing can refer to themselves or each other. + * I have attempted to clarify these usages here: + * + * Blocks, transactions, and outputs are all identified by a hash. + * For storage efficiency, a 64-bit integer ID is used instead of the hash + * inside the DB. Tables exist to map between hash and ID. A block ID is + * also referred to as its "height". Transactions and outputs generally are + * not referred to by ID outside of this module, but the tx ID is returned + * by tx_exists() and used by get_tx_amount_output_indices(). Like their + * corresponding hashes, IDs are globally unique. + * + * The remaining uses of the word "index" refer to local offsets, and are + * not globally unique. An "amount output index" N refers to the Nth output + * of a specific amount. An "output local index" N refers to the Nth output + * of a specific tx. + * + * Exceptions: + * DB_ERROR -- generic + * DB_OPEN_FAILURE + * DB_CREATE_FAILURE + * DB_SYNC_FAILURE + * BLOCK_DNE + * BLOCK_PARENT_DNE + * BLOCK_EXISTS + * BLOCK_INVALID -- considering making this multiple errors + * TX_DNE + * TX_EXISTS + * OUTPUT_DNE + * OUTPUT_EXISTS + * KEY_IMAGE_EXISTS */ namespace cryptonote @@ -80,6 +113,15 @@ struct output_data_t }; #pragma pack(pop) +#pragma pack(push, 1) +struct tx_data_t +{ + uint64_t tx_id; + uint64_t unlock_time; + uint64_t block_id; +}; +#pragma pack(pop) + /*********************************** * Exception Definitions ***********************************/ @@ -311,14 +353,18 @@ private: * and the other data passed here, not the separate outputs of the * transaction. * + * It returns a tx ID, which is a mapping from the tx_hash. The tx ID + * is used in #add_tx_amount_output_indices(). + * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION * * @param blk_hash the hash of the block containing the transaction * @param tx the transaction to be added * @param tx_hash the hash of the transaction + * @return the transaction ID */ - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; /** * @brief remove data about a transaction @@ -348,6 +394,9 @@ private: * future, this tracking (of the number, at least) should be moved to * this class, as it is necessary and the same among all BlockchainDB. * + * It returns an amount output index, which is the index of the output + * for its specified amount. + * * This data should be stored in such a manner that the only thing needed to * reverse the process is the tx_out. * @@ -358,25 +407,24 @@ private: * @param tx_output the output * @param local_index index of the output in its transaction * @param unlock_time unlock time/height of the output + * @return amount output index */ - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0; + virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0; /** - * @brief remove an output + * @brief store amount output indices for a tx's outputs * - * The subclass implementing this will remove all output data it stored - * in add_output(). - * - * In addition, the subclass is responsible for correctly decrementing - * its global output counter (this may be automatic for some, such as using - * a database backend "count" feature). + * The subclass implementing this will add the amount output indices to its + * backing store in a suitable manner. The tx_id will be the same one that + * was returned from #add_output(). * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION * - * @param tx_output the output to be removed + * @param tx_id ID of the transaction containing these outputs + * @param amount_output_indices the amount output indices of the transaction */ - virtual void remove_output(const tx_out& tx_output) = 0; + virtual void add_tx_amount_output_indices(const uint64_t tx_id, const std::vector& amount_output_indices) = 0; /** * @brief store a spent key @@ -414,18 +462,6 @@ private: */ void pop_block(); - /** - * @brief helper function for add_transactions, to add each individual transaction - * - * This function is called by add_transactions() for each transaction to be - * added. - * - * @param blk_hash hash of the block which has the transaction - * @param tx the transaction to add - * @param tx_hash_ptr the hash of the transaction, if already calculated - */ - void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); - // helper function to remove transaction from blockchain /** * @brief helper function to remove transaction from the blockchain @@ -444,6 +480,18 @@ private: protected: + /** + * @brief helper function for add_transactions, to add each individual transaction + * + * This function is called by add_transactions() for each transaction to be + * added. + * + * @param blk_hash hash of the block which has the transaction + * @param tx the transaction to add + * @param tx_hash_ptr the hash of the transaction, if already calculated + */ + void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); + mutable uint64_t time_tx_exists = 0; //!< a performance metric uint64_t time_commit1 = 0; //!< a performance metric bool m_auto_remove_logs = true; //!< whether or not to automatically remove old logs @@ -930,10 +978,12 @@ public: * given hash and return true if so, false otherwise. * * @param h the hash to check against + * @param tx_id (optional) returns the tx_id for the tx hash * * @return true if the transaction exists, otherwise false */ virtual bool tx_exists(const crypto::hash& h) const = 0; + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_id) const = 0; // return unlock time of tx with hash /** @@ -1124,37 +1174,21 @@ public: */ virtual bool can_thread_bulk_indices() const = 0; - /** - * @brief gets output indices (global) for a transaction's outputs - * - * The subclass should fetch the global output indices for each output - * in the transaction with the given hash. - * - * If the transaction does not exist, the subclass should throw TX_DNE. - * - * If an output cannot be found, the subclass should throw OUTPUT_DNE. - * - * @param h a transaction hash - * - * @return a list of global output indices - */ - virtual std::vector get_tx_output_indices(const crypto::hash& h) const = 0; - /** * @brief gets output indices (amount-specific) for a transaction's outputs * * The subclass should fetch the amount-specific output indices for each - * output in the transaction with the given hash. + * output in the transaction with the given ID. * * If the transaction does not exist, the subclass should throw TX_DNE. * * If an output cannot be found, the subclass should throw OUTPUT_DNE. * - * @param h a transaction hash + * @param tx_id a transaction ID * * @return a list of amount-specific output indices */ - virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const = 0; + virtual std::vector get_tx_amount_output_indices(const uint64_t tx_id) const = 0; /** * @brief check if a key image is stored as spent diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 51fba058..8c51c09b 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -46,7 +46,7 @@ using epee::string_tools::pod_to_hex; // Increase when the DB changes in a non backward compatible way, and there // is no automatic conversion, so that a full resync is needed. -#define VERSION 0 +#define VERSION 1 namespace { @@ -65,6 +65,8 @@ inline void throw1(const T &e) throw e; } +#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} + template struct MDB_val_copy: public MDB_val { @@ -110,18 +112,10 @@ private: int compare_uint64(const MDB_val *a, const MDB_val *b) { -#ifdef MISALIGNED_OK - const uint64_t va = *(const uint64_t*)a->mv_data; - const uint64_t vb = *(const uint64_t*)b->mv_data; -#else - uint64_t va, vb; - memcpy(&va, a->mv_data, sizeof(uint64_t)); - memcpy(&vb, b->mv_data, sizeof(uint64_t)); -#endif - if (va < vb) return -1; - else if (va == vb) return 0; - else return 1; -}; + const uint64_t va = *(const uint64_t *)a->mv_data; + const uint64_t vb = *(const uint64_t *)b->mv_data; + return (va < vb) ? -1 : va > vb; +} int compare_uint8(const MDB_val *a, const MDB_val *b) { @@ -151,23 +145,41 @@ int compare_string(const MDB_val *a, const MDB_val *b) return strcmp(va, vb); } +/* DB schema: + * + * Table Key Data + * ----- --- ---- + * blocks block ID block blob + * block_heights block hash block height + * block_info block ID {block metadata} + * + * txs txn ID txn blob + * tx_indices txn hash {txn ID, metadata} + * tx_outputs txn ID [txn amount output indices] + * + * output_txs output ID {txn hash, local index} + * output_amounts amount [{amount output index, metadata}...] + * + * spent_keys input hash - + * + * Note: where the data items are of uniform size, DUPFIXED tables have + * been used to save space. In most of these cases, a dummy "zerokval" + * key is used when accessing the table; the Key listed above will be + * attached as a prefix on the Data to serve as the DUPSORT key. + * (DUPFIXED saves 8 bytes per record.) + * + * The output_amounts table doesn't use a dummy key, but uses DUPSORT. + */ const char* const LMDB_BLOCKS = "blocks"; -const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; -const char* const LMDB_BLOCK_HASHES = "block_hashes"; -const char* const LMDB_BLOCK_SIZES = "block_sizes"; -const char* const LMDB_BLOCK_DIFFS = "block_diffs"; -const char* const LMDB_BLOCK_COINS = "block_coins"; +const char* const LMDB_BLOCK_INFO = "block_info"; const char* const LMDB_TXS = "txs"; -const char* const LMDB_TX_UNLOCKS = "tx_unlocks"; -const char* const LMDB_TX_HEIGHTS = "tx_heights"; +const char* const LMDB_TX_INDICES = "tx_indices"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; const char* const LMDB_OUTPUT_TXS = "output_txs"; -const char* const LMDB_OUTPUT_INDICES = "output_indices"; const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts"; -const char* const LMDB_OUTPUT_KEYS = "output_keys"; const char* const LMDB_SPENT_KEYS = "spent_keys"; const char* const LMDB_HF_STARTING_HEIGHTS = "hf_starting_heights"; @@ -175,6 +187,8 @@ const char* const LMDB_HF_VERSIONS = "hf_versions"; const char* const LMDB_PROPERTIES = "properties"; +const char zerokey[8] = {0}; +const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; const std::string lmdb_error(const std::string& error_string, int mdb_res) { @@ -214,6 +228,39 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { + +typedef struct mdb_block_info +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_size; // a size_t really but we need 32-bit compat + difficulty_type bi_diff; + crypto::hash bi_hash; +} mdb_block_info; + +typedef struct blk_height { + crypto::hash bh_hash; + uint64_t bh_height; +} blk_height; + +typedef struct txindex { + crypto::hash key; + tx_data_t data; +} txindex; + +typedef struct outkey { + uint64_t amount_index; + uint64_t output_id; + output_data_t data; +} outkey; + +typedef struct outtx { + uint64_t output_id; + crypto::hash tx_hash; + uint64_t local_index; +} outtx; + std::atomic mdb_txn_safe::num_active_txns{0}; std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT; @@ -539,70 +586,55 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const mdb_txn_cursors *m_cursors = &m_wcursors; CURSOR(block_heights) - MDB_val_copy val_h(blk_hash); - if (mdb_cursor_get(m_cur_block_heights, &val_h, NULL, MDB_SET) == 0) + blk_height bh = {blk_hash, m_height}; + MDB_val_set(val_h, bh); + if (mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); if (m_height > 0) { - MDB_val_copy parent_key(blk.prev_id); - MDB_val parent_h; - int result = mdb_cursor_get(m_cur_block_heights, &parent_key, &parent_h, MDB_SET); + MDB_val_set(parent_key, blk.prev_id); + int result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &parent_key, MDB_GET_BOTH); if (result) { LOG_PRINT_L3("m_height: " << m_height); LOG_PRINT_L3("parent_key: " << blk.prev_id); throw0(DB_ERROR(lmdb_error("Failed to get top block hash to check for new block's parent: ", result).c_str())); } - uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; - if (parent_height != m_height - 1) + blk_height *prev = (blk_height *)parent_key.mv_data; + if (prev->bh_height != m_height - 1) throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); } int result = 0; - MDB_val_copy key(m_height); + MDB_val_set(key, m_height); CURSOR(blocks) - CURSOR(block_sizes) - CURSOR(block_timestamps) - CURSOR(block_diffs) - CURSOR(block_coins) - CURSOR(block_hashes) + CURSOR(block_info) MDB_val_copy blob(block_to_blob(blk)); result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block blob to db transaction: ", result).c_str())); - MDB_val_copy sz(block_size); - result = mdb_cursor_put(m_cur_block_sizes, &key, &sz, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block size to db transaction: ", result).c_str())); + mdb_block_info bi; + bi.bi_height = m_height; + bi.bi_timestamp = blk.timestamp; + bi.bi_coins = coins_generated; + bi.bi_size = block_size; + bi.bi_diff = cumulative_difficulty; + bi.bi_hash = blk_hash; - MDB_val_copy ts(blk.timestamp); - result = mdb_cursor_put(m_cur_block_timestamps, &key, &ts, MDB_APPEND); + MDB_val_set(val, bi); + result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP); if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block timestamp to db transaction: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add block info to db transaction: ", result).c_str())); - MDB_val_copy diff(cumulative_difficulty); - result = mdb_cursor_put(m_cur_block_diffs, &key, &diff, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block cumulative difficulty to db transaction: ", result).c_str())); - - MDB_val_copy coinsgen(coins_generated); - result = mdb_cursor_put(m_cur_block_coins, &key, &coinsgen, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block total generated coins to db transaction: ", result).c_str())); - - result = mdb_cursor_put(m_cur_block_heights, &val_h, &key, 0); + result = mdb_cursor_put(m_cur_block_heights, (MDB_val *)&zerokval, &val_h, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str())); - result = mdb_cursor_put(m_cur_block_hashes, &key, &val_h, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block hash to db transaction: ", result).c_str())); - m_cum_size += block_size; m_cum_count++; } @@ -617,66 +649,80 @@ void BlockchainLMDB::remove_block() if (m_height == 0) throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(block_info) + CURSOR(block_heights) + CURSOR(blocks) MDB_val_copy k(m_height - 1); - MDB_val h; - if ((result = mdb_get(*m_write_txn, m_block_hashes, &k, &h))) + MDB_val h = k; + if ((result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &h, MDB_GET_BOTH))) throw1(BLOCK_DNE(lmdb_error("Attempting to remove block that's not in the db: ", result).c_str())); - if ((result = mdb_del(*m_write_txn, m_blocks, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str())); - - if ((result = mdb_del(*m_write_txn, m_block_sizes, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block size to db transaction: ", result).c_str())); - - if ((result = mdb_del(*m_write_txn, m_block_diffs, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block cumulative difficulty to db transaction: ", result).c_str())); - - if ((result = mdb_del(*m_write_txn, m_block_coins, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block total generated coins to db transaction: ", result).c_str())); - - if ((result = mdb_del(*m_write_txn, m_block_timestamps, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block timestamp to db transaction: ", result).c_str())); - - if ((result = mdb_del(*m_write_txn, m_block_heights, &h, NULL))) + // must use h now; deleting from m_block_info will invalidate it + mdb_block_info *bi = (mdb_block_info *)h.mv_data; + blk_height bh = {bi->bi_hash, 0}; + h.mv_data = (void *)&bh; + h.mv_size = sizeof(bh); + if ((result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &h, MDB_GET_BOTH))) + throw1(DB_ERROR(lmdb_error("Failed to locate block height by hash for removal: ", result).c_str())); + if ((result = mdb_cursor_del(m_cur_block_heights, 0))) throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str())); - if ((result = mdb_del(*m_write_txn, m_block_hashes, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block hash to db transaction: ", result).c_str())); + if ((result = mdb_cursor_get(m_cur_blocks, &k, NULL, MDB_SET))) + throw1(DB_ERROR(lmdb_error("Failed to locate block for removal: ", result).c_str())); + if ((result = mdb_cursor_del(m_cur_blocks, 0))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str())); + + if ((result = mdb_cursor_del(m_cur_block_info, 0))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str())); } -void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); mdb_txn_cursors *m_cursors = &m_wcursors; - int result = 0; + int result; + uint64_t tx_id = m_num_txs; CURSOR(txs) - CURSOR(tx_heights) - CURSOR(tx_unlocks) + CURSOR(tx_indices) - MDB_val_copy val_h(tx_hash); - MDB_val unused; - if (mdb_cursor_get(m_cur_txs, &val_h, &unused, MDB_SET) == 0) - throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); + MDB_val_set(val_tx_id, tx_id); + MDB_val_set(val_h, tx_hash); + result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH); + if (result == 0) { + txindex *tip = (txindex *)val_h.mv_data; + throw1(TX_EXISTS(std::string("Attempting to add transaction that's already in the db (tx id ").append(boost::lexical_cast(tip->data.tx_id)).append(")").c_str())); + } else if (result != MDB_NOTFOUND) { + throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str())); + } + + txindex ti; + ti.key = tx_hash; + ti.data.tx_id = tx_id; + ti.data.unlock_time = tx.unlock_time; + ti.data.block_id = m_height; // we don't need blk_hash since we know m_height + + val_h.mv_size = sizeof(ti); + val_h.mv_data = (void *)&ti; + + result = mdb_cursor_put(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); MDB_val_copy blob(tx_to_blob(tx)); - result = mdb_cursor_put(m_cur_txs, &val_h, &blob, 0); + result = mdb_cursor_put(m_cur_txs, &val_tx_id, &blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str())); - MDB_val_copy height(m_height); - result = mdb_cursor_put(m_cur_tx_heights, &val_h, &height, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx block height to db transaction: ", result).c_str())); - - MDB_val_copy unlock_time(tx.unlock_time); - result = mdb_cursor_put(m_cur_tx_unlocks, &val_h, &unlock_time, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx unlock time to db transaction: ", result).c_str())); + m_num_txs++; + return tx_id; } +// TODO: compare pros and cons of looking up the tx hash's tx index once and +// passing it in to functions like this void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) { int result; @@ -684,28 +730,49 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val val_h = {sizeof(tx_hash), (void *)&tx_hash}; - MDB_val unused; - if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(tx_indices) + CURSOR(txs) + CURSOR(tx_outputs) + + MDB_val_set(val_h, tx_hash); + + if (mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + txindex *tip = (txindex *)val_h.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); - if ((result = mdb_del(*m_write_txn, m_txs, &val_h, NULL))) + if ((result = mdb_cursor_get(m_cur_txs, &val_tx_id, NULL, MDB_SET))) + throw1(DB_ERROR(lmdb_error("Failed to locate tx for removal: ", result).c_str())); + result = mdb_cursor_del(m_cur_txs, 0); + if (result) throw1(DB_ERROR(lmdb_error("Failed to add removal of tx to db transaction: ", result).c_str())); - if ((result = mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of tx unlock time to db transaction: ", result).c_str())); - if ((result = mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of tx block height to db transaction: ", result).c_str())); - remove_tx_outputs(&val_h, tx); + remove_tx_outputs(tip->data.tx_id, tx); - result = mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL); + result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_id, NULL, MDB_SET); if (result == MDB_NOTFOUND) LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash); else if (result) - throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str())); + throw1(DB_ERROR(lmdb_error("Failed to locate tx outputs for removal: ", result).c_str())); + if (!result) + { + result = mdb_cursor_del(m_cur_tx_outputs, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str())); + } + + // Don't delete the tx_indices entry until the end, after we're done with val_tx_id + if (mdb_cursor_del(m_cur_tx_indices, 0)) + throw1(DB_ERROR("Failed to add removal of tx index to db transaction")); + + m_num_txs--; } -void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) +uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash, + const tx_out& tx_output, + const uint64_t& local_index, + const uint64_t unlock_time) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -714,189 +781,132 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou int result = 0; CURSOR(output_txs) - CURSOR(tx_outputs) - CURSOR(output_indices) CURSOR(output_amounts) - CURSOR(output_keys) - MDB_val_copy k(m_num_outputs); - MDB_val_copy v(tx_hash); + if (tx_output.target.type() != typeid(txout_to_key)) + throw0(DB_ERROR("Wrong output type: expected txout_to_key")); - result = mdb_cursor_put(m_cur_output_txs, &k, &v, MDB_APPEND); + outtx ot = {m_num_outputs, tx_hash, local_index}; + MDB_val_set(vot, ot); + + result = mdb_cursor_put(m_cur_output_txs, (MDB_val *)&zerokval, &vot, MDB_APPENDDUP); if (result) throw0(DB_ERROR(lmdb_error("Failed to add output tx hash to db transaction: ", result).c_str())); - result = mdb_cursor_put(m_cur_tx_outputs, &v, &k, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add to db transaction: ", result).c_str())); - - MDB_val_copy val_local_index(local_index); - result = mdb_cursor_put(m_cur_output_indices, &k, &val_local_index, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx output index to db transaction: ", result).c_str())); + outkey ok; + MDB_val data; MDB_val_copy val_amount(tx_output.amount); - result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &k, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add output amount to db transaction: ", result).c_str())); - - if (tx_output.target.type() == typeid(txout_to_key)) - { - output_data_t od; - od.pubkey = boost::get < txout_to_key > (tx_output.target).key; - od.unlock_time = unlock_time; - od.height = m_height; - - MDB_val_copy data(od); - //MDB_val_copy val_pubkey(boost::get(tx_output.target).key); - if ((result = mdb_cursor_put(m_cur_output_keys, &k, &data, MDB_APPEND))) - throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str())); - } + result = mdb_cursor_get(m_cur_output_amounts, &val_amount, &data, MDB_SET); + if (!result) + { + mdb_size_t num_elems = 0; + result = mdb_cursor_count(m_cur_output_amounts, &num_elems); + if (result) + throw0(DB_ERROR(std::string("Failed to get number of outputs for amount: ").append(mdb_strerror(result)).c_str())); + ok.amount_index = num_elems; + } + else if (result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Failed to get output amount in db transaction: ", result).c_str())); else - { - throw0(DB_ERROR("Wrong output type: expected txout_to_key")); - } + ok.amount_index = 0; + ok.output_id = m_num_outputs; + ok.data.pubkey = boost::get < txout_to_key > (tx_output.target).key; + ok.data.unlock_time = unlock_time; + ok.data.height = m_height; + data.mv_data = &ok; + data.mv_size = sizeof(ok); + + if ((result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &data, MDB_APPENDDUP))) + throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str())); m_num_outputs++; + return ok.amount_index; } -void BlockchainLMDB::remove_tx_outputs(const MDB_val *tx_hash, const transaction& tx) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - - mdb_txn_cursors *m_cursors = &m_wcursors; - MDB_val v; - CURSOR(tx_outputs) - - auto result = mdb_cursor_get(m_cur_tx_outputs, (MDB_val *)tx_hash, &v, MDB_SET); - if (result == MDB_NOTFOUND) - { - LOG_PRINT_L2("tx has no outputs, so no global output indices"); - } - else if (result) - { - throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - } - else - { - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_tx_outputs, &num_elems); - - mdb_cursor_get(m_cur_tx_outputs, (MDB_val *)tx_hash, &v, MDB_LAST_DUP); - - for (uint64_t i = num_elems; i > 0; --i) - { - const tx_out tx_output = tx.vout[i-1]; -#ifndef MISALIGNED_OK - uint64_t tv; - memcpy(&tv, v.mv_data, sizeof(uint64_t)); - v.mv_data = &tv; -#endif - remove_output((const MDB_val *)&v, tx_output.amount); - if (i > 1) - { - mdb_cursor_get(m_cur_tx_outputs, (MDB_val *)tx_hash, &v, MDB_PREV_DUP); - } - } - } -} - -// TODO: probably remove this function -void BlockchainLMDB::remove_output(const tx_out& tx_output) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); - return; -} - -void BlockchainLMDB::remove_output(const MDB_val *out_index, const uint64_t amount) +void BlockchainLMDB::add_tx_amount_output_indices(const uint64_t tx_id, + const std::vector& amount_output_indices) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(tx_outputs) - auto result = mdb_del(*m_write_txn, m_output_indices, (MDB_val *)out_index, NULL); + int result = 0; + + int num_outputs = amount_output_indices.size(); + + MDB_val_set(k_tx_id, tx_id); + MDB_val v; + v.mv_data = (void *)amount_output_indices.data(); + v.mv_size = sizeof(uint64_t) * num_outputs; + // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); + + result = mdb_cursor_put(m_cur_tx_outputs, &k_tx_id, &v, MDB_APPEND); + if (result) + throw0(DB_ERROR(std::string("Failed to add to db transaction: ").append(mdb_strerror(result)).c_str())); +} + +void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + std::vector amount_output_indices = get_tx_amount_output_indices(tx_id); + + if (amount_output_indices.empty()) + { + if (tx.vout.empty()) + LOG_PRINT_L2("tx has no outputs, so no output indices"); + else + throw0(DB_ERROR("tx has outputs, but no output indices found")); + } + + for (uint64_t i = tx.vout.size(); i > 0; --i) + { + const tx_out tx_output = tx.vout[i-1]; + remove_output(tx_output.amount, amount_output_indices[i-1]); + } +} + +void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_index) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(output_amounts); + CURSOR(output_txs); + + MDB_val_set(k, amount); + MDB_val_set(v, out_index); + + auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); if (result == MDB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); - } + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) - { - throw1(DB_ERROR(lmdb_error("Error adding removal of output tx index to db transaction", result).c_str())); - } + throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - result = mdb_del(*m_write_txn, m_output_txs, (MDB_val *)out_index, NULL); - // if (result != 0 && result != MDB_NOTFOUND) - // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + outkey *ok = (outkey *)v.mv_data; + MDB_val_set(otxk, ok->output_id); + result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &otxk, MDB_GET_BOTH); if (result == MDB_NOTFOUND) { LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); } else if (result) { - throw1(DB_ERROR(lmdb_error("Error adding removal of output tx hash to db transaction", result).c_str())); + throw1(DB_ERROR(lmdb_error("Error adding removal of output tx to db transaction", result).c_str())); } + result = mdb_cursor_del(m_cur_output_txs, 0); + if (result) + throw0(DB_ERROR(lmdb_error(std::string("Error deleting output index ").append(boost::lexical_cast(out_index).append(": ")).c_str(), result).c_str())); - result = mdb_del(*m_write_txn, m_output_keys, (MDB_val *)out_index, NULL); - if (result == MDB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); - } - else if (result) - throw1(DB_ERROR(lmdb_error("Error adding removal of output pubkey to db transaction", result).c_str())); - - remove_amount_output_index(amount, out_index); + // now delete the amount + result = mdb_cursor_del(m_cur_output_amounts, 0); + if (result) + throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast(out_index).append(": ")).c_str(), result).c_str())); m_num_outputs--; } -void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const MDB_val *global_output_index) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - mdb_txn_cursors *m_cursors = &m_wcursors; - CURSOR(output_amounts); - - MDB_val_copy k(amount); - MDB_val v; - - auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_output_amounts, &num_elems); - - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_LAST_DUP); - - uint64_t amount_output_index = 0; - bool found_index = false; - for (uint64_t i = num_elems; i > 0; --i) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); - if (!memcmp(v.mv_data, global_output_index->mv_data, sizeof(uint64_t))) - { - amount_output_index = i-1; - found_index = true; - break; - } - if (i > 1) - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV_DUP); - } - if (found_index) - { - // found the amount output index - // now delete it - int result = mdb_cursor_del(m_cur_output_amounts, 0); - if (result) - throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount output index ").append(boost::lexical_cast(amount_output_index).append(": ")).c_str(), result).c_str())); - } - else - { - // not found - throw1(OUTPUT_DNE("Failed to find amount output index")); - } -} - void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -905,27 +915,33 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) CURSOR(spent_keys) - MDB_val_copy val_key(k_image); - MDB_val unused; - if (mdb_cursor_get(m_cur_spent_keys, &val_key, &unused, MDB_SET) == 0) + MDB_val k = {sizeof(k_image), (void *)&k_image}; + if (auto result = mdb_cursor_put(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_NODUPDATA)) { + if (result == MDB_KEYEXIST) throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); - - char anything = '\0'; - unused.mv_size = sizeof(char); - unused.mv_data = &anything; - if (auto result = mdb_cursor_put(m_cur_spent_keys, &val_key, &unused, 0)) - throw1(DB_ERROR(lmdb_error("Error adding spent key image to db transaction: ", result).c_str())); + else + throw1(DB_ERROR(lmdb_error("Error adding spent key image to db transaction: ", result).c_str())); + } } void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; - MDB_val_copy k(k_image); - auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL); + CURSOR(spent_keys) + + MDB_val k = {sizeof(k_image), (void *)&k_image}; + auto result = mdb_cursor_get(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_GET_BOTH); if (result != 0 && result != MDB_NOTFOUND) - throw1(DB_ERROR(lmdb_error("Error adding removal of key image to db transaction", result).c_str())); + throw1(DB_ERROR(lmdb_error("Error finding spent key to remove", result).c_str())); + if (!result) + { + result = mdb_cursor_del(m_cur_spent_keys, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error adding removal of key image to db transaction", result).c_str())); + } } blobdata BlockchainLMDB::output_to_blob(const tx_out& output) const @@ -951,19 +967,6 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const return o; } -uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - std::vector offsets; - std::vector global_indices; - offsets.push_back(index); - get_output_global_indices(amount, offsets, global_indices); - if (!global_indices.size()) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - - return global_indices[0]; -} - void BlockchainLMDB::check_open() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1076,39 +1079,31 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // uses macros to avoid having to change things too many places lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); - lmdb_db_open(txn, LMDB_BLOCK_TIMESTAMPS, MDB_INTEGERKEY | MDB_CREATE, m_block_timestamps, "Failed to open db handle for m_block_timestamps"); - lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights"); - lmdb_db_open(txn, LMDB_BLOCK_HASHES, MDB_INTEGERKEY | MDB_CREATE, m_block_hashes, "Failed to open db handle for m_block_hashes"); - lmdb_db_open(txn, LMDB_BLOCK_SIZES, MDB_INTEGERKEY | MDB_CREATE, m_block_sizes, "Failed to open db handle for m_block_sizes"); - lmdb_db_open(txn, LMDB_BLOCK_DIFFS, MDB_INTEGERKEY | MDB_CREATE, m_block_diffs, "Failed to open db handle for m_block_diffs"); - lmdb_db_open(txn, LMDB_BLOCK_COINS, MDB_INTEGERKEY | MDB_CREATE, m_block_coins, "Failed to open db handle for m_block_coins"); + lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for m_block_info"); + lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for m_block_heights"); - lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); - lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); - lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); - lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); + lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); + lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); - lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); - lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); + lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_txs, "Failed to open db handle for m_output_txs"); lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); - lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys"); - lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys"); + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); lmdb_db_open(txn, LMDB_HF_STARTING_HEIGHTS, MDB_CREATE, m_hf_starting_heights, "Failed to open db handle for m_hf_starting_heights"); - lmdb_db_open(txn, LMDB_HF_VERSIONS, MDB_CREATE, m_hf_versions, "Failed to open db handle for m_hf_versions"); + lmdb_db_open(txn, LMDB_HF_VERSIONS, MDB_INTEGERKEY | MDB_CREATE, m_hf_versions, "Failed to open db handle for m_hf_versions"); lmdb_db_open(txn, LMDB_PROPERTIES, MDB_CREATE, m_properties, "Failed to open db handle for m_properties"); + mdb_set_dupsort(txn, m_spent_keys, compare_hash32); + mdb_set_dupsort(txn, m_block_heights, compare_hash32); + mdb_set_dupsort(txn, m_tx_indices, compare_hash32); mdb_set_dupsort(txn, m_output_amounts, compare_uint64); - mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); - mdb_set_compare(txn, m_spent_keys, compare_hash32); - mdb_set_compare(txn, m_block_heights, compare_hash32); - mdb_set_compare(txn, m_txs, compare_hash32); - mdb_set_compare(txn, m_tx_unlocks, compare_hash32); - mdb_set_compare(txn, m_tx_heights, compare_hash32); + mdb_set_dupsort(txn, m_output_txs, compare_uint64); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); - mdb_set_compare(txn, m_hf_versions, compare_uint64); mdb_set_compare(txn, m_properties, compare_string); // get and keep current height @@ -1118,34 +1113,18 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); m_height = db_stats.ms_entries; + // get and keep current number of txs + if ((result = mdb_stat(txn, m_txs, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str())); + m_num_txs = db_stats.ms_entries; + // get and keep current number of outputs - if ((result = mdb_stat(txn, m_output_indices, &db_stats))) - throw0(DB_ERROR(lmdb_error("Failed to query m_output_indices: ", result).c_str())); + if ((result = mdb_stat(txn, m_output_txs, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str())); m_num_outputs = db_stats.ms_entries; bool compatible = true; - // ND: This "new" version of the lmdb database is incompatible with - // the previous version. Ensure that the output_keys database is - // sizeof(output_data_t) in length. Otherwise, inform user and - // terminate. - if(m_height > 0) - { - MDB_val_copy k(0); - MDB_val v; - auto get_result = mdb_get(txn, m_output_keys, &k, &v); - if(get_result != MDB_SUCCESS) - { - txn.abort(); - m_open = false; - return; - } - - // LOG_PRINT_L0("Output keys size: " << v.mv_size); - if(v.mv_size != sizeof(output_data_t)) - compatible = false; - } - MDB_val_copy k("version"); MDB_val v; auto get_result = mdb_get(txn, m_properties, &k, &v); @@ -1159,13 +1138,21 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) #if VERSION > 0 else if (*(const uint32_t*)v.mv_data < VERSION) { - compatible = false; + // Note that there was a schema change within version 0 as well. + // See commit e5d2680094ee15889934fe28901e4e133cda56f2 2015/07/10 + // We don't handle the old format previous to that commit. + txn.commit(); + m_open = true; + migrate(*(const uint32_t *)v.mv_data); + return; } #endif } else { - // if not found, but we're on version 0, it's fine. If the DB's empty, it's fine too. + // if not found, and the DB is non-empty, this is probably + // an "old" version 0, which we don't handle. If the DB is + // empty it's fine. if (VERSION > 0 && m_height > 0) compatible = false; } @@ -1244,20 +1231,12 @@ void BlockchainLMDB::reset() if (auto result = mdb_txn_begin(m_env, NULL, 0, txn)) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); mdb_drop(txn, m_blocks, 0); - mdb_drop(txn, m_block_timestamps, 0); + mdb_drop(txn, m_block_info, 0); mdb_drop(txn, m_block_heights, 0); - mdb_drop(txn, m_block_hashes, 0); - mdb_drop(txn, m_block_sizes, 0); - mdb_drop(txn, m_block_diffs, 0); - mdb_drop(txn, m_block_coins, 0); mdb_drop(txn, m_txs, 0); - mdb_drop(txn, m_tx_unlocks, 0); - mdb_drop(txn, m_tx_heights, 0); mdb_drop(txn, m_tx_outputs, 0); mdb_drop(txn, m_output_txs, 0); - mdb_drop(txn, m_output_indices, 0); mdb_drop(txn, m_output_amounts, 0); - mdb_drop(txn, m_output_keys, 0); mdb_drop(txn, m_spent_keys, 0); mdb_drop(txn, m_hf_starting_heights, 0); mdb_drop(txn, m_hf_versions, 0); @@ -1367,8 +1346,8 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const RCURSOR(block_heights); bool ret = false; - MDB_val_copy key(h); - auto get_result = mdb_cursor_get(m_cur_block_heights, &key, NULL, MDB_SET); + MDB_val_set(key, h); + auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); @@ -1398,15 +1377,15 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const TXN_PREFIX_RDONLY(); RCURSOR(block_heights); - MDB_val_copy key(h); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_heights, &key, &result, MDB_SET); + MDB_val_set(key, h); + auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); - uint64_t ret = *(const uint64_t *)result.mv_data; + blk_height *bhp = (blk_height *)key.mv_data; + uint64_t ret = bhp->bh_height; TXN_POSTFIX_RDONLY(); return ret; } @@ -1456,11 +1435,10 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(block_timestamps); + RCURSOR(block_info); - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_timestamps, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast(height)).append(" failed -- timestamp not in db").c_str())); @@ -1468,7 +1446,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); - uint64_t ret = *(const uint64_t *)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + uint64_t ret = bi->bi_timestamp; TXN_POSTFIX_RDONLY(); return ret; } @@ -1493,11 +1472,10 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(block_sizes); + RCURSOR(block_info); - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_sizes, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get block size from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); @@ -1505,7 +1483,8 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); - size_t ret = *(const size_t *)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + size_t ret = bi->bi_size; TXN_POSTFIX_RDONLY(); return ret; } @@ -1516,11 +1495,10 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(block_diffs); + RCURSOR(block_info); - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_diffs, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); @@ -1528,7 +1506,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); - difficulty_type ret = *(const difficulty_type*)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + difficulty_type ret = bi->bi_diff; TXN_POSTFIX_RDONLY(); return ret; } @@ -1556,11 +1535,10 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(block_coins); + RCURSOR(block_info); - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_coins, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); @@ -1568,7 +1546,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); - uint64_t ret = *(const uint64_t*)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + uint64_t ret = bi->bi_coins; TXN_POSTFIX_RDONLY(); return ret; } @@ -1579,11 +1558,10 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(block_hashes); + RCURSOR(block_info); - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_hashes, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); @@ -1591,7 +1569,8 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) else if (get_result) throw0(DB_ERROR(lmdb_error("Error attempting to retrieve a block hash from the db: ", get_result).c_str())); - crypto::hash ret = *(const crypto::hash*)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + crypto::hash ret = bi->bi_hash; TXN_POSTFIX_RDONLY(); return ret; } @@ -1664,18 +1643,61 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); RCURSOR(txs); - MDB_val_copy key(h); - MDB_val result; + MDB_val_set(key, h); + bool tx_found = false; TIME_MEASURE_START(time1); - auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); + if (get_result == 0) + tx_found = true; + else if (get_result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", get_result).c_str())); + + // This isn't needed as part of the check. we're not checking consistency of db. + // get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); TIME_MEASURE_FINISH(time1); time_tx_exists += time1; TXN_POSTFIX_RDONLY(); + if (! tx_found) + { + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + + // Below not needed due to above comment. + // if (get_result == MDB_NOTFOUND) + // throw0(DB_ERROR(std::string("transaction with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found at index").c_str())); + // else if (get_result) + // throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction ") + epee::string_tools::pod_to_hex(h) + " at index: ", get_result).c_str())); + return true; +} + +bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_id) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); + + MDB_val_set(v, h); + + TIME_MEASURE_START(time1); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; + if (!get_result) { + txindex *tip = (txindex *)v.mv_data; + tx_id = tip->data.tx_id; + } + + TXN_POSTFIX_RDONLY(); + bool ret = false; if (get_result == MDB_NOTFOUND) { @@ -1695,17 +1717,17 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(tx_unlocks); + RCURSOR(tx_indices); - MDB_val_copy key(h); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_unlocks, &key, &result, MDB_SET); + MDB_val_set(v, h); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + throw1(TX_DNE(lmdb_error(std::string("tx data with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str())); else if (get_result) - throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx unlock time from hash", get_result).c_str())); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx data from hash: ", get_result).c_str())); - uint64_t ret = *(const uint64_t*)result.mv_data; + txindex *tip = (txindex *)v.mv_data; + uint64_t ret = tip->data.unlock_time; TXN_POSTFIX_RDONLY(); return ret; } @@ -1716,11 +1738,18 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); RCURSOR(txs); - MDB_val_copy key(h); + MDB_val_set(v, h); MDB_val result; - auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (get_result == 0) + { + txindex *tip = (txindex *)v.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + get_result = mdb_cursor_get(m_cur_txs, &val_tx_id, &result, MDB_SET); + } if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) @@ -1746,8 +1775,8 @@ uint64_t BlockchainLMDB::get_tx_count() const TXN_PREFIX_RDONLY(); MDB_stat db_stats; - if (mdb_stat(m_txn, m_txs, &db_stats)) - throw0(DB_ERROR("Failed to query m_txs")); + if (mdb_stat(m_txn, m_tx_indices, &db_stats)) + throw0(DB_ERROR("Failed to query m_tx_indices")); TXN_POSTFIX_RDONLY(); @@ -1774,19 +1803,19 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(tx_heights); + RCURSOR(tx_indices); - MDB_val_copy key(h); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_heights, &key, &result, MDB_SET); + MDB_val_set(v, h); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { - throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + throw1(TX_DNE(std::string("tx_data_t with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); } else if (get_result) throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx height from hash", get_result).c_str())); - uint64_t ret = *(const uint64_t*)result.mv_data; + txindex *tip = (txindex *)v.mv_data; + uint64_t ret = tip->data.block_id; TXN_POSTFIX_RDONLY(); return ret; } @@ -1815,24 +1844,54 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } +// This is a lot harder now that we've removed the output_keys index output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); check_open(); - TXN_PREFIX_RDONLY(); - RCURSOR(output_keys); + RCURSOR(output_txs); + RCURSOR(tx_indices); + RCURSOR(txs); - MDB_val_copy k(global_index); - MDB_val v; - auto get_result = mdb_cursor_get(m_cur_output_keys, &k, &v, MDB_SET); + output_data_t od; + MDB_val_set(v, global_index); + auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); + throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); - output_data_t ret = *(const output_data_t *) v.mv_data; + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + outtx *ot = (outtx *)v.mv_data; + + MDB_val_set(val_h, ot->tx_hash); + get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH); + if (get_result) + throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(ot->tx_hash) + ": ", get_result).c_str())); + + txindex *tip = (txindex *)val_h.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + MDB_val result; + get_result = mdb_cursor_get(m_cur_txs, &val_tx_id, &result, MDB_SET); + if (get_result == MDB_NOTFOUND) + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(ot->tx_hash)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); + + blobdata bd; + bd.assign(reinterpret_cast(result.mv_data), result.mv_size); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + + const tx_out tx_output = tx.vout[ot->local_index]; + od.unlock_time = tip->data.unlock_time; + od.height = tip->data.block_id; + od.pubkey = boost::get(tx_output.target).key; + TXN_POSTFIX_RDONLY(); - return ret; + return od; } output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) @@ -1840,37 +1899,41 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6 LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - uint64_t glob_index = get_output_global_index(amount, index); - return get_output_key(glob_index); + TXN_PREFIX_RDONLY(); + RCURSOR(output_amounts); + + MDB_val_set(k, amount); + MDB_val_set(v, index); + auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get output pubkey by index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + outkey *okp = (outkey *)v.mv_data; + output_data_t ret = okp->data; + TXN_POSTFIX_RDONLY(); + return ret; } -tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& index) const +tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& output_id) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); TXN_PREFIX_RDONLY(); RCURSOR(output_txs); - RCURSOR(output_indices); - MDB_val_copy k(index); - MDB_val v; + MDB_val_set(v, output_id); - auto get_result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - crypto::hash tx_hash = *(const crypto::hash*)v.mv_data; + outtx *ot = (outtx *)v.mv_data; + tx_out_index ret = tx_out_index(ot->tx_hash, ot->local_index); - get_result = mdb_cursor_get(m_cur_output_indices, &k, &v, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx index")); - - tx_out_index ret = tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); TXN_POSTFIX_RDONLY(); return ret; } @@ -1888,128 +1951,55 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con return indices[0]; } -std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const +std::vector BlockchainLMDB::get_tx_amount_output_indices(const uint64_t tx_id) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); - std::vector index_vec; TXN_PREFIX_RDONLY(); RCURSOR(tx_outputs); - MDB_val_copy k(h); + int result = 0; + MDB_val_set(k_tx_id, tx_id); MDB_val v; - auto result = mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_SET); + std::vector amount_output_indices; + + result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, MDB_SET); if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " + "tx_outputs, but it should have an empty entry even if it's a tx without " + "outputs"); else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); + throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str())); - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_tx_outputs, &num_elems); + uint64_t* indices = (uint64_t*)v.mv_data; + int num_outputs = v.mv_size / sizeof(uint64_t); - mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_FIRST_DUP); - - for (uint64_t i = 0; i < num_elems; ++i) + for (int i = 0; i < num_outputs; ++i) { - mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_GET_CURRENT); - index_vec.push_back(*(const uint64_t *)v.mv_data); - mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_NEXT_DUP); + // LOG_PRINT_L0("amount output index[" << 2*i << "]" << ": " << paired_indices[2*i] << " global output index: " << paired_indices[2*i+1]); + amount_output_indices.push_back(indices[i]); } + indices = nullptr; TXN_POSTFIX_RDONLY(); - - return index_vec; + return amount_output_indices; } -std::vector BlockchainLMDB::get_tx_amount_output_indices(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - std::vector index_vec; - std::vector index_vec2; - - // get the transaction's global output indices first - index_vec = get_tx_output_indices(h); - // these are next used to obtain the amount output indices - - transaction tx = get_tx(h); - - TXN_PREFIX_RDONLY(); - RCURSOR(output_amounts); - - uint64_t i = 0; - uint64_t global_index; - BOOST_FOREACH(const auto& vout, tx.vout) - { - uint64_t amount = vout.amount; - - global_index = index_vec[i]; - - MDB_val_copy k(amount); - MDB_val v; - - auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_output_amounts, &num_elems); - - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_FIRST_DUP); - - uint64_t amount_output_index = 0; - uint64_t output_index = 0; - bool found_index = false; - for (uint64_t j = 0; j < num_elems; ++j) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); - output_index = *(const uint64_t *)v.mv_data; - if (output_index == global_index) - { - amount_output_index = j; - found_index = true; - break; - } - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP); - } - if (found_index) - { - index_vec2.push_back(amount_output_index); - } - else - { - // not found - TXN_POSTFIX_RDONLY(); - throw1(OUTPUT_DNE("specified output not found in db")); - } - - ++i; - } - - TXN_POSTFIX_RDONLY(); - - return index_vec2; -} - - bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + bool ret; + TXN_PREFIX_RDONLY(); RCURSOR(spent_keys); - bool ret = false; - MDB_val_copy val_key(img); - if (mdb_cursor_get(m_cur_spent_keys, &val_key, NULL, MDB_SET) == 0) - { - ret = true; - } + MDB_val k = {sizeof(img), (void *)&img}; + ret = (mdb_cursor_get(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_GET_BOTH) == 0); TXN_POSTFIX_RDONLY(); return ret; @@ -2023,10 +2013,10 @@ bool BlockchainLMDB::for_all_key_images(std::functionkey; + k.mv_data = (void *)&ti->data.tx_id; + k.mv_size = sizeof(ti->data.tx_id); + ret = mdb_cursor_get(m_cur_txs, &k, &v, MDB_SET); + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); blobdata bd; bd.assign(reinterpret_cast(v.mv_data), v.mv_size); transaction tx; @@ -2149,8 +2149,8 @@ bool BlockchainLMDB::for_all_outputs(std::functionoutput_id); if (!f(amount, toi.first, toi.second)) { ret = false; break; @@ -2269,6 +2269,10 @@ void BlockchainLMDB::batch_abort() void BlockchainLMDB::set_batch_transactions(bool batch_transactions) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if ((batch_transactions) && (m_batch_transactions)) + { + LOG_PRINT_L0("WARNING: batch transaction mode already enabled, but asked to enable batch mode"); + } m_batch_transactions = batch_transactions; LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } @@ -2433,6 +2437,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c } } + uint64_t num_txs = m_num_txs; uint64_t num_outputs = m_num_outputs; try { @@ -2444,6 +2449,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c } catch (...) { + m_num_txs = num_txs; m_num_outputs = num_outputs; block_txn_abort(); throw; @@ -2459,6 +2465,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) block_txn_start(false); + uint64_t num_txs = m_num_txs; uint64_t num_outputs = m_num_outputs; try { @@ -2467,6 +2474,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) } catch (...) { + m_num_txs = num_txs; m_num_outputs = num_outputs; block_txn_abort(); throw; @@ -2484,161 +2492,25 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector k(index); - MDB_val v; + MDB_val_set(v, output_id); - auto get_result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - crypto::hash tx_hash = *(const crypto::hash*) v.mv_data; - - get_result = mdb_cursor_get(m_cur_output_indices, &k, &v, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx index")); - - auto result = tx_out_index(tx_hash, *(const uint64_t *) v.mv_data); + outtx *ot = (outtx *)v.mv_data; + auto result = tx_out_index(ot->tx_hash, ot->local_index); tx_out_indices.push_back(result); } TXN_POSTFIX_RDONLY(); } -void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std::vector &offsets, - std::vector &global_indices) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - TIME_MEASURE_START(txx); - check_open(); - global_indices.clear(); - - uint64_t max = 0; - for (const uint64_t &index : offsets) - { - if (index > max) - max = index; - } - - TXN_PREFIX_RDONLY(); - RCURSOR(output_amounts); - - MDB_val_copy k(amount); - MDB_val v; - auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_output_amounts, &num_elems); - if (max <= 1 && num_elems <= max) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - - uint64_t t_dbmul = 0; - uint64_t t_dbscan = 0; - if (max <= 1) - { - for (const uint64_t& index : offsets) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_FIRST_DUP); - for (uint64_t i = 0; i < index; ++i) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP); - } - - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); - uint64_t glob_index = *(const uint64_t*) v.mv_data; - LOG_PRINT_L3("Amount: " << amount << " M0->v: " << glob_index); - global_indices.push_back(glob_index); - } - } - else - { - uint32_t curcount = 0; - uint32_t blockstart = 0; - for (const uint64_t& index : offsets) - { - if (index >= num_elems) - { - LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index"); - break; - } - if (!curcount && index > num_elems/2) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_LAST_DUP); - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV); /* kludge to unset C_EOF */ - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT); - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_MULTIPLE); - - curcount = num_elems; - while(1) - { - TIME_MEASURE_START(db1); - int count = v.mv_size / sizeof(uint64_t); - curcount -= count; - if (curcount > index) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV_MULTIPLE); - } else - { - blockstart = curcount; - curcount += count; - break; - } - TIME_MEASURE_FINISH(db1); - t_dbmul += db1; - } - - } else - { - while (index >= curcount) - { - TIME_MEASURE_START(db1); - if (mdb_cursor_get(m_cur_output_amounts, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0) - { - // allow partial results - result = false; - break; - } - - int count = v.mv_size / sizeof(uint64_t); - - blockstart = curcount; - curcount += count; - TIME_MEASURE_FINISH(db1); - t_dbmul += db1; - } - } - - LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index); - TIME_MEASURE_START(db2); - uint64_t actual_index = index - blockstart; - uint64_t glob_index = ((const uint64_t*) v.mv_data)[actual_index]; - - LOG_PRINT_L3("Amount: " << amount << " M1->v: " << glob_index); - global_indices.push_back(glob_index); - - TIME_MEASURE_FINISH(db2); - t_dbscan += db2; - - } - } - - TXN_POSTFIX_RDONLY(); - - TIME_MEASURE_FINISH(txx); - LOG_PRINT_L3("txx: " << txx << " db1: " << t_dbmul << " db2: " << t_dbscan); -} - void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2647,29 +2519,25 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector global_indices; - get_output_global_indices(amount, offsets, global_indices); - if (global_indices.size() > 0) + RCURSOR(output_amounts); + + MDB_val_set(k, amount); + for (const uint64_t &index : offsets) { - RCURSOR(output_keys); + MDB_val_set(v, index); - for (const uint64_t &index : global_indices) - { - MDB_val_copy k(index); - MDB_val v; - - auto get_result = mdb_cursor_get(m_cur_output_keys, &k, &v, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); - else if (get_result) - throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); - - output_data_t data = *(const output_data_t *) v.mv_data; - outputs.push_back(data); - } + auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); + outkey *okp = (outkey *)v.mv_data; + output_data_t data = okp->data; + outputs.push_back(data); } + TXN_POSTFIX_RDONLY(); TIME_MEASURE_FINISH(db3); @@ -2682,13 +2550,30 @@ void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std:: check_open(); indices.clear(); - std::vector global_indices; - get_output_global_indices(amount, offsets, global_indices); + std::vector tx_indices; + TXN_PREFIX_RDONLY(); + + RCURSOR(output_amounts); + + MDB_val_set(k, amount); + for (const uint64_t &index : offsets) + { + MDB_val_set(v, index); + + auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get output by index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output from the db", get_result).c_str())); + + outkey *okp = (outkey *)v.mv_data; + tx_indices.push_back(okp->output_id); + } TIME_MEASURE_START(db3); - if(global_indices.size() > 0) + if(tx_indices.size() > 0) { - get_output_tx_and_index_from_global(global_indices, indices); + get_output_tx_and_index_from_global(tx_indices, indices); } TIME_MEASURE_FINISH(db3); LOG_PRINT_L3("db3: " << db3); @@ -2815,7 +2700,7 @@ uint64_t BlockchainLMDB::get_hard_fork_starting_height(uint8_t version) const MDB_val_copy val_key(version); MDB_val val_ret; - uint64_t ret; + uint64_t ret = 0; auto result = mdb_get(m_txn, m_hf_starting_heights, &val_key, &val_ret); if (result == MDB_SUCCESS) { @@ -2892,4 +2777,558 @@ void BlockchainLMDB::fixup() BlockchainDB::fixup(); } +#define RENAME_DB(name) \ + k.mv_data = (void *)name; \ + k.mv_size = sizeof(name)-1; \ + result = mdb_cursor_open(txn, 1, &c_cur); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for " name ": ", result).c_str())); \ + result = mdb_cursor_get(c_cur, &k, NULL, MDB_SET_KEY); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to get DB record for " name ": ", result).c_str())); \ + ptr = (char *)k.mv_data; \ + ptr[sizeof(name)-2] = 's' + +#define LOGIF(y) if (y <= epee::log_space::log_singletone::get_log_detalisation_level()) + +void BlockchainLMDB::migrate_0_1() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i, z; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + + LOG_PRINT_YELLOW("Migrating blockchain from DB version 0 to 1 - this may take a while:", LOG_LEVEL_0); + LOG_PRINT_L0("updating blocks, hf_versions, outputs, txs, and spent_keys tables..."); + + LOG_PRINT_L0("Total number of blocks: " << m_height); + LOG_PRINT_L1("block migration will update block_heights, block_info, and hf_versions..."); + + do { + LOG_PRINT_L1("migrating block_heights:"); + MDB_dbi o_heights; + + unsigned int flags; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_flags(txn, m_block_heights, &flags); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve block_heights flags: ", result).c_str())); + /* if the flags are what we expect, this table has already been migrated */ + if ((flags & (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) == (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) { + txn.abort(); + LOG_PRINT_L1(" block_heights already migrated"); + break; + } + + /* the block_heights table name is the same but the old version and new version + * have incompatible DB flags. Create a new table with the right flags. We want + * the name to be similar to the old name so that it will occupy the same location + * in the DB. + */ + o_heights = m_block_heights; + lmdb_db_open(txn, "block_heightr", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for block_heightr"); + mdb_set_dupsort(txn, m_block_heights, compare_hash32); + + MDB_cursor *c_old, *c_cur; + blk_height bh; + MDB_val_set(nv, bh); + + /* old table was k(hash), v(height). + * new table is DUPFIXED, k(zeroval), v{hash, height}. + */ + i = 0; + z = m_height; + while(1) { + if (!(i % 2000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_heights, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_heightr: ", result).c_str())); + result = mdb_cursor_open(txn, o_heights, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_heights: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_block_heights, &ms); + i = ms.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_heights: ", result).c_str())); + bh.bh_hash = *(crypto::hash *)k.mv_data; + bh.bh_height = *(uint64_t *)v.mv_data; + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_heightr: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_heights: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_heights, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_heights table: ", result).c_str())); + + RENAME_DB("block_heightr"); + + /* close and reopen to get old dbi slot back */ + mdb_dbi_close(m_env, m_block_heights); + lmdb_db_open(txn, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for block_heights"); + mdb_set_dupsort(txn, m_block_heights, compare_hash32); + txn.commit(); + + } while(0); + + /* old tables are k(height), v(value). + * new table is DUPFIXED, k(zeroval), v{height, values...}. + */ + do { + LOG_PRINT_L1("migrating block info:"); + + MDB_dbi coins; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_open(txn, "block_coins", 0, &coins); + if (result == MDB_NOTFOUND) { + txn.abort(); + LOG_PRINT_L1(" block_info already migrated"); + break; + } + MDB_dbi diffs, hashes, sizes, timestamps; + mdb_block_info bi; + MDB_val_set(nv, bi); + + lmdb_db_open(txn, "block_diffs", 0, diffs, "Failed to open db handle for block_diffs"); + lmdb_db_open(txn, "block_hashes", 0, hashes, "Failed to open db handle for block_hashes"); + lmdb_db_open(txn, "block_sizes", 0, sizes, "Failed to open db handle for block_sizes"); + lmdb_db_open(txn, "block_timestamps", 0, timestamps, "Failed to open db handle for block_timestamps"); + MDB_cursor *c_cur, *c_coins, *c_diffs, *c_hashes, *c_sizes, *c_timestamps; + i = 0; + z = m_height; + while(1) { + MDB_val k, v; + if (!(i % 2000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + result = mdb_cursor_open(txn, coins, &c_coins); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_coins: ", result).c_str())); + result = mdb_cursor_open(txn, diffs, &c_diffs); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_diffs: ", result).c_str())); + result = mdb_cursor_open(txn, hashes, &c_hashes); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_hashes: ", result).c_str())); + result = mdb_cursor_open(txn, sizes, &c_sizes); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_coins: ", result).c_str())); + result = mdb_cursor_open(txn, timestamps, &c_timestamps); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_timestamps: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_block_info, &ms); + i = ms.ms_entries; + } + } + result = mdb_cursor_get(c_coins, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + break; + } else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_coins: ", result).c_str())); + bi.bi_height = *(uint64_t *)k.mv_data; + bi.bi_coins = *(uint64_t *)v.mv_data; + result = mdb_cursor_get(c_diffs, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_diffs: ", result).c_str())); + bi.bi_diff = *(uint64_t *)v.mv_data; + result = mdb_cursor_get(c_hashes, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_hashes: ", result).c_str())); + bi.bi_hash = *(crypto::hash *)v.mv_data; + result = mdb_cursor_get(c_sizes, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_sizes: ", result).c_str())); + if (v.mv_size == sizeof(uint32_t)) + bi.bi_size = *(uint32_t *)v.mv_data; + else + bi.bi_size = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 + result = mdb_cursor_get(c_timestamps, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_timestamps: ", result).c_str())); + bi.bi_timestamp = *(uint64_t *)v.mv_data; + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_info: ", result).c_str())); + result = mdb_cursor_del(c_coins, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_coins: ", result).c_str())); + result = mdb_cursor_del(c_diffs, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_diffs: ", result).c_str())); + result = mdb_cursor_del(c_hashes, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_hashes: ", result).c_str())); + result = mdb_cursor_del(c_sizes, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_sizes: ", result).c_str())); + result = mdb_cursor_del(c_timestamps, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_timestamps: ", result).c_str())); + i++; + } + mdb_cursor_close(c_timestamps); + mdb_cursor_close(c_sizes); + mdb_cursor_close(c_hashes); + mdb_cursor_close(c_diffs); + mdb_cursor_close(c_coins); + result = mdb_drop(txn, timestamps, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_timestamps from the db: ", result).c_str())); + result = mdb_drop(txn, sizes, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_sizes from the db: ", result).c_str())); + result = mdb_drop(txn, hashes, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_hashes from the db: ", result).c_str())); + result = mdb_drop(txn, diffs, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_diffs from the db: ", result).c_str())); + result = mdb_drop(txn, coins, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_coins from the db: ", result).c_str())); + txn.commit(); + } while(0); + + do { + LOG_PRINT_L1("migrating hf_versions:"); + MDB_dbi o_hfv; + + unsigned int flags; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_flags(txn, m_hf_versions, &flags); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve hf_versions flags: ", result).c_str())); + /* if the flags are what we expect, this table has already been migrated */ + if (flags & MDB_INTEGERKEY) { + txn.abort(); + LOG_PRINT_L1(" hf_versions already migrated"); + break; + } + + /* the hf_versions table name is the same but the old version and new version + * have incompatible DB flags. Create a new table with the right flags. + */ + o_hfv = m_hf_versions; + lmdb_db_open(txn, "hf_versionr", MDB_INTEGERKEY | MDB_CREATE, m_hf_versions, "Failed to open db handle for hf_versionr"); + + MDB_cursor *c_old, *c_cur; + i = 0; + z = m_height; + + while(1) { + if (!(i % 2000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_hf_versions, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keyr: ", result).c_str())); + result = mdb_cursor_open(txn, o_hfv, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keys: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_hf_versions, &ms); + i = ms.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from hf_versions: ", result).c_str())); + result = mdb_cursor_put(c_cur, &k, &v, MDB_APPEND); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into hf_versionr: ", result).c_str())); + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from hf_versions: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_hfv, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old hf_versions table: ", result).c_str())); + RENAME_DB("hf_versionr"); + mdb_dbi_close(m_env, m_hf_versions); + lmdb_db_open(txn, "hf_versions", MDB_INTEGERKEY, m_hf_versions, "Failed to open db handle for hf_versions"); + + txn.commit(); + } while(0); + + do { + LOG_PRINT_L1("deleting old indices:"); + + /* Delete all other tables, we're just going to recreate them */ + MDB_dbi dbi; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + result = mdb_dbi_open(txn, "tx_unlocks", 0, &dbi); + if (result == MDB_NOTFOUND) { + txn.abort(); + LOG_PRINT_L1(" old indices already deleted"); + break; + } + txn.abort(); + +#define DELETE_DB(x) do { \ + LOG_PRINT_L1(" " x ":"); \ + result = mdb_txn_begin(m_env, NULL, 0, txn); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); \ + result = mdb_dbi_open(txn, x, 0, &dbi); \ + if (!result) { \ + result = mdb_drop(txn, dbi, 1); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to delete " x ": ", result).c_str())); \ + txn.commit(); \ + } } while(0) + + DELETE_DB("tx_heights"); + DELETE_DB("output_txs"); + DELETE_DB("output_indices"); + DELETE_DB("output_keys"); + DELETE_DB("spent_keys"); + DELETE_DB("output_amounts"); + DELETE_DB("tx_outputs"); + DELETE_DB("tx_unlocks"); + + /* reopen new DBs with correct flags */ + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_txs, "Failed to open db handle for m_output_txs"); + mdb_set_dupsort(txn, m_output_txs, compare_uint64); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); + mdb_set_dupsort(txn, m_spent_keys, compare_hash32); + lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); + mdb_set_dupsort(txn, m_output_amounts, compare_uint64); + txn.commit(); + m_num_txs = 0; + m_num_outputs = 0; + } while(0); + + do { + LOG_PRINT_L1("migrating txs and outputs:"); + + unsigned int flags; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_flags(txn, m_txs, &flags); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve txs flags: ", result).c_str())); + /* if the flags are what we expect, this table has already been migrated */ + if (flags & MDB_INTEGERKEY) { + txn.abort(); + LOG_PRINT_L1(" txs already migrated"); + break; + } + + MDB_dbi o_txs; + blobdata bd; + block b; + MDB_val hk; + + o_txs = m_txs; + mdb_set_compare(txn, o_txs, compare_hash32); + lmdb_db_open(txn, "txr", MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for txr"); + + txn.commit(); + + MDB_cursor *c_blocks, *c_txs, *c_props, *c_cur; + i = 0; + z = m_height; + + hk.mv_size = sizeof(crypto::hash); + set_batch_transactions(true); + batch_start(1000); + txn.m_txn = m_write_txn->m_txn; + m_height = 0; + + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + MDB_val_set(pk, "txblk"); + MDB_val_set(pv, m_height); + result = mdb_cursor_put(c_props, &pk, &pv, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update txblk property: ", result).c_str())); + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + m_write_txn->m_txn = txn.m_txn; + m_write_batch_txn->m_txn = txn.m_txn; + memset(&m_wcursors, 0, sizeof(m_wcursors)); + } + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + result = mdb_cursor_open(txn, m_properties, &c_props); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for properties: ", result).c_str())); + result = mdb_cursor_open(txn, o_txs, &c_txs); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_output_txs, &ms); + m_num_outputs = ms.ms_entries; + mdb_stat(txn, m_txs, &ms); + m_num_txs = i = ms.ms_entries; + if (i) { + m_num_txs = i; + MDB_val_set(pk, "txblk"); + result = mdb_cursor_get(c_props, &pk, &k, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from properties: ", result).c_str())); + m_height = *(uint64_t *)k.mv_data; + } + } + if (i) { + result = mdb_cursor_get(c_blocks, &k, &v, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str())); + } + } + result = mdb_cursor_get(c_blocks, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + MDB_val_set(pk, "txblk"); + mdb_cursor_get(c_props, &pk, &v, MDB_SET); + mdb_cursor_del(c_props, 0); + batch_stop(); + break; + } else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str())); + + bd.assign(reinterpret_cast(v.mv_data), v.mv_size); + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + + add_transaction(null_hash, b.miner_tx); + for (unsigned int j = 0; j(v.mv_data), v.mv_size); + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + add_transaction(null_hash, tx, &b.tx_hashes[j]); + result = mdb_cursor_del(c_txs, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str())); + } + i++; + m_height = i; + } + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_drop(txn, o_txs, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete txs from the db: ", result).c_str())); + + RENAME_DB("txr"); + + mdb_dbi_close(m_env, m_txs); + + lmdb_db_open(txn, "txs", MDB_INTEGERKEY, m_txs, "Failed to open db handle for txs"); + + txn.commit(); + } while(0); + + uint32_t version = 1; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_copy vk("version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + +void BlockchainLMDB::migrate(const uint32_t oldversion) +{ + switch(oldversion) { + case 0: + migrate_0_1(); /* FALLTHRU */ + default: + ; + } +} + } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 6cd3e0e8..c7121bf6 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -43,20 +43,13 @@ typedef struct mdb_txn_cursors { MDB_cursor *m_txc_blocks; MDB_cursor *m_txc_block_heights; - MDB_cursor *m_txc_block_hashes; - MDB_cursor *m_txc_block_timestamps; - MDB_cursor *m_txc_block_sizes; - MDB_cursor *m_txc_block_diffs; - MDB_cursor *m_txc_block_coins; + MDB_cursor *m_txc_block_info; MDB_cursor *m_txc_output_txs; - MDB_cursor *m_txc_output_indices; MDB_cursor *m_txc_output_amounts; - MDB_cursor *m_txc_output_keys; MDB_cursor *m_txc_txs; - MDB_cursor *m_txc_tx_heights; - MDB_cursor *m_txc_tx_unlocks; + MDB_cursor *m_txc_tx_indices; MDB_cursor *m_txc_tx_outputs; MDB_cursor *m_txc_spent_keys; @@ -66,18 +59,11 @@ typedef struct mdb_txn_cursors #define m_cur_blocks m_cursors->m_txc_blocks #define m_cur_block_heights m_cursors->m_txc_block_heights -#define m_cur_block_hashes m_cursors->m_txc_block_hashes -#define m_cur_block_timestamps m_cursors->m_txc_block_timestamps -#define m_cur_block_sizes m_cursors->m_txc_block_sizes -#define m_cur_block_diffs m_cursors->m_txc_block_diffs -#define m_cur_block_coins m_cursors->m_txc_block_coins +#define m_cur_block_info m_cursors->m_txc_block_info #define m_cur_output_txs m_cursors->m_txc_output_txs -#define m_cur_output_indices m_cursors->m_txc_output_indices #define m_cur_output_amounts m_cursors->m_txc_output_amounts -#define m_cur_output_keys m_cursors->m_txc_output_keys #define m_cur_txs m_cursors->m_txc_txs -#define m_cur_tx_heights m_cursors->m_txc_tx_heights -#define m_cur_tx_unlocks m_cursors->m_txc_tx_unlocks +#define m_cur_tx_indices m_cursors->m_txc_tx_indices #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs #define m_cur_spent_keys m_cursors->m_txc_spent_keys #define m_cur_hf_versions m_cursors->m_txc_hf_versions @@ -87,18 +73,11 @@ typedef struct mdb_rflags bool m_rf_txn; bool m_rf_blocks; bool m_rf_block_heights; - bool m_rf_block_hashes; - bool m_rf_block_timestamps; - bool m_rf_block_sizes; - bool m_rf_block_diffs; - bool m_rf_block_coins; + bool m_rf_block_info; bool m_rf_output_txs; - bool m_rf_output_indices; bool m_rf_output_amounts; - bool m_rf_output_keys; bool m_rf_txs; - bool m_rf_tx_heights; - bool m_rf_tx_unlocks; + bool m_rf_tx_indices; bool m_rf_tx_outputs; bool m_rf_spent_keys; bool m_rf_hf_versions; @@ -223,6 +202,7 @@ public: virtual uint64_t height() const; virtual bool tx_exists(const crypto::hash& h) const; + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const; virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; @@ -246,10 +226,8 @@ public: virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector &offsets, std::vector &indices); - virtual void get_output_global_indices(const uint64_t& amount, const std::vector &offsets, std::vector &indices); - virtual std::vector get_tx_output_indices(const crypto::hash& h) const; - virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const; + virtual std::vector get_tx_amount_output_indices(const uint64_t tx_id) const; virtual bool has_key_image(const crypto::key_image& img) const; @@ -306,18 +284,23 @@ private: virtual void remove_block(); - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time); + virtual uint64_t add_output(const crypto::hash& tx_hash, + const tx_out& tx_output, + const uint64_t& local_index, + const uint64_t unlock_time + ); - virtual void remove_output(const tx_out& tx_output); + virtual void add_tx_amount_output_indices(const uint64_t tx_id, + const std::vector& amount_output_indices + ); - void remove_tx_outputs(const MDB_val *tx_hash, const transaction& tx); + void remove_tx_outputs(const uint64_t tx_id, const transaction& tx); - void remove_output(const MDB_val *out_index, const uint64_t amount); - void remove_amount_output_index(const uint64_t amount, const MDB_val *global_output_index); + void remove_output(const uint64_t amount, const uint64_t& out_index); virtual void add_spent_key(const crypto::key_image& k_image); @@ -349,16 +332,6 @@ private: */ tx_out output_from_blob(const blobdata& blob) const; - /** - * @brief get the global index of the index-th output of the given amount - * - * @param amount the output amount - * @param index the index into the set of outputs of that amount - * - * @return the global index of the desired output - */ - uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); - void check_open() const; virtual bool is_read_only() const; @@ -366,25 +339,24 @@ private: // fix up anything that may be wrong due to past bugs virtual void fixup(); + // migrate from older DB version to current + void migrate(const uint32_t oldversion); + + // migrate from DB version 0 to 1 + void migrate_0_1(); + MDB_env* m_env; MDB_dbi m_blocks; MDB_dbi m_block_heights; - MDB_dbi m_block_hashes; - MDB_dbi m_block_timestamps; - MDB_dbi m_block_sizes; - MDB_dbi m_block_diffs; - MDB_dbi m_block_coins; + MDB_dbi m_block_info; MDB_dbi m_txs; - MDB_dbi m_tx_unlocks; - MDB_dbi m_tx_heights; + MDB_dbi m_tx_indices; MDB_dbi m_tx_outputs; MDB_dbi m_output_txs; - MDB_dbi m_output_indices; MDB_dbi m_output_amounts; - MDB_dbi m_output_keys; MDB_dbi m_spent_keys; @@ -394,6 +366,7 @@ private: MDB_dbi m_properties; uint64_t m_height; + uint64_t m_num_txs; uint64_t m_num_outputs; mutable uint64_t m_cum_size; // used in batch size estimation mutable int m_cum_count; diff --git a/src/blockchain_utilities/cn_deserialize.cpp b/src/blockchain_utilities/cn_deserialize.cpp index bf02dc15..b8d38c9a 100644 --- a/src/blockchain_utilities/cn_deserialize.cpp +++ b/src/blockchain_utilities/cn_deserialize.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/tx_extra.h" #include "cryptonote_core/blockchain.h" #include "blockchain_utilities.h" #include "common/command_line.h" @@ -140,6 +141,7 @@ int main(int argc, char* argv[]) cryptonote::block block; cryptonote::transaction tx; + std::vector fields; if (cryptonote::parse_and_validate_block_from_blob(blob, block)) { std::cout << "Parsed block:" << std::endl; @@ -149,6 +151,25 @@ int main(int argc, char* argv[]) { std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; + + if (cryptonote::parse_tx_extra(tx.extra, fields)) + { + std::cout << "tx_extra has " << fields.size() << " field(s)" << std::endl; + for (size_t n = 0; n < fields.size(); ++n) + { + std::cout << "field " << n << ": "; + if (typeid(cryptonote::tx_extra_padding) == fields[n].type()) std::cout << "extra padding: " << boost::get(fields[n]).size << " bytes"; + else if (typeid(cryptonote::tx_extra_pub_key) == fields[n].type()) std::cout << "extra pub key: " << boost::get(fields[n]).pub_key; + else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << boost::get(fields[n]).nonce; + else if (typeid(cryptonote::tx_extra_merge_mining_tag) == fields[n].type()) std::cout << "extra merge mining tag: depth " << boost::get(fields[n]).depth << ", merkle root " << boost::get(fields[n]).merkle_root; + else std::cout << "unknown"; + std::cout << std::endl; + } + } + else + { + std::cout << "Failed to parse tx_extra" << std::endl; + } } else { diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index e47aab0f..e251d0ec 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -83,7 +83,7 @@ namespace crypto { /* generate a random 32-byte (256-bit) integer and copy it to res */ static inline void random_scalar(ec_scalar &res) { unsigned char tmp[64]; - generate_random_bytes(64, tmp); + generate_random_bytes_not_thread_safe(64, tmp); sc_reduce(tmp); memcpy(&res, tmp, 32); } diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 883aa521..fa55c2aa 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -117,13 +117,20 @@ namespace crypto { const public_key *const *, std::size_t, const signature *); }; + /* Generate N random bytes + */ + inline void rand(size_t N, uint8_t *bytes) { + boost::lock_guard lock(random_lock); + generate_random_bytes_not_thread_safe(N, bytes); + } + /* Generate a value filled with random bytes. */ template typename std::enable_if::value, T>::type rand() { typename std::remove_cv::type res; boost::lock_guard lock(random_lock); - generate_random_bytes(sizeof(T), &res); + generate_random_bytes_not_thread_safe(sizeof(T), &res); return res; } diff --git a/src/crypto/random.c b/src/crypto/random.c index f8a50d85..6a9f63c1 100644 --- a/src/crypto/random.c +++ b/src/crypto/random.c @@ -113,7 +113,7 @@ INITIALIZER(init_random) { #endif } -void generate_random_bytes(size_t n, void *result) { +void generate_random_bytes_not_thread_safe(size_t n, void *result) { #if !defined(NDEBUG) assert(curstate == 1); curstate = 2; diff --git a/src/crypto/random.h b/src/crypto/random.h index 322b5bad..b0d2303b 100644 --- a/src/crypto/random.h +++ b/src/crypto/random.h @@ -32,4 +32,4 @@ #include -void generate_random_bytes(size_t n, void *result); +void generate_random_bytes_not_thread_safe(size_t n, void *result); diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index b4445756..d332f899 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1971,14 +1971,15 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vectortx_exists(tx_id)) + uint64_t tx_index; + if (!m_db->tx_exists(tx_id, tx_index)) { LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); return false; } // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts - indexs = m_db->get_tx_amount_output_indices(tx_id); + indexs = m_db->get_tx_amount_output_indices(tx_index); CHECK_AND_ASSERT_MES(indexs.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty"); return true; diff --git a/src/cryptonote_core/cryptonote_format_utils.cpp b/src/cryptonote_core/cryptonote_format_utils.cpp index 3b9dcc8a..ff752ae4 100644 --- a/src/cryptonote_core/cryptonote_format_utils.cpp +++ b/src/cryptonote_core/cryptonote_format_utils.cpp @@ -291,14 +291,14 @@ namespace cryptonote { tx_extra_field field; bool r = ::do_serialize(ar, field); - CHECK_AND_NO_ASSERT_MES(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); + CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); tx_extra_fields.push_back(field); std::ios_base::iostate state = iss.rdstate(); eof = (EOF == iss.peek()); iss.clear(state); } - CHECK_AND_NO_ASSERT_MES(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); + CHECK_AND_NO_ASSERT_MES_L1(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); return true; } @@ -357,7 +357,7 @@ namespace cryptonote { tx_extra_field field; bool r = ::do_serialize(ar, field); - CHECK_AND_NO_ASSERT_MES(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); + CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); if (field.type() != typeid(tx_extra_nonce)) ::do_serialize(newar, field); @@ -365,7 +365,7 @@ namespace cryptonote eof = (EOF == iss.peek()); iss.clear(state); } - CHECK_AND_NO_ASSERT_MES(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); + CHECK_AND_NO_ASSERT_MES_L1(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); tx_extra.clear(); std::string s = oss.str(); tx_extra.reserve(s.size()); diff --git a/src/cryptonote_core/cryptonote_format_utils.h b/src/cryptonote_core/cryptonote_format_utils.h index 4142829e..056940e9 100644 --- a/src/cryptonote_core/cryptonote_format_utils.h +++ b/src/cryptonote_core/cryptonote_format_utils.h @@ -44,7 +44,7 @@ namespace cryptonote crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx); - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 1, uint8_t hard_fork_version = 1); + bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key); bool decrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 4bed148f..a7a54b59 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -104,12 +104,12 @@ namespace cryptonote MAP_JON_RPC_WE("getblockheaderbyhash", on_get_block_header_by_hash, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH) MAP_JON_RPC_WE("getblockheaderbyheight", on_get_block_header_by_height, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT) MAP_JON_RPC_WE("getblock", on_get_block, COMMAND_RPC_GET_BLOCK) - MAP_JON_RPC_WE("get_connections", on_get_connections, COMMAND_RPC_GET_CONNECTIONS) + MAP_JON_RPC_WE_IF("get_connections", on_get_connections, COMMAND_RPC_GET_CONNECTIONS, !m_restricted) MAP_JON_RPC_WE("get_info", on_get_info_json, COMMAND_RPC_GET_INFO) MAP_JON_RPC_WE("hard_fork_info", on_hard_fork_info, COMMAND_RPC_HARD_FORK_INFO) - MAP_JON_RPC_WE("setbans", on_set_bans, COMMAND_RPC_SETBANS) - MAP_JON_RPC_WE("getbans", on_get_bans, COMMAND_RPC_GETBANS) - MAP_JON_RPC_WE("flush_txpool", on_flush_txpool, COMMAND_RPC_FLUSH_TRANSACTION_POOL) + MAP_JON_RPC_WE_IF("setbans", on_set_bans, COMMAND_RPC_SETBANS, !m_restricted) + MAP_JON_RPC_WE_IF("getbans", on_get_bans, COMMAND_RPC_GETBANS, !m_restricted) + MAP_JON_RPC_WE_IF("flush_txpool", on_flush_txpool, COMMAND_RPC_FLUSH_TRANSACTION_POOL, !m_restricted) MAP_JON_RPC_WE("get_output_histogram", on_get_output_histogram, COMMAND_RPC_GET_OUTPUT_HISTOGRAM) END_JSON_RPC_MAP() END_URI_MAP2() diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index cc7b7138..7d28de9c 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -1010,6 +1010,7 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m m_wallet.reset(new tools::wallet2(testnet)); m_wallet->callback(this); + m_wallet->set_refresh_from_block_height(field_scan_from_height); try { @@ -1055,8 +1056,6 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m return false; } - m_wallet->set_refresh_from_block_height(field_scan_from_height); - wallet_file = m_wallet_file; return r; @@ -1175,7 +1174,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } } - if (!m_restore_height) + if (!m_restore_height && m_generate_new.empty()) { std::string heightstr = command_line::input_line("Restore from specific blockchain height (optional, default 0): "); if (std::cin.eof()) @@ -1451,6 +1450,16 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string m_wallet->callback(this); m_wallet->set_seed_language(mnemonic_language); + // for a totally new account, we don't care about older blocks. + if (!m_generate_new.empty()) + { + std::string err; + m_wallet->set_refresh_from_block_height(get_daemon_blockchain_height(err)); + } else if (m_restore_height) + { + m_wallet->set_refresh_from_block_height(m_restore_height); + } + crypto::secret_key recovery_val; try { @@ -1466,15 +1475,6 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string } m_wallet->init(m_daemon_address); - // for a totally new account, we don't care about older blocks. - if (!m_restore_deterministic_wallet) - { - std::string err; - m_wallet->set_refresh_from_block_height(get_daemon_blockchain_height(err)); - } else if (m_restore_height) - { - m_wallet->set_refresh_from_block_height(m_restore_height); - } // convert rng value to electrum-style word list std::string electrum_words; @@ -1507,6 +1507,8 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string m_wallet.reset(new tools::wallet2(testnet)); m_wallet->callback(this); + if (m_restore_height) + m_wallet->set_refresh_from_block_height(m_restore_height); try { @@ -1522,7 +1524,6 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string } m_wallet->init(m_daemon_address); - m_wallet->set_refresh_from_block_height(m_restore_height); return true; } @@ -1534,6 +1535,8 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string m_wallet.reset(new tools::wallet2(testnet)); m_wallet->callback(this); + if (m_restore_height) + m_wallet->set_refresh_from_block_height(m_restore_height); try { @@ -1548,7 +1551,6 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string } m_wallet->init(m_daemon_address); - m_wallet->set_refresh_from_block_height(m_restore_height); return true; } diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 108aa057..6f05929b 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -519,7 +519,8 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " << miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms"); }else { - LOG_PRINT_L2( "Skipped block by timestamp, height: " << height << ", block time " << b.timestamp << ", account time " << m_account.get_createtime()); + if (!(height % 100)) + LOG_PRINT_L2( "Skipped block by timestamp, height: " << height << ", block time " << b.timestamp << ", account time " << m_account.get_createtime()); } m_blockchain.push_back(bl_id); ++m_local_bc_height; @@ -796,7 +797,8 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, { std::list hashes; size_t current_index = m_blockchain.size(); - while(current_index < stop_height) + + while(m_run.load(std::memory_order_relaxed) && current_index < stop_height) { pull_hashes(0, blocks_start_height, short_chain_history, hashes); if (hashes.size() < 3) @@ -823,7 +825,8 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, { if(current_index >= m_blockchain.size()) { - LOG_PRINT_L2( "Skipped block by height: " << current_index); + if (!(current_index % 1000)) + LOG_PRINT_L2( "Skipped block by height: " << current_index); m_blockchain.push_back(bl_id); ++m_local_bc_height; @@ -860,6 +863,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // pull the first set of blocks get_short_chain_history(short_chain_history); + m_run.store(true, std::memory_order_relaxed); if (start_height > m_blockchain.size() || m_refresh_from_block_height > m_blockchain.size()) { if (!start_height) start_height = m_refresh_from_block_height; @@ -877,7 +881,6 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // subsequent pulls in this refresh. start_height = 0; - m_run.store(true, std::memory_order_relaxed); while(m_run.load(std::memory_order_relaxed)) { try @@ -1057,6 +1060,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const std::string& p value2.SetInt(m_refresh_type); json.AddMember("refresh_type", value2, json.GetAllocator()); + value2.SetUint64(m_refresh_from_block_height); + json.AddMember("refresh_height", value2, json.GetAllocator()); + // Serialize the JSON object rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); @@ -1163,6 +1169,9 @@ bool wallet2::load_keys(const std::string& keys_file_name, const std::string& pa else LOG_PRINT_L0("Unknown refresh-type value (" << field_refresh_type << "), using default"); } + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, refresh_height, uint64_t, Uint64, false); + if (field_refresh_height_found) + m_refresh_from_block_height = field_refresh_height; } const cryptonote::account_keys& keys = m_account.get_keys(); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index faddf9c7..85bf33e3 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -309,6 +309,7 @@ namespace tools template inline void serialize(t_archive &a, const unsigned int ver) { + uint64_t dummy_refresh_height = 0; // moved to keys file if(ver < 5) return; a & m_blockchain; @@ -329,7 +330,7 @@ namespace tools a & m_confirmed_txs; if(ver < 11) return; - a & m_refresh_from_block_height; + a & dummy_refresh_height; if(ver < 12) return; a & m_tx_notes; diff --git a/tests/core_tests/chaingen.cpp b/tests/core_tests/chaingen.cpp index 4fdd0a60..efd0bf1e 100644 --- a/tests/core_tests/chaingen.cpp +++ b/tests/core_tests/chaingen.cpp @@ -222,7 +222,7 @@ bool test_generator::construct_block_manually(block& blk, const block& prev_bloc blk.timestamp = actual_params & bf_timestamp ? timestamp : prev_block.timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN; // Keep difficulty unchanged blk.prev_id = actual_params & bf_prev_id ? prev_id : get_block_hash(prev_block); blk.tx_hashes = actual_params & bf_tx_hashes ? tx_hashes : std::vector(); - max_outs = actual_params & bf_max_outs ? max_outs : 1; + max_outs = actual_params & bf_max_outs ? max_outs : 9999; size_t height = get_block_height(prev_block) + 1; uint64_t already_generated_coins = get_already_generated_coins(prev_block); diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index 652413b8..0e5dbb0e 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -241,7 +241,7 @@ public: const cryptonote::account_base& miner_acc, int actual_params = bf_none, uint8_t major_ver = 0, uint8_t minor_ver = 0, uint64_t timestamp = 0, const crypto::hash& prev_id = crypto::hash(), const cryptonote::difficulty_type& diffic = 1, const cryptonote::transaction& miner_tx = cryptonote::transaction(), - const std::vector& tx_hashes = std::vector(), size_t txs_sizes = 0, size_t max_outs = 0); + const std::vector& tx_hashes = std::vector(), size_t txs_sizes = 0, size_t max_outs = 999); bool construct_block_manually_tx(cryptonote::block& blk, const cryptonote::block& prev_block, const cryptonote::account_base& miner_acc, const std::vector& tx_hashes, size_t txs_size); diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index 9f8a5782..e20f7a15 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -168,9 +168,9 @@ int main(int argc, char* argv[]) GENERATE_AND_PLAY(gen_v2_tx_mixable_0_mixin); GENERATE_AND_PLAY(gen_v2_tx_mixable_low_mixin); - GENERATE_AND_PLAY(gen_v2_tx_unmixable_only); - GENERATE_AND_PLAY(gen_v2_tx_unmixable_one); - GENERATE_AND_PLAY(gen_v2_tx_unmixable_two); +// GENERATE_AND_PLAY(gen_v2_tx_unmixable_only); +// GENERATE_AND_PLAY(gen_v2_tx_unmixable_one); +// GENERATE_AND_PLAY(gen_v2_tx_unmixable_two); GENERATE_AND_PLAY(gen_v2_tx_dust); std::cout << (failed_tests.empty() ? concolor::green : concolor::magenta); diff --git a/tests/core_tests/v2_tests.cpp b/tests/core_tests/v2_tests.cpp index d0545628..fe6b8b27 100644 --- a/tests/core_tests/v2_tests.cpp +++ b/tests/core_tests/v2_tests.cpp @@ -38,7 +38,7 @@ using namespace cryptonote; //---------------------------------------------------------------------------------------------------------------------- // Tests -bool gen_v2_tx_validation_base::generate_with(std::vector& events, const int *out_idx, int mixin, uint64_t amount_paid, size_t max_outs, bool valid) const +bool gen_v2_tx_validation_base::generate_with(std::vector& events, const int *out_idx, int mixin, uint64_t amount_paid, bool valid) const { uint64_t ts_start = 1338224400; @@ -52,9 +52,9 @@ bool gen_v2_tx_validation_base::generate_with(std::vector& eve for (size_t n = 0; n < 4; ++n) { miner_accounts[n].generate(); CHECK_AND_ASSERT_MES(generator.construct_block_manually(blocks[n], *prev_block, miner_accounts[n], - test_generator::bf_major_ver | test_generator::bf_minor_ver | test_generator::bf_timestamp | test_generator::bf_max_outs, + test_generator::bf_major_ver | test_generator::bf_minor_ver | test_generator::bf_timestamp, 2, 2, prev_block->timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN * 2, // v2 has blocks twice as long - crypto::hash(), 0, transaction(), std::vector(), 0, max_outs), + crypto::hash(), 0, transaction(), std::vector(), 0, 0), false, "Failed to generate block"); events.push_back(blocks[n]); prev_block = blocks + n; @@ -68,9 +68,9 @@ bool gen_v2_tx_validation_base::generate_with(std::vector& eve { cryptonote::block blk; CHECK_AND_ASSERT_MES(generator.construct_block_manually(blk, blk_last, miner_account, - test_generator::bf_major_ver | test_generator::bf_minor_ver | test_generator::bf_timestamp | test_generator::bf_max_outs, + test_generator::bf_major_ver | test_generator::bf_minor_ver | test_generator::bf_timestamp, 2, 2, blk_last.timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN * 2, // v2 has blocks twice as long - crypto::hash(), 0, transaction(), std::vector(), 0, max_outs), + crypto::hash(), 0, transaction(), std::vector(), 0, 0), false, "Failed to generate block"); events.push_back(blk); blk_last = blk; @@ -123,8 +123,7 @@ bool gen_v2_tx_mixable_0_mixin::generate(std::vector& events) const int mixin = 0; const int out_idx[] = {1, -1}; const uint64_t amount_paid = 10000; - const size_t max_outs = 11; - return generate_with(events, out_idx, mixin, amount_paid, max_outs, false); + return generate_with(events, out_idx, mixin, amount_paid, false); } bool gen_v2_tx_mixable_low_mixin::generate(std::vector& events) const @@ -132,8 +131,7 @@ bool gen_v2_tx_mixable_low_mixin::generate(std::vector& events const int mixin = 1; const int out_idx[] = {1, -1}; const uint64_t amount_paid = 10000; - const size_t max_outs = 11; - return generate_with(events, out_idx, mixin, amount_paid, max_outs, false); + return generate_with(events, out_idx, mixin, amount_paid, false); } bool gen_v2_tx_unmixable_only::generate(std::vector& events) const @@ -141,8 +139,7 @@ bool gen_v2_tx_unmixable_only::generate(std::vector& events) c const int mixin = 0; const int out_idx[] = {0, -1}; const uint64_t amount_paid = 10000; - const size_t max_outs = 1; - return generate_with(events, out_idx, mixin, amount_paid, max_outs, true); + return generate_with(events, out_idx, mixin, amount_paid, true); } bool gen_v2_tx_unmixable_one::generate(std::vector& events) const @@ -150,8 +147,7 @@ bool gen_v2_tx_unmixable_one::generate(std::vector& events) co const int mixin = 0; const int out_idx[] = {0, 1, -1}; const uint64_t amount_paid = 10000; - const size_t max_outs = 11; - return generate_with(events, out_idx, mixin, amount_paid, max_outs, true); + return generate_with(events, out_idx, mixin, amount_paid, true); } bool gen_v2_tx_unmixable_two::generate(std::vector& events) const @@ -159,8 +155,7 @@ bool gen_v2_tx_unmixable_two::generate(std::vector& events) co const int mixin = 0; const int out_idx[] = {0, 1, 2, -1}; const uint64_t amount_paid = 10000; - const size_t max_outs = 11; - return generate_with(events, out_idx, mixin, amount_paid, max_outs, false); + return generate_with(events, out_idx, mixin, amount_paid, false); } bool gen_v2_tx_dust::generate(std::vector& events) const @@ -168,6 +163,5 @@ bool gen_v2_tx_dust::generate(std::vector& events) const const int mixin = 2; const int out_idx[] = {1, -1}; const uint64_t amount_paid = 10001; - const size_t max_outs = 11; - return generate_with(events, out_idx, mixin, amount_paid, max_outs, false); + return generate_with(events, out_idx, mixin, amount_paid, false); } diff --git a/tests/core_tests/v2_tests.h b/tests/core_tests/v2_tests.h index c296f0d8..10049ec9 100644 --- a/tests/core_tests/v2_tests.h +++ b/tests/core_tests/v2_tests.h @@ -70,7 +70,7 @@ struct gen_v2_tx_validation_base : public test_chain_unit_base } bool generate_with(std::vector& events, const int *out_idx, int mixin, - uint64_t amount_paid, size_t max_outs, bool valid) const; + uint64_t amount_paid, bool valid) const; private: size_t m_invalid_tx_index; diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index 2e510022..1b272cf5 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -50,7 +50,8 @@ set(unit_tests_sources test_peerlist.cpp test_protocol_pack.cpp hardfork.cpp - unbound.cpp) + unbound.cpp + varint.cpp) set(unit_tests_headers unit_tests_utils.h) @@ -70,7 +71,7 @@ target_link_libraries(unit_tests ${Boost_REGEX_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} - ${UNBOUND_LIBRARIES} + ${UNBOUND_LIBRARY} ${EXTRA_LIBRARIES}) set_property(TARGET unit_tests PROPERTY diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index cc3eba8e..6d4a100d 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -78,6 +78,7 @@ public: virtual block get_top_block() const { return block(); } virtual uint64_t height() const { return blocks.size(); } virtual bool tx_exists(const crypto::hash& h) const { return false; } + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const { return false; } virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const { return 0; } virtual transaction get_tx(const crypto::hash& h) const { return transaction(); } virtual uint64_t get_tx_count() const { return 0; } @@ -93,13 +94,13 @@ public: virtual void get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs) {} virtual bool can_thread_bulk_indices() const { return false; } virtual std::vector get_tx_output_indices(const crypto::hash& h) const { return std::vector(); } - virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const { return std::vector(); } + virtual std::vector get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector(); } virtual bool has_key_image(const crypto::key_image& img) const { return false; } virtual void remove_block() { blocks.pop_back(); } - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {} + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {return 0;} virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) {} - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {} - virtual void remove_output(const tx_out& tx_output) {} + virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {return 0;} + virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices) {} virtual void add_spent_key(const crypto::key_image& k_image) {} virtual void remove_spent_key(const crypto::key_image& k_image) {} diff --git a/tests/unit_tests/varint.cpp b/tests/unit_tests/varint.cpp new file mode 100644 index 00000000..a483cbd5 --- /dev/null +++ b/tests/unit_tests/varint.cpp @@ -0,0 +1,66 @@ +// Copyright (c) 2014-2016, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include +#include +#include +#include +#include +#include +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/cryptonote_basic_impl.h" +#include "serialization/serialization.h" +#include "serialization/binary_archive.h" +#include "serialization/json_archive.h" +#include "serialization/debug_archive.h" +#include "serialization/variant.h" +#include "serialization/vector.h" +#include "serialization/binary_utils.h" +#include "gtest/gtest.h" +using namespace std; + +TEST(varint, equal) +{ + for (uint64_t idx = 0; idx < 65537; ++idx) + { + char buf[12]; + char *bufptr = buf; + tools::write_varint(bufptr, idx); + uint64_t bytes = bufptr - buf; + ASSERT_TRUE (bytes > 0 && bytes <= sizeof(buf)); + + uint64_t idx2; + bufptr = buf; + std::string s(buf, bytes); + int read = tools::read_varint(s.begin(), s.end(), idx2); + ASSERT_EQ (read, bytes); + ASSERT_TRUE(idx2 == idx); + } +}