Merge branch 'performance' of https://github.com/LMDB/bitmonero

external/db_drivers/liblmdb/mdb.c

@@ -6771,8 +6771,8 @@ set1:
 				if (op == MDB_GET_BOTH || rc > 0)
 					return MDB_NOTFOUND;
 				rc = 0;
-				*data = olddata;
 			}
+			*data = olddata;
 
 		} else {
 			if (mc->mc_xcursor)
@@ -10501,8 +10501,11 @@ mdb_drop0(MDB_cursor *mc, int subs)
 
 		/* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves.
 		 * This also avoids any P_LEAF2 pages, which have no nodes.
+		 * Also if the DB doesn't have sub-DBs and has no overflow
+		 * pages, omit scanning leaves.
 		 */
-		if (mc->mc_flags & C_SUB)
+		if ((mc->mc_flags & C_SUB) ||
+			(!subs && !mc->mc_db->md_overflow_pages))
 			mdb_cursor_pop(mc);
 
 		mdb_cursor_copy(mc, &mx);
@@ -10529,6 +10532,9 @@ mdb_drop0(MDB_cursor *mc, int subs)
 							pg, omp->mp_pages);
 						if (rc)
 							goto done;
+						mc->mc_db->md_overflow_pages -= omp->mp_pages;
+						if (!mc->mc_db->md_overflow_pages && !subs)
+							break;
 					} else if (subs && (ni->mn_flags & F_SUBDATA)) {
 						mdb_xcursor_init1(mc, ni);
 						rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0);
@@ -10536,6 +10542,8 @@ mdb_drop0(MDB_cursor *mc, int subs)
 							goto done;
 					}
 				}
+				if (!subs && !mc->mc_db->md_overflow_pages)
+					goto pop;
 			} else {
 				if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0)
 					goto done;
@@ -10557,6 +10565,7 @@ mdb_drop0(MDB_cursor *mc, int subs)
 				/* no more siblings, go back to beginning
 				 * of previous level.
 				 */
+pop:
 				mdb_cursor_pop(mc);
 				mc->mc_ki[0] = 0;
 				for (i=1; i<mc->mc_snum; i++) {

src/blockchain_db/blockchain_db.cpp

@@ -82,14 +82,17 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti
     }
   }
 
-  add_transaction_data(blk_hash, tx, tx_hash);
+  uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash);
+
+  std::vector<uint64_t> amount_output_indices;
 
   // iterate tx.vout using indices instead of C++11 foreach syntax because
   // we need the index
   for (uint64_t i = 0; i < tx.vout.size(); ++i)
   {
-    add_output(tx_hash, tx.vout[i], i, tx.unlock_time);
+    amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time));
   }
+  add_tx_amount_output_indices(tx_id, amount_output_indices);
 }
 
 uint64_t BlockchainDB::add_block( const block& blk

src/blockchain_db/blockchain_db.h

@@ -59,6 +59,39 @@
  * Unspent transaction outputs are duplicated to quickly gather random
  * outputs to use for mixins
  *
+ * Indices and Identifiers:
+ * The word "index" is used ambiguously throughout this code. It is
+ * particularly confusing when talking about the output or transaction
+ * tables since their indexing can refer to themselves or each other.
+ * I have attempted to clarify these usages here:
+ *
+ * Blocks, transactions, and outputs are all identified by a hash.
+ * For storage efficiency, a 64-bit integer ID is used instead of the hash
+ * inside the DB. Tables exist to map between hash and ID. A block ID is
+ * also referred to as its "height". Transactions and outputs generally are
+ * not referred to by ID outside of this module, but the tx ID is returned
+ * by tx_exists() and used by get_tx_amount_output_indices(). Like their
+ * corresponding hashes, IDs are globally unique.
+ *
+ * The remaining uses of the word "index" refer to local offsets, and are
+ * not globally unique. An "amount output index" N refers to the Nth output
+ * of a specific amount. An "output local index" N refers to the Nth output
+ * of a specific tx.
+ *
+ * Exceptions:
+ *   DB_ERROR -- generic
+ *   DB_OPEN_FAILURE
+ *   DB_CREATE_FAILURE
+ *   DB_SYNC_FAILURE
+ *   BLOCK_DNE
+ *   BLOCK_PARENT_DNE
+ *   BLOCK_EXISTS
+ *   BLOCK_INVALID -- considering making this multiple errors
+ *   TX_DNE
+ *   TX_EXISTS
+ *   OUTPUT_DNE
+ *   OUTPUT_EXISTS
+ *   KEY_IMAGE_EXISTS
  */
 
 namespace cryptonote
@@ -80,6 +113,15 @@ struct output_data_t
 };
 #pragma pack(pop)
 
+#pragma pack(push, 1)
+struct tx_data_t
+{
+  uint64_t tx_id;
+  uint64_t unlock_time;
+  uint64_t block_id;
+};
+#pragma pack(pop)
+
 /***********************************
  * Exception Definitions
  ***********************************/
@@ -311,14 +353,18 @@ private:
    * and the other data passed here, not the separate outputs of the
    * transaction.
    *
+   * It returns a tx ID, which is a mapping from the tx_hash. The tx ID
+   * is used in #add_tx_amount_output_indices().
+   *
    * If any of this cannot be done, the subclass should throw the corresponding
    * subclass of DB_EXCEPTION
    *
    * @param blk_hash the hash of the block containing the transaction
    * @param tx the transaction to be added
    * @param tx_hash the hash of the transaction
+   * @return the transaction ID
    */
-  virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0;
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0;
 
   /**
    * @brief remove data about a transaction
@@ -348,6 +394,9 @@ private:
    * future, this tracking (of the number, at least) should be moved to
    * this class, as it is necessary and the same among all BlockchainDB.
    *
+   * It returns an amount output index, which is the index of the output
+   * for its specified amount.
+   *
    * This data should be stored in such a manner that the only thing needed to
    * reverse the process is the tx_out.
    *
@@ -358,25 +407,24 @@ private:
    * @param tx_output the output
    * @param local_index index of the output in its transaction
    * @param unlock_time unlock time/height of the output
+   * @return amount output index
    */
-  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0;
+  virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0;
 
   /**
-   * @brief remove an output
+   * @brief store amount output indices for a tx's outputs
    *
-   * The subclass implementing this will remove all output data it stored
+   * The subclass implementing this will add the amount output indices to its
-   * in add_output().
+   * backing store in a suitable manner. The tx_id will be the same one that
-   *
+   * was returned from #add_output().
-   * In addition, the subclass is responsible for correctly decrementing
-   * its global output counter (this may be automatic for some, such as using
-   * a database backend "count" feature).
    *
    * If any of this cannot be done, the subclass should throw the corresponding
    * subclass of DB_EXCEPTION
    *
-   * @param tx_output the output to be removed
+   * @param tx_id ID of the transaction containing these outputs
+   * @param amount_output_indices the amount output indices of the transaction
    */
-  virtual void remove_output(const tx_out& tx_output) = 0;
+  virtual void add_tx_amount_output_indices(const uint64_t tx_id, const std::vector<uint64_t>& amount_output_indices) = 0;
 
   /**
    * @brief store a spent key
@@ -414,18 +462,6 @@ private:
    */
   void pop_block();
 
-  /**
-   * @brief helper function for add_transactions, to add each individual transaction
-   *
-   * This function is called by add_transactions() for each transaction to be
-   * added.
-   *
-   * @param blk_hash hash of the block which has the transaction
-   * @param tx the transaction to add
-   * @param tx_hash_ptr the hash of the transaction, if already calculated
-   */
-  void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL);
-
   // helper function to remove transaction from blockchain
   /**
    * @brief helper function to remove transaction from the blockchain
@@ -444,6 +480,18 @@ private:
 
 protected:
 
+  /**
+   * @brief helper function for add_transactions, to add each individual transaction
+   *
+   * This function is called by add_transactions() for each transaction to be
+   * added.
+   *
+   * @param blk_hash hash of the block which has the transaction
+   * @param tx the transaction to add
+   * @param tx_hash_ptr the hash of the transaction, if already calculated
+   */
+  void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL);
+
   mutable uint64_t time_tx_exists = 0;  //!< a performance metric
   uint64_t time_commit1 = 0;  //!< a performance metric
   bool m_auto_remove_logs = true;  //!< whether or not to automatically remove old logs
@@ -930,10 +978,12 @@ public:
    * given hash and return true if so, false otherwise.
    *
    * @param h the hash to check against
+   * @param tx_id (optional) returns the tx_id for the tx hash
    *
    * @return true if the transaction exists, otherwise false
    */
   virtual bool tx_exists(const crypto::hash& h) const = 0;
+  virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_id) const = 0;
 
   // return unlock time of tx with hash <h>
   /**
@@ -1124,37 +1174,21 @@ public:
    */
   virtual bool can_thread_bulk_indices() const = 0;
 
-  /**
-   * @brief gets output indices (global) for a transaction's outputs
-   *
-   * The subclass should fetch the global output indices for each output
-   * in the transaction with the given hash.
-   *
-   * If the transaction does not exist, the subclass should throw TX_DNE.
-   *
-   * If an output cannot be found, the subclass should throw OUTPUT_DNE.
-   *
-   * @param h a transaction hash
-   *
-   * @return a list of global output indices
-   */
-  virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const = 0;
-
   /**
    * @brief gets output indices (amount-specific) for a transaction's outputs
    *
    * The subclass should fetch the amount-specific output indices for each
-   * output in the transaction with the given hash.
+   * output in the transaction with the given ID.
    *
    * If the transaction does not exist, the subclass should throw TX_DNE.
    *
    * If an output cannot be found, the subclass should throw OUTPUT_DNE.
    *
-   * @param h a transaction hash
+   * @param tx_id a transaction ID
    *
    * @return a list of amount-specific output indices
    */
-  virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const = 0;
+  virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const = 0;
 
   /**
    * @brief check if a key image is stored as spent

src/blockchain_db/lmdb/db_lmdb.h

@@ -43,20 +43,13 @@ typedef struct mdb_txn_cursors
 {
   MDB_cursor *m_txc_blocks;
   MDB_cursor *m_txc_block_heights;
-  MDB_cursor *m_txc_block_hashes;
+  MDB_cursor *m_txc_block_info;
-  MDB_cursor *m_txc_block_timestamps;
-  MDB_cursor *m_txc_block_sizes;
-  MDB_cursor *m_txc_block_diffs;
-  MDB_cursor *m_txc_block_coins;
 
   MDB_cursor *m_txc_output_txs;
-  MDB_cursor *m_txc_output_indices;
   MDB_cursor *m_txc_output_amounts;
-  MDB_cursor *m_txc_output_keys;
 
   MDB_cursor *m_txc_txs;
-  MDB_cursor *m_txc_tx_heights;
+  MDB_cursor *m_txc_tx_indices;
-  MDB_cursor *m_txc_tx_unlocks;
   MDB_cursor *m_txc_tx_outputs;
 
   MDB_cursor *m_txc_spent_keys;
@@ -66,18 +59,11 @@ typedef struct mdb_txn_cursors
 
 #define m_cur_blocks	m_cursors->m_txc_blocks
 #define m_cur_block_heights	m_cursors->m_txc_block_heights
-#define m_cur_block_hashes	m_cursors->m_txc_block_hashes
+#define m_cur_block_info	m_cursors->m_txc_block_info
-#define m_cur_block_timestamps	m_cursors->m_txc_block_timestamps
-#define m_cur_block_sizes	m_cursors->m_txc_block_sizes
-#define m_cur_block_diffs	m_cursors->m_txc_block_diffs
-#define m_cur_block_coins	m_cursors->m_txc_block_coins
 #define m_cur_output_txs	m_cursors->m_txc_output_txs
-#define m_cur_output_indices	m_cursors->m_txc_output_indices
 #define m_cur_output_amounts	m_cursors->m_txc_output_amounts
-#define m_cur_output_keys	m_cursors->m_txc_output_keys
 #define m_cur_txs	m_cursors->m_txc_txs
-#define m_cur_tx_heights	m_cursors->m_txc_tx_heights
+#define m_cur_tx_indices	m_cursors->m_txc_tx_indices
-#define m_cur_tx_unlocks	m_cursors->m_txc_tx_unlocks
 #define m_cur_tx_outputs	m_cursors->m_txc_tx_outputs
 #define m_cur_spent_keys	m_cursors->m_txc_spent_keys
 #define m_cur_hf_versions	m_cursors->m_txc_hf_versions
@@ -87,18 +73,11 @@ typedef struct mdb_rflags
   bool m_rf_txn;
   bool m_rf_blocks;
   bool m_rf_block_heights;
-  bool m_rf_block_hashes;
+  bool m_rf_block_info;
-  bool m_rf_block_timestamps;
-  bool m_rf_block_sizes;
-  bool m_rf_block_diffs;
-  bool m_rf_block_coins;
   bool m_rf_output_txs;
-  bool m_rf_output_indices;
   bool m_rf_output_amounts;
-  bool m_rf_output_keys;
   bool m_rf_txs;
-  bool m_rf_tx_heights;
+  bool m_rf_tx_indices;
-  bool m_rf_tx_unlocks;
   bool m_rf_tx_outputs;
   bool m_rf_spent_keys;
   bool m_rf_hf_versions;
@@ -223,6 +202,7 @@ public:
   virtual uint64_t height() const;
 
   virtual bool tx_exists(const crypto::hash& h) const;
+  virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const;
 
   virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const;
 
@@ -246,10 +226,8 @@ public:
 
   virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index);
   virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices);
-  virtual void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &indices);
 
-  virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const;
+  virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const;
-  virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const;
 
   virtual bool has_key_image(const crypto::key_image& img) const;
 
@@ -306,18 +284,23 @@ private:
 
   virtual void remove_block();
 
-  virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash);
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash);
 
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
 
-  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time);
+  virtual uint64_t add_output(const crypto::hash& tx_hash,
+      const tx_out& tx_output,
+      const uint64_t& local_index,
+      const uint64_t unlock_time
+      );
 
-  virtual void remove_output(const tx_out& tx_output);
+  virtual void add_tx_amount_output_indices(const uint64_t tx_id,
+      const std::vector<uint64_t>& amount_output_indices
+      );
 
-  void remove_tx_outputs(const MDB_val *tx_hash, const transaction& tx);
+  void remove_tx_outputs(const uint64_t tx_id, const transaction& tx);
 
-  void remove_output(const MDB_val *out_index, const uint64_t amount);
+  void remove_output(const uint64_t amount, const uint64_t& out_index);
-  void remove_amount_output_index(const uint64_t amount, const MDB_val *global_output_index);
 
   virtual void add_spent_key(const crypto::key_image& k_image);
 
@@ -349,16 +332,6 @@ private:
    */
   tx_out output_from_blob(const blobdata& blob) const;
 
-  /**
-   * @brief get the global index of the index-th output of the given amount
-   *
-   * @param amount the output amount
-   * @param index the index into the set of outputs of that amount
-   *
-   * @return the global index of the desired output
-   */
-  uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index);
-
   void check_open() const;
 
   virtual bool is_read_only() const;
@@ -366,25 +339,24 @@ private:
   // fix up anything that may be wrong due to past bugs
   virtual void fixup();
 
+  // migrate from older DB version to current
+  void migrate(const uint32_t oldversion);
+
+  // migrate from DB version 0 to 1
+  void migrate_0_1();
+
   MDB_env* m_env;
 
   MDB_dbi m_blocks;
   MDB_dbi m_block_heights;
-  MDB_dbi m_block_hashes;
+  MDB_dbi m_block_info;
-  MDB_dbi m_block_timestamps;
-  MDB_dbi m_block_sizes;
-  MDB_dbi m_block_diffs;
-  MDB_dbi m_block_coins;
 
   MDB_dbi m_txs;
-  MDB_dbi m_tx_unlocks;
+  MDB_dbi m_tx_indices;
-  MDB_dbi m_tx_heights;
   MDB_dbi m_tx_outputs;
 
   MDB_dbi m_output_txs;
-  MDB_dbi m_output_indices;
   MDB_dbi m_output_amounts;
-  MDB_dbi m_output_keys;
 
   MDB_dbi m_spent_keys;
 
@@ -394,6 +366,7 @@ private:
   MDB_dbi m_properties;
 
   uint64_t m_height;
+  uint64_t m_num_txs;
   uint64_t m_num_outputs;
   mutable uint64_t m_cum_size;	// used in batch size estimation
   mutable int m_cum_count;

src/cryptonote_core/blockchain.cpp

@@ -1971,14 +1971,15 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
-  if (!m_db->tx_exists(tx_id))
+  uint64_t tx_index;
+  if (!m_db->tx_exists(tx_id, tx_index))
   {
     LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id);
     return false;
   }
 
   // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts
-  indexs = m_db->get_tx_amount_output_indices(tx_id);
+  indexs = m_db->get_tx_amount_output_indices(tx_index);
   CHECK_AND_ASSERT_MES(indexs.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty");
 
   return true;

tests/unit_tests/hardfork.cpp

@@ -78,6 +78,7 @@ public:
   virtual block get_top_block() const { return block(); }
   virtual uint64_t height() const { return blocks.size(); }
   virtual bool tx_exists(const crypto::hash& h) const { return false; }
+  virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const { return false; }
   virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const { return 0; }
   virtual transaction get_tx(const crypto::hash& h) const { return transaction(); }
   virtual uint64_t get_tx_count() const { return 0; }
@@ -93,13 +94,13 @@ public:
   virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) {}
   virtual bool can_thread_bulk_indices() const { return false; }
   virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); }
-  virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); }
+  virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector<uint64_t>(); }
   virtual bool has_key_image(const crypto::key_image& img) const { return false; }
   virtual void remove_block() { blocks.pop_back(); }
-  virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {}
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {return 0;}
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) {}
-  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {}
+  virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {return 0;}
-  virtual void remove_output(const tx_out& tx_output) {}
+  virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) {}
   virtual void add_spent_key(const crypto::key_image& k_image) {}
   virtual void remove_spent_key(const crypto::key_image& k_image) {}