Merge pull request #5256

4b21d38d blockchain: speed up getting N blocks weights/long term weights (moneromooo-monero)

src/blockchain_db/blockchain_db.h

@@ -946,6 +946,17 @@ public:
    */
   virtual size_t get_block_weight(const uint64_t& height) const = 0;
 
+  /**
+   * @brief fetch the last N blocks' weights
+   *
+   * If there are fewer than N blocks, the returned array will be smaller than N
+   *
+   * @param count the number of blocks requested
+   *
+   * @return the weights
+   */
+  virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const = 0;
+
   /**
    * @brief fetch a block's cumulative difficulty
    *
@@ -999,6 +1010,17 @@ public:
    */
   virtual uint64_t get_block_long_term_weight(const uint64_t& height) const = 0;
 
+  /**
+   * @brief fetch the last N blocks' long term weights
+   *
+   * If there are fewer than N blocks, the returned array will be smaller than N
+   *
+   * @param count the number of blocks requested
+   *
+   * @return the weights
+   */
+  virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const = 0;
+
   /**
    * @brief fetch a block's hash
    *

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -2445,6 +2445,70 @@ size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const
   return ret;
 }
 
+std::vector<uint64_t> BlockchainLMDB::get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(block_info);
+
+  const uint64_t h = height();
+  if (start_height >= h)
+    throw0(DB_ERROR(("Height " + std::to_string(start_height) + " not in blockchain").c_str()));
+
+  std::vector<uint64_t> ret;
+  ret.reserve(count);
+
+  MDB_val v;
+  uint64_t range_begin = 0, range_end = 0;
+  for (uint64_t height = start_height; height < h && count--; ++height)
+  {
+    if (height >= range_begin && height < range_end)
+    {
+      // nothing to do
+    }
+    else
+    {
+      int result = 0;
+      if (range_end > 0)
+      {
+        MDB_val k2;
+        result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT_MULTIPLE);
+        range_begin = ((const mdb_block_info*)v.mv_data)->bi_height;
+        range_end = range_begin + v.mv_size / sizeof(mdb_block_info); // whole records please
+        if (height < range_begin || height >= range_end)
+          throw0(DB_ERROR(("Height " + std::to_string(height) + " not included in multiple record range: " + std::to_string(range_begin) + "-" + std::to_string(range_end)).c_str()));
+      }
+      else
+      {
+        v.mv_size = sizeof(uint64_t);
+        v.mv_data = (void*)&height;
+        result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
+        range_begin = height;
+        range_end = range_begin + 1;
+      }
+      if (result)
+        throw0(DB_ERROR(lmdb_error("Error attempting to retrieve block_info from the db: ", result).c_str()));
+    }
+    const mdb_block_info *bi = ((const mdb_block_info *)v.mv_data) + (height - range_begin);
+    ret.push_back(*(const uint64_t*)(((const char*)bi) + offset));
+  }
+
+  TXN_POSTFIX_RDONLY();
+  return ret;
+}
+
+std::vector<uint64_t> BlockchainLMDB::get_block_weights(uint64_t start_height, size_t count) const
+{
+  return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_weight));
+}
+
+std::vector<uint64_t> BlockchainLMDB::get_long_term_block_weights(uint64_t start_height, size_t count) const
+{
+  return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_long_term_block_weight));
+}
+
 difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__ << "  height: " << height);

src/blockchain_db/lmdb/db_lmdb.h

@@ -219,6 +219,8 @@ public:
 
   virtual size_t get_block_weight(const uint64_t& height) const;
 
+  virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const;
+
   virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const;
 
   virtual difficulty_type get_block_difficulty(const uint64_t& height) const;
@@ -227,6 +229,8 @@ public:
 
   virtual uint64_t get_block_long_term_weight(const uint64_t& height) const;
 
+  virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const;
+
   virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const;
 
   virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const;
@@ -394,6 +398,8 @@ private:
 
   virtual uint64_t get_database_size() const;
 
+  std::vector<uint64_t> get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const;
+
   // fix up anything that may be wrong due to past bugs
   virtual void fixup();
 

src/blockchain_db/testdb.h

@@ -72,10 +72,12 @@ public:
   virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const { return {}; }
   virtual uint64_t get_top_block_timestamp() const { return 0; }
   virtual size_t get_block_weight(const uint64_t& height) const { return 128; }
+  virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const { return {}; }
   virtual cryptonote::difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const { return 10; }
   virtual cryptonote::difficulty_type get_block_difficulty(const uint64_t& height) const { return 0; }
   virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const { return 10000000000; }
   virtual uint64_t get_block_long_term_weight(const uint64_t& height) const { return 128; }
+  virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const { return {}; }
   virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const { return crypto::hash(); }
   virtual std::vector<cryptonote::block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<cryptonote::block>(); }
   virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<crypto::hash>(); }

src/cryptonote_core/blockchain.cpp

@@ -1269,15 +1269,17 @@ void Blockchain::get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_
   if(h == 0)
     return;
 
-  m_db->block_txn_start(true);
   // add weight of last <count> blocks to vector <weights> (or less, if blockchain size < count)
   size_t start_offset = h - std::min<size_t>(h, count);
-  weights.reserve(weights.size() + h - start_offset);
-  for(size_t i = start_offset; i < h; i++)
-  {
-    weights.push_back(m_db->get_block_weight(i));
-  }
-  m_db->block_txn_stop();
+  weights = m_db->get_block_weights(start_offset, count);
+}
+//------------------------------------------------------------------
+void Blockchain::get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const
+{
+  LOG_PRINT_L3("Blockchain::" << __func__);
+  CRITICAL_REGION_LOCAL(m_blockchain_lock);
+
+  weights = m_db->get_long_term_block_weights(start_height, count);
 }
 //------------------------------------------------------------------
 uint64_t Blockchain::get_current_cumulative_block_weight_limit() const
@@ -3804,9 +3806,7 @@ uint64_t Blockchain::get_next_long_term_block_weight(uint64_t block_weight) cons
     return block_weight;
 
   std::vector<uint64_t> weights;
-  weights.resize(nblocks);
-  for (uint64_t h = 0; h < nblocks; ++h)
-    weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h);
+  get_long_term_block_weights(weights, db_height - nblocks, nblocks);
   uint64_t long_term_median = epee::misc_utils::median(weights);
   uint64_t long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);
 
@@ -3850,9 +3850,7 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti
       uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
       if (nblocks == db_height)
         --nblocks;
-      weights.resize(nblocks);
-      for (uint64_t h = 0; h < nblocks; ++h)
-        weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h - 1);
+      get_long_term_block_weights(weights, db_height - nblocks - 1, nblocks);
       new_weights = weights;
       long_term_median = epee::misc_utils::median(weights);
     }

src/cryptonote_core/blockchain.h

@@ -1288,13 +1288,24 @@ namespace cryptonote
     /**
      * @brief gets recent block weights for median calculation
      *
-     * get the block weights of the last <count> blocks, and return by reference <sz>.
+     * get the block weights of the last <count> blocks, and return by reference <weights>.
      *
-     * @param sz return-by-reference the list of weights
+     * @param weights return-by-reference the list of weights
      * @param count the number of blocks to get weights for
      */
     void get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_t count) const;
 
+    /**
+     * @brief gets recent block long term weights for median calculation
+     *
+     * get the block long term weights of the last <count> blocks, and return by reference <weights>.
+     *
+     * @param weights return-by-reference the list of weights
+     * @param start_height the block height of the first block to query
+     * @param count the number of blocks to get weights for
+     */
+    void get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const;
+
     /**
      * @brief checks if a transaction is unlocked (its outputs spendable)
      *

tests/block_weight/block_weight.cpp

@@ -72,6 +72,18 @@ public:
   virtual uint64_t height() const override { return blocks.size(); }
   virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; }
   virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; }
+  virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const override {
+    std::vector<uint64_t> ret;
+    ret.reserve(count);
+    while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].weight);
+    return ret;
+  }
+  virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const override {
+    std::vector<uint64_t> ret;
+    ret.reserve(count);
+    while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].long_term_weight);
+    return ret;
+  }
   virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override {
     uint64_t h = height();
     crypto::hash top = crypto::null_hash;

tests/unit_tests/long_term_block_weight.cpp

@@ -64,6 +64,18 @@ public:
   virtual uint64_t height() const override { return blocks.size(); }
   virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; }
   virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; }
+  virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const override {
+    std::vector<uint64_t> ret;
+    ret.reserve(count);
+    while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].weight);
+    return ret;
+  }
+  virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const override {
+    std::vector<uint64_t> ret;
+    ret.reserve(count);
+    while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].long_term_weight);
+    return ret;
+  }
   virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override {
     uint64_t h = height();
     crypto::hash top = crypto::null_hash;