Make difficulty 128 bit instead of 64 bit

Based on Boolberry work by:
  jahrsg <jahr@jahr.me>
  cr.zoidberg <crypto.zoidberg@gmail.com>

LICENSE

@@ -29,3 +29,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 Parts of the project are originally copyright (c) 2012-2013 The Cryptonote
 developers
+
+Parts of the project are originally copyright (c) 2014 The Boolberry
+developers, distributed under the MIT licence:
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -54,7 +54,7 @@ using epee::string_tools::pod_to_hex;
 using namespace crypto;
 
 // Increase when the DB structure changes
-#define VERSION 4
+#define VERSION 5
 
 namespace
 {
@@ -274,7 +274,7 @@ typedef struct mdb_block_info_1
   uint64_t bi_timestamp;
   uint64_t bi_coins;
   uint64_t bi_weight; // a size_t really but we need 32-bit compat
-  difficulty_type bi_diff;
+  uint64_t bi_diff;
   crypto::hash bi_hash;
 } mdb_block_info_1;
 
@@ -284,7 +284,7 @@ typedef struct mdb_block_info_2
   uint64_t bi_timestamp;
   uint64_t bi_coins;
   uint64_t bi_weight; // a size_t really but we need 32-bit compat
-  difficulty_type bi_diff;
+  uint64_t bi_diff;
   crypto::hash bi_hash;
   uint64_t bi_cum_rct;
 } mdb_block_info_2;
@@ -295,13 +295,26 @@ typedef struct mdb_block_info_3
   uint64_t bi_timestamp;
   uint64_t bi_coins;
   uint64_t bi_weight; // a size_t really but we need 32-bit compat
-  difficulty_type bi_diff;
+  uint64_t bi_diff;
   crypto::hash bi_hash;
   uint64_t bi_cum_rct;
   uint64_t bi_long_term_block_weight;
 } mdb_block_info_3;
 
-typedef mdb_block_info_3 mdb_block_info;
+typedef struct mdb_block_info_4
+{
+  uint64_t bi_height;
+  uint64_t bi_timestamp;
+  uint64_t bi_coins;
+  uint64_t bi_weight; // a size_t really but we need 32-bit compat
+  uint64_t bi_diff_lo;
+  uint64_t bi_diff_hi;
+  crypto::hash bi_hash;
+  uint64_t bi_cum_rct;
+  uint64_t bi_long_term_block_weight;
+} mdb_block_info_4;
+
+typedef mdb_block_info_4 mdb_block_info;
 
 typedef struct blk_height {
     crypto::hash bh_hash;
@@ -757,7 +770,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l
   bi.bi_timestamp = blk.timestamp;
   bi.bi_coins = coins_generated;
   bi.bi_weight = block_weight;
-  bi.bi_diff = cumulative_difficulty;
+  bi.bi_diff_hi = (cumulative_difficulty >> 64).convert_to<uint64_t>();
+  bi.bi_diff_lo = (cumulative_difficulty << 64 >> 64).convert_to<uint64_t>();
   bi.bi_hash = blk_hash;
   bi.bi_cum_rct = num_rct_outs;
   if (blk.major_version >= 4)
@@ -2527,7 +2541,9 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t&
     throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db"));
 
   mdb_block_info *bi = (mdb_block_info *)result.mv_data;
-  difficulty_type ret = bi->bi_diff;
+  difficulty_type ret = bi->bi_diff_hi;
+  ret <<= 64;
+  ret |= bi->bi_diff_lo;
   TXN_POSTFIX_RDONLY();
   return ret;
 }
@@ -5040,6 +5056,133 @@ void BlockchainLMDB::migrate_3_4()
   txn.commit();
 }
 
+void BlockchainLMDB::migrate_4_5()
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  uint64_t i;
+  int result;
+  mdb_txn_safe txn(false);
+  MDB_val k, v;
+  char *ptr;
+
+  MGINFO_YELLOW("Migrating blockchain from DB version 4 to 5 - this may take a while:");
+
+  do {
+    LOG_PRINT_L1("migrating block info:");
+
+    result = mdb_txn_begin(m_env, NULL, 0, txn);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+
+    MDB_stat db_stats;
+    if ((result = mdb_stat(txn, m_blocks, &db_stats)))
+      throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
+    const uint64_t blockchain_height = db_stats.ms_entries;
+
+    /* the block_info table name is the same but the old version and new version
+     * have incompatible data. Create a new table. We want the name to be similar
+     * to the old name so that it will occupy the same location in the DB.
+     */
+    MDB_dbi o_block_info = m_block_info;
+    lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
+    mdb_set_dupsort(txn, m_block_info, compare_uint64);
+
+
+    MDB_cursor *c_blocks;
+    result = mdb_cursor_open(txn, m_blocks, &c_blocks);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str()));
+
+    MDB_cursor *c_old, *c_cur;
+    i = 0;
+    while(1) {
+      if (!(i % 1000)) {
+        if (i) {
+          LOGIF(el::Level::Info) {
+            std::cout << i << " / " << blockchain_height << "  \r" << std::flush;
+          }
+          txn.commit();
+          result = mdb_txn_begin(m_env, NULL, 0, txn);
+          if (result)
+            throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+        }
+        result = mdb_cursor_open(txn, m_block_info, &c_cur);
+        if (result)
+          throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str()));
+        result = mdb_cursor_open(txn, o_block_info, &c_old);
+        if (result)
+          throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str()));
+        if (!i) {
+          MDB_stat db_stat;
+          result = mdb_stat(txn, m_block_info, &db_stats);
+          if (result)
+            throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str()));
+          i = db_stats.ms_entries;
+        }
+      }
+      result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT);
+      if (result == MDB_NOTFOUND) {
+        txn.commit();
+        break;
+      }
+      else if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str()));
+      const mdb_block_info_3 *bi_old = (const mdb_block_info_3*)v.mv_data;
+      mdb_block_info_4 bi;
+      bi.bi_height = bi_old->bi_height;
+      bi.bi_timestamp = bi_old->bi_timestamp;
+      bi.bi_coins = bi_old->bi_coins;
+      bi.bi_weight = bi_old->bi_weight;
+      bi.bi_diff_lo = bi_old->bi_diff;
+      bi.bi_diff_hi = 0;
+      bi.bi_hash = bi_old->bi_hash;
+      bi.bi_cum_rct = bi_old->bi_cum_rct;
+      bi.bi_long_term_block_weight = bi_old->bi_long_term_block_weight;
+
+      MDB_val_set(nv, bi);
+      result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP);
+      if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str()));
+      /* we delete the old records immediately, so the overall DB and mapsize should not grow.
+       * This is a little slower than just letting mdb_drop() delete it all at the end, but
+       * it saves a significant amount of disk space.
+       */
+      result = mdb_cursor_del(c_old, 0);
+      if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str()));
+      i++;
+    }
+
+    result = mdb_txn_begin(m_env, NULL, 0, txn);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+    /* Delete the old table */
+    result = mdb_drop(txn, o_block_info, 1);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str()));
+
+    RENAME_DB("block_infn");
+    mdb_dbi_close(m_env, m_block_info);
+
+    lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
+    mdb_set_dupsort(txn, m_block_info, compare_uint64);
+
+    txn.commit();
+  } while(0);
+
+  uint32_t version = 5;
+  v.mv_data = (void *)&version;
+  v.mv_size = sizeof(version);
+  MDB_val_str(vk, "version");
+  result = mdb_txn_begin(m_env, NULL, 0, txn);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+  result = mdb_put(txn, m_properties, &vk, &v, 0);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str()));
+  txn.commit();
+}
+
 void BlockchainLMDB::migrate(const uint32_t oldversion)
 {
   if (oldversion < 1)
@@ -5050,6 +5193,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion)
     migrate_2_3();
   if (oldversion < 4)
     migrate_3_4();
+  if (oldversion < 5)
+    migrate_4_5();
 }
 
 }  // namespace cryptonote

src/blockchain_db/lmdb/db_lmdb.h

@@ -418,6 +418,9 @@ private:
   // migrate from DB version 3 to 4
   void migrate_3_4();
 
+  // migrate from DB version 4 to 5
+  void migrate_4_5();
+
   void cleanup_batch();
 
 private:

src/blockchain_utilities/blockchain_import.cpp

@@ -294,7 +294,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
   }
 
   // 4 byte magic + (currently) 1024 byte header structures
-  bootstrap.seek_to_first_chunk(import_file);
+  uint8_t major_version, minor_version;
+  bootstrap.seek_to_first_chunk(import_file, major_version, minor_version);
 
   std::string str1;
   char buffer1[1024];
@@ -415,7 +416,23 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
     {
       str1.assign(buffer_block, chunk_size);
       bootstrap::block_package bp;
-      if (! ::serialization::parse_binary(str1, bp))
+      bool res;
+      if (major_version == 0)
+      {
+        bootstrap::block_package_1 bp1;
+        res = ::serialization::parse_binary(str1, bp1);
+        if (res)
+        {
+          bp.block = std::move(bp1.block);
+          bp.txs = std::move(bp1.txs);
+          bp.block_weight = bp1.block_weight;
+          bp.cumulative_difficulty = bp1.cumulative_difficulty;
+          bp.coins_generated = bp1.coins_generated;
+        }
+      }
+      else
+        res = ::serialization::parse_binary(str1, bp);
+      if (!res)
         throw std::runtime_error("Error in deserialization of chunk");
 
       int display_interval = 1000;

src/blockchain_utilities/bootstrap_file.cpp

@@ -124,8 +124,8 @@ bool BootstrapFile::initialize_file()
   *m_raw_data_file << blob;
 
   bootstrap::file_info bfi;
-  bfi.major_version = 0;
+  bfi.major_version = 1;
-  bfi.minor_version = 1;
+  bfi.minor_version = 0;
   bfi.header_size = header_size;
 
   bootstrap::blocks_info bbi;
@@ -323,7 +323,7 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
   return BootstrapFile::close();
 }
 
-uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file)
+uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version)
 {
   uint32_t file_magic;
 
@@ -371,6 +371,8 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file)
   uint64_t full_header_size = sizeof(file_magic) + bfi.header_size;
   import_file.seekg(full_header_size);
 
+  major_version = bfi.major_version;
+  minor_version = bfi.minor_version;
   return full_header_size;
 }
 
@@ -461,7 +463,8 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s
   }
 
   uint64_t full_header_size; // 4 byte magic + length of header structures
-  full_header_size = seek_to_first_chunk(import_file);
+  uint8_t major_version, minor_version;
+  full_header_size = seek_to_first_chunk(import_file, major_version, minor_version);
 
   MINFO("Scanning blockchain from bootstrap file...");
   bool quit = false;

src/blockchain_utilities/bootstrap_file.h

@@ -60,7 +60,7 @@ public:
   uint64_t count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit);
   uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height);
   uint64_t count_blocks(const std::string& dir_path);
-  uint64_t seek_to_first_chunk(std::ifstream& import_file);
+  uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version);
 
   bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp,
       boost::filesystem::path& output_file, uint64_t use_block_height=0);

src/blockchain_utilities/bootstrap_serialization.h

@@ -29,7 +29,7 @@
 #pragma once
 
 #include "cryptonote_basic/cryptonote_boost_serialization.h"
-#include "cryptonote_basic/difficulty.h"
+#include "serialization/difficulty_type.h"
 
 
 namespace cryptonote
@@ -66,6 +66,23 @@ namespace cryptonote
       END_SERIALIZE()
     };
 
+    struct block_package_1
+    {
+      cryptonote::block block;
+      std::vector<transaction> txs;
+      size_t block_weight;
+      uint64_t cumulative_difficulty;
+      uint64_t coins_generated;
+
+      BEGIN_SERIALIZE()
+        FIELD(block)
+        FIELD(txs)
+        VARINT_FIELD(block_weight)
+        VARINT_FIELD(cumulative_difficulty)
+        VARINT_FIELD(coins_generated)
+      END_SERIALIZE()
+    };
+
     struct block_package
     {
       cryptonote::block block;
@@ -78,7 +95,7 @@ namespace cryptonote
         FIELD(block)
         FIELD(txs)
         VARINT_FIELD(block_weight)
-        VARINT_FIELD(cumulative_difficulty)
+        FIELD(cumulative_difficulty)
         VARINT_FIELD(coins_generated)
       END_SERIALIZE()
     };

src/cryptonote_basic/cryptonote_boost_serialization.h

@@ -40,6 +40,7 @@
 #include <boost/archive/portable_binary_iarchive.hpp>
 #include <boost/archive/portable_binary_oarchive.hpp>
 #include "cryptonote_basic.h"
+#include "difficulty.h"
 #include "common/unordered_containers_boost_serialization.h"
 #include "crypto/crypto.h"
 #include "ringct/rctTypes.h"
@@ -346,6 +347,34 @@ namespace boost
     a & x.range_proof_type;
     a & x.bp_version;
   }
+
+  template <class Archive>
+  inline void serialize(Archive &a, cryptonote::difficulty_type &x, const boost::serialization::version_type ver)
+  {
+    if (Archive::is_loading::value)
+    {
+      // load high part
+      uint64_t v = 0;
+      a & v;
+      x = v;
+      // load low part
+      x = x << 64;
+      a & v;
+      x += v;
+    }
+    else
+    {
+      // store high part
+      cryptonote::difficulty_type x_ = x >> 64;
+      uint64_t v = x_.convert_to<uint64_t>();
+      a & v;
+      // store low part
+      x_ = x << 64 >> 64;
+      v = x_.convert_to<uint64_t>();
+      a & v;
+    }
+  }
+
 }
 }
 

src/cryptonote_basic/difficulty.cpp

@@ -102,7 +102,7 @@ namespace cryptonote {
     return a + b < a || (c && a + b == (uint64_t) -1);
   }
 
-  bool check_hash(const crypto::hash &hash, difficulty_type difficulty) {
+  bool check_hash_64(const crypto::hash &hash, uint64_t difficulty) {
     uint64_t low, high, top, cur;
     // First check the highest word, this will most likely fail for a random hash.
     mul(swap64le(((const uint64_t *) &hash)[3]), difficulty, top, high);
@@ -119,7 +119,7 @@ namespace cryptonote {
     return !carry;
   }
 
-  difficulty_type next_difficulty(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds) {
+  uint64_t next_difficulty_64(std::vector<std::uint64_t> timestamps, std::vector<uint64_t> cumulative_difficulties, size_t target_seconds) {
 
     if(timestamps.size() > DIFFICULTY_WINDOW)
     {
@@ -128,6 +128,87 @@ namespace cryptonote {
     }
 
 
+    size_t length = timestamps.size();
+    assert(length == cumulative_difficulties.size());
+    if (length <= 1) {
+      return 1;
+    }
+    static_assert(DIFFICULTY_WINDOW >= 2, "Window is too small");
+    assert(length <= DIFFICULTY_WINDOW);
+    sort(timestamps.begin(), timestamps.end());
+    size_t cut_begin, cut_end;
+    static_assert(2 * DIFFICULTY_CUT <= DIFFICULTY_WINDOW - 2, "Cut length is too large");
+    if (length <= DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) {
+      cut_begin = 0;
+      cut_end = length;
+    } else {
+      cut_begin = (length - (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) + 1) / 2;
+      cut_end = cut_begin + (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT);
+    }
+    assert(/*cut_begin >= 0 &&*/ cut_begin + 2 <= cut_end && cut_end <= length);
+    uint64_t time_span = timestamps[cut_end - 1] - timestamps[cut_begin];
+    if (time_span == 0) {
+      time_span = 1;
+    }
+    uint64_t total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin];
+    assert(total_work > 0);
+    uint64_t low, high;
+    mul(total_work, target_seconds, low, high);
+    // blockchain errors "difficulty overhead" if this function returns zero.
+    // TODO: consider throwing an exception instead
+    if (high != 0 || low + time_span - 1 < low) {
+      return 0;
+    }
+    return (low + time_span - 1) / time_span;
+  }
+
+#if defined(_MSC_VER)
+#ifdef max
+#undef max
+#endif
+#endif
+
+  const difficulty_type max64bit(std::numeric_limits<std::uint64_t>::max());
+  const boost::multiprecision::uint256_t max128bit(std::numeric_limits<boost::multiprecision::uint128_t>::max());
+  const boost::multiprecision::uint512_t max256bit(std::numeric_limits<boost::multiprecision::uint256_t>::max());
+
+#define FORCE_FULL_128_BITS
+
+  bool check_hash_128(const crypto::hash &hash, difficulty_type difficulty) {
+#ifndef FORCE_FULL_128_BITS
+    // fast check
+    if (difficulty >= max64bit && ((const uint64_t *) &hash)[3] > 0)
+      return false;
+#endif
+    // usual slow check
+    boost::multiprecision::uint512_t hashVal = 0;
+#ifdef FORCE_FULL_128_BITS
+    for(int i = 0; i < 4; i++) { // highest word is zero
+#else
+    for(int i = 1; i < 4; i++) { // highest word is zero
+#endif
+      hashVal <<= 64;
+      hashVal |= swap64le(((const uint64_t *) &hash)[3 - i]);
+    }
+    return hashVal * difficulty <= max256bit;
+  }
+
+  bool check_hash(const crypto::hash &hash, difficulty_type difficulty) {
+    if (difficulty <= max64bit) // if can convert to small difficulty - do it
+      return check_hash_64(hash, difficulty.convert_to<std::uint64_t>());
+    else
+      return check_hash_128(hash, difficulty);
+  }
+
+  difficulty_type next_difficulty(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds) {
+    //cutoff DIFFICULTY_LAG
+    if(timestamps.size() > DIFFICULTY_WINDOW)
+    {
+      timestamps.resize(DIFFICULTY_WINDOW);
+      cumulative_difficulties.resize(DIFFICULTY_WINDOW);
+    }
+
+
     size_t length = timestamps.size();
     assert(length == cumulative_difficulties.size());
     if (length <= 1) {
@@ -152,14 +233,10 @@ namespace cryptonote {
     }
     difficulty_type total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin];
     assert(total_work > 0);
-    uint64_t low, high;
+    boost::multiprecision::uint256_t res =  (boost::multiprecision::uint256_t(total_work) * target_seconds + time_span - 1) / time_span;
-    mul(total_work, target_seconds, low, high);
+    if(res > max128bit)
-    // blockchain errors "difficulty overhead" if this function returns zero.
+      return 0; // to behave like previous implementation, may be better return max128bit?
-    // TODO: consider throwing an exception instead
+    return res.convert_to<difficulty_type>();
-    if (high != 0 || low + time_span - 1 < low) {
-      return 0;
-    }
-    return (low + time_span - 1) / time_span;
   }
 
 }

src/cryptonote_basic/difficulty.h

@@ -32,12 +32,13 @@
 
 #include <cstdint>
 #include <vector>
+#include <boost/multiprecision/cpp_int.hpp>
 
 #include "crypto/hash.h"
 
 namespace cryptonote
 {
-    typedef std::uint64_t difficulty_type;
+    typedef boost::multiprecision::uint128_t difficulty_type;
 
     /**
      * @brief checks if a hash fits the given difficulty
@@ -51,6 +52,10 @@ namespace cryptonote
      *
      * @return true if valid, else false
      */
+    bool check_hash_64(const crypto::hash &hash, uint64_t difficulty);
+    uint64_t next_difficulty_64(std::vector<std::uint64_t> timestamps, std::vector<uint64_t> cumulative_difficulties, size_t target_seconds);
+
+    bool check_hash_128(const crypto::hash &hash, difficulty_type difficulty);
     bool check_hash(const crypto::hash &hash, difficulty_type difficulty);
     difficulty_type next_difficulty(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds);
 }

src/cryptonote_core/blockchain.cpp

@@ -1997,7 +1997,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc
   return true;
 }
 //------------------------------------------------------------------
-uint64_t Blockchain::block_difficulty(uint64_t i) const
+difficulty_type Blockchain::block_difficulty(uint64_t i) const
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   // WARNING: this function does not take m_blockchain_lock, and thus should only call read only
@@ -2196,7 +2196,11 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc
 
   bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, resp.start_height, resp.total_height);
   if (result)
-    resp.cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1);
+  {
+    cryptonote::difficulty_type wide_cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1);
+    resp.cumulative_difficulty = (wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>();
+    resp.cumulative_difficulty_top64 = (wide_cumulative_difficulty >> 64).convert_to<uint64_t>();
+  }
 
   return result;
 }

src/cryptonote_core/blockchain.h

@@ -653,7 +653,7 @@ namespace cryptonote
      *
      * @return the difficulty
      */
-    uint64_t block_difficulty(uint64_t i) const;
+    difficulty_type block_difficulty(uint64_t i) const;
 
     /**
      * @brief gets blocks based on a list of block hashes

src/cryptonote_protocol/cryptonote_protocol_defs.h

@@ -34,6 +34,7 @@
 #include "serialization/keyvalue_serialization.h"
 #include "cryptonote_basic/cryptonote_basic.h"
 #include "cryptonote_basic/blobdatatype.h"
+
 namespace cryptonote
 {
 
@@ -208,6 +209,7 @@ namespace cryptonote
   {
     uint64_t current_height;
     uint64_t cumulative_difficulty;
+    uint64_t cumulative_difficulty_top64;
     crypto::hash  top_id;
     uint8_t top_version;
     uint32_t pruning_seed;
@@ -215,6 +217,7 @@ namespace cryptonote
     BEGIN_KV_SERIALIZE_MAP()
       KV_SERIALIZE(current_height)
       KV_SERIALIZE(cumulative_difficulty)
+      KV_SERIALIZE(cumulative_difficulty_top64)
       KV_SERIALIZE_VAL_POD_AS_BLOB(top_id)
       KV_SERIALIZE_OPT(top_version, (uint8_t)0)
       KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0)
@@ -245,12 +248,14 @@ namespace cryptonote
       uint64_t start_height;
       uint64_t total_height;
       uint64_t cumulative_difficulty;
+      uint64_t cumulative_difficulty_top64;
       std::vector<crypto::hash> m_block_ids;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(start_height)
         KV_SERIALIZE(total_height)
         KV_SERIALIZE(cumulative_difficulty)
+        KV_SERIALIZE(cumulative_difficulty_top64)
         KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_ids)
       END_KV_SERIALIZE_MAP()
     };

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -398,7 +398,9 @@ namespace cryptonote
   {
     m_core.get_blockchain_top(hshd.current_height, hshd.top_id);
     hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height);
-    hshd.cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height);
+    difficulty_type wide_cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height);
+    hshd.cumulative_difficulty = (wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>();
+    hshd.cumulative_difficulty_top64 = (wide_cumulative_difficulty >> 64).convert_to<uint64_t>();
     hshd.current_height +=1;
     hshd.pruning_seed = m_core.get_blockchain_pruning_seed();
     return true;

src/rpc/core_rpc_server.cpp

@@ -70,6 +70,13 @@ namespace
   {
     return (value + quantum - 1) / quantum * quantum;
   }
+
+  void store_difficulty(cryptonote::difficulty_type difficulty, uint64_t &sdiff, std::string &swdiff, uint64_t &stop64)
+  {
+    sdiff = (difficulty << 64 >> 64).convert_to<uint64_t>();
+    swdiff = difficulty.convert_to<std::string>();
+    stop64 = (difficulty >> 64).convert_to<uint64_t>();
+  }
 }
 
 namespace cryptonote
@@ -219,7 +226,7 @@ namespace cryptonote
     ++res.height; // turn top block height into blockchain height
     res.top_block_hash = string_tools::pod_to_hex(top_hash);
     res.target_height = m_core.get_target_blockchain_height();
-    res.difficulty = m_core.get_blockchain_storage().get_difficulty_for_next_block();
+    store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), res.difficulty, res.wide_difficulty, res.difficulty_top64);
     res.target = m_core.get_blockchain_storage().get_difficulty_target();
     res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase
     res.tx_pool_size = m_core.get_pool_transactions_count();
@@ -236,7 +243,8 @@ namespace cryptonote
     res.testnet = net_type == TESTNET;
     res.stagenet = net_type == STAGENET;
     res.nettype = net_type == MAINNET ? "mainnet" : net_type == TESTNET ? "testnet" : net_type == STAGENET ? "stagenet" : "fakechain";
-    res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1);
+    store_difficulty(m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1),
+        res.cumulative_difficulty, res.wide_cumulative_difficulty, res.cumulative_difficulty_top64);
     res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit();
     res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median();
     res.status = CORE_RPC_STATUS_OK;
@@ -1196,13 +1204,15 @@ namespace cryptonote
     block b;
     cryptonote::blobdata blob_reserve;
     blob_reserve.resize(req.reserve_size, 0);
-    if(!m_core.get_block_template(b, info.address, res.difficulty, res.height, res.expected_reward, blob_reserve))
+    cryptonote::difficulty_type wdiff;
+    if(!m_core.get_block_template(b, info.address, wdiff, res.height, res.expected_reward, blob_reserve))
     {
       error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
       error_resp.message = "Internal error: failed to create block template";
       LOG_ERROR("Failed to create block template");
       return false;
     }
+    store_difficulty(wdiff, res.difficulty, res.wide_difficulty, res.difficulty_top64);
     blobdata block_blob = t_serializable_object_to_blob(b);
     crypto::public_key tx_pub_key = cryptonote::get_tx_pub_key_from_extra(b.miner_tx);
     if(tx_pub_key == crypto::null_pkey)
@@ -1375,8 +1385,10 @@ namespace cryptonote
     response.height = height;
     response.depth = m_core.get_current_blockchain_height() - height - 1;
     response.hash = string_tools::pod_to_hex(hash);
-    response.difficulty = m_core.get_blockchain_storage().block_difficulty(height);
+    store_difficulty(m_core.get_blockchain_storage().block_difficulty(height),
-    response.cumulative_difficulty = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(height);
+        response.difficulty, response.wide_difficulty, response.difficulty_top64);
+    store_difficulty(m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(height),
+        response.cumulative_difficulty, response.wide_cumulative_difficulty, response.cumulative_difficulty_top64);
     response.reward = get_block_reward(blk);
     response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height);
     response.num_txes = blk.tx_hashes.size();
@@ -1707,7 +1719,8 @@ namespace cryptonote
     ++res.height; // turn top block height into blockchain height
     res.top_block_hash = string_tools::pod_to_hex(top_hash);
     res.target_height = m_core.get_target_blockchain_height();
-    res.difficulty = m_core.get_blockchain_storage().get_difficulty_for_next_block();
+    store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(),
+        res.difficulty, res.wide_difficulty, res.difficulty_top64);
     res.target = m_core.get_blockchain_storage().get_current_hard_fork_version() < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2;
     res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase
     res.tx_pool_size = m_core.get_pool_transactions_count();
@@ -1725,7 +1738,8 @@ namespace cryptonote
     res.stagenet = net_type == STAGENET;
     res.nettype = net_type == MAINNET ? "mainnet" : net_type == TESTNET ? "testnet" : net_type == STAGENET ? "stagenet" : "fakechain";
 
-    res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1);
+    store_difficulty(m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1),
+        res.cumulative_difficulty, res.wide_cumulative_difficulty, res.cumulative_difficulty_top64);
     res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit();
     res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median();
     res.status = CORE_RPC_STATUS_OK;
@@ -1947,7 +1961,9 @@ namespace cryptonote
       std::list<std::pair<Blockchain::block_extended_info, std::vector<crypto::hash>>> chains = m_core.get_blockchain_storage().get_alternative_chains();
       for (const auto &i: chains)
       {
-        res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second.size(), i.first.cumulative_difficulty, {}, std::string()});
+        difficulty_type wdiff = i.first.cumulative_difficulty;
+        res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second.size(), 0, "", 0, {}, std::string()});
+        store_difficulty(wdiff, res.chains.back().difficulty, res.chains.back().wide_difficulty, res.chains.back().difficulty_top64);
         res.chains.back().block_hashes.reserve(i.second.size());
         for (const crypto::hash &block_id: i.second)
           res.chains.back().block_hashes.push_back(epee::string_tools::pod_to_hex(block_id));

src/rpc/core_rpc_server_commands_defs.h

@@ -943,6 +943,8 @@ namespace cryptonote
       uint64_t height;
       uint64_t target_height;
       uint64_t difficulty;
+      std::string wide_difficulty;
+      uint64_t difficulty_top64;
       uint64_t target;
       uint64_t tx_count;
       uint64_t tx_pool_size;
@@ -958,6 +960,8 @@ namespace cryptonote
       std::string nettype;
       std::string top_block_hash;
       uint64_t cumulative_difficulty;
+      std::string wide_cumulative_difficulty;
+      uint64_t cumulative_difficulty_top64;
       uint64_t block_size_limit;
       uint64_t block_weight_limit;
       uint64_t block_size_median;
@@ -978,6 +982,8 @@ namespace cryptonote
         KV_SERIALIZE(height)
         KV_SERIALIZE(target_height)
         KV_SERIALIZE(difficulty)
+        KV_SERIALIZE(wide_difficulty)
+        KV_SERIALIZE(difficulty_top64)
         KV_SERIALIZE(target)
         KV_SERIALIZE(tx_count)
         KV_SERIALIZE(tx_pool_size)
@@ -993,6 +999,8 @@ namespace cryptonote
         KV_SERIALIZE(nettype)
         KV_SERIALIZE(top_block_hash)
         KV_SERIALIZE(cumulative_difficulty)
+        KV_SERIALIZE(wide_cumulative_difficulty)
+        KV_SERIALIZE(cumulative_difficulty_top64)
         KV_SERIALIZE(block_size_limit)
         KV_SERIALIZE_OPT(block_weight_limit, (uint64_t)0)
         KV_SERIALIZE(block_size_median)
@@ -1149,6 +1157,8 @@ namespace cryptonote
     struct response_t
     {
       uint64_t difficulty;
+      std::string wide_difficulty;
+      uint64_t difficulty_top64;
       uint64_t height;
       uint64_t reserved_offset;
       uint64_t expected_reward;
@@ -1160,6 +1170,8 @@ namespace cryptonote
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(difficulty)
+        KV_SERIALIZE(wide_difficulty)
+        KV_SERIALIZE(difficulty_top64)
         KV_SERIALIZE(height)
         KV_SERIALIZE(reserved_offset)
         KV_SERIALIZE(expected_reward)
@@ -1226,8 +1238,12 @@ namespace cryptonote
       uint64_t height;
       uint64_t depth;
       std::string hash;
-      difficulty_type difficulty;
+      uint64_t difficulty;
-      difficulty_type cumulative_difficulty;
+      std::string wide_difficulty;
+      uint64_t difficulty_top64;
+      uint64_t cumulative_difficulty;
+      std::string wide_cumulative_difficulty;
+      uint64_t cumulative_difficulty_top64;
       uint64_t reward;
       uint64_t block_size;
       uint64_t block_weight;
@@ -1246,7 +1262,11 @@ namespace cryptonote
         KV_SERIALIZE(depth)
         KV_SERIALIZE(hash)
         KV_SERIALIZE(difficulty)
+        KV_SERIALIZE(wide_difficulty)
+        KV_SERIALIZE(difficulty_top64)
         KV_SERIALIZE(cumulative_difficulty)
+        KV_SERIALIZE(wide_cumulative_difficulty)
+        KV_SERIALIZE(cumulative_difficulty_top64)
         KV_SERIALIZE(reward)
         KV_SERIALIZE(block_size)
         KV_SERIALIZE_OPT(block_weight, (uint64_t)0)
@@ -2248,6 +2268,8 @@ namespace cryptonote
       uint64_t height;
       uint64_t length;
       uint64_t difficulty;
+      std::string wide_difficulty;
+      uint64_t difficulty_top64;
       std::vector<std::string> block_hashes;
       std::string main_chain_parent_block;
 
@@ -2256,6 +2278,8 @@ namespace cryptonote
         KV_SERIALIZE(height)
         KV_SERIALIZE(length)
         KV_SERIALIZE(difficulty)
+        KV_SERIALIZE(wide_difficulty)
+        KV_SERIALIZE(difficulty_top64)
         KV_SERIALIZE(block_hashes)
         KV_SERIALIZE(main_chain_parent_block)
       END_KV_SERIALIZE_MAP()

src/rpc/daemon_handler.cpp

@@ -436,7 +436,8 @@ namespace rpc
 
     auto& chain = m_core.get_blockchain_storage();
 
-    res.info.difficulty = chain.get_difficulty_for_next_block();
+    res.info.wide_difficulty = chain.get_difficulty_for_next_block();
+    res.info.difficulty = (res.info.wide_difficulty << 64 >> 64).convert_to<uint64_t>();
 
     res.info.target = chain.get_difficulty_target();
 
@@ -457,7 +458,8 @@ namespace rpc
     res.info.mainnet = m_core.get_nettype() == MAINNET;
     res.info.testnet = m_core.get_nettype() == TESTNET;
     res.info.stagenet = m_core.get_nettype() == STAGENET;
-    res.info.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1);
+    res.info.wide_cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1);
+    res.info.cumulative_difficulty = (res.info.wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>();
     res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit();
     res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median();
     res.info.start_time = (uint64_t)m_core.get_start_time();
@@ -826,7 +828,8 @@ namespace rpc
       header.reward += out.amount;
     }
 
-    header.difficulty = m_core.get_blockchain_storage().block_difficulty(header.height);
+    header.wide_difficulty = m_core.get_blockchain_storage().block_difficulty(header.height);
+    header.difficulty = (header.wide_difficulty << 64 >> 64).convert_to<uint64_t>();
 
     return true;
   }

src/rpc/message_data_structs.h

@@ -30,6 +30,7 @@
 
 #include "crypto/hash.h"
 #include "cryptonote_basic/cryptonote_basic.h"
+#include "cryptonote_basic/difficulty.h"
 #include "ringct/rctSigs.h"
 #include "rpc/rpc_handler.h"
 
@@ -165,6 +166,7 @@ namespace rpc
     uint64_t height;
     uint64_t depth;
     crypto::hash hash;
+    cryptonote::difficulty_type wide_difficulty;
     uint64_t difficulty;
     uint64_t reward;
   };
@@ -173,6 +175,7 @@ namespace rpc
   {
     uint64_t height;
     uint64_t target_height;
+    cryptonote::difficulty_type wide_difficulty;
     uint64_t difficulty;
     uint64_t target;
     uint64_t tx_count;
@@ -187,6 +190,7 @@ namespace rpc
     bool stagenet;
     std::string nettype;
     crypto::hash top_block_hash;
+    cryptonote::difficulty_type wide_cumulative_difficulty;
     uint64_t cumulative_difficulty;
     uint64_t block_size_limit;
     uint64_t block_weight_limit;

src/serialization/difficulty_type.h

@@ -0,0 +1,65 @@
+// Copyright (c) 2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include "cryptonote_basic/difficulty.h"
+#include "serialization.h"
+
+template<> struct is_basic_type<cryptonote::difficulty_type> { typedef boost::true_type type; };
+
+template <template <bool> class Archive>
+inline bool do_serialize(Archive<false>& ar, cryptonote::difficulty_type &diff)
+{
+  uint64_t hi, lo;
+  ar.serialize_varint(hi);
+  if (!ar.stream().good())
+    return false;
+  ar.serialize_varint(lo);
+  if (!ar.stream().good())
+    return false;
+  diff = hi;
+  diff <<= 64;
+  diff += lo;
+  return true;
+}
+
+template <template <bool> class Archive>
+inline bool do_serialize(Archive<true>& ar, cryptonote::difficulty_type &diff)
+{
+  if (!ar.stream().good())
+    return false;
+  const uint64_t hi = (diff >> 64).convert_to<uint64_t>();
+  const uint64_t lo = (diff << 64 >> 64).convert_to<uint64_t>();
+  ar.serialize_varint(hi);
+  ar.serialize_varint(lo);
+  if (!ar.stream().good())
+    return false;
+  return true;
+}
+

tests/difficulty/CMakeLists.txt

@@ -45,3 +45,6 @@ set_property(TARGET difficulty-tests
 add_test(
   NAME    difficulty
   COMMAND difficulty-tests "${CMAKE_CURRENT_SOURCE_DIR}/data.txt")
+add_test(
+  NAME    wide_difficulty
+  COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/wide_difficulty.py" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/gen_wide_data.py" "${CMAKE_CURRENT_BINARY_DIR}/difficulty-tests" "${CMAKE_CURRENT_BINARY_DIR}/wide_data.txt")

tests/difficulty/difficulty.cpp

@@ -43,16 +43,15 @@ using namespace std;
 
 #define DEFAULT_TEST_DIFFICULTY_TARGET        120
 
-int main(int argc, char *argv[]) {
+static int test_wide_difficulty(const char *filename)
-    if (argc != 2) {
+{
-        cerr << "Wrong arguments" << endl;
+    std::vector<uint64_t> timestamps;
-        return 1;
+    std::vector<cryptonote::difficulty_type> cumulative_difficulties;
-    }
+    fstream data(filename, fstream::in);
-    vector<uint64_t> timestamps, cumulative_difficulties;
-    fstream data(argv[1], fstream::in);
     data.exceptions(fstream::badbit);
     data.clear(data.rdstate());
-    uint64_t timestamp, difficulty, cumulative_difficulty = 0;
+    uint64_t timestamp;
+    cryptonote::difficulty_type difficulty, cumulative_difficulty = 0;
     size_t n = 0;
     while (data >> timestamp >> difficulty) {
         size_t begin, end;
@@ -63,11 +62,11 @@ int main(int argc, char *argv[]) {
             end = n - DIFFICULTY_LAG;
             begin = end - DIFFICULTY_WINDOW;
         }
-        uint64_t res = cryptonote::next_difficulty(
+        cryptonote::difficulty_type res = cryptonote::next_difficulty(
-            vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end),
+            std::vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end),
-            vector<uint64_t>(cumulative_difficulties.begin() + begin, cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET);
+            std::vector<cryptonote::difficulty_type>(cumulative_difficulties.begin() + begin, cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET);
         if (res != difficulty) {
-            cerr << "Wrong difficulty for block " << n << endl
+            cerr << "Wrong wide difficulty for block " << n << endl
                 << "Expected: " << difficulty << endl
                 << "Found: " << res << endl;
             return 1;
@@ -81,3 +80,60 @@ int main(int argc, char *argv[]) {
     }
     return 0;
 }
+
+int main(int argc, char *argv[]) {
+    if (argc < 2) {
+        cerr << "Wrong arguments" << endl;
+        return 1;
+    }
+    if (argc == 3 && strcmp(argv[1], "--wide") == 0)
+    {
+        return test_wide_difficulty(argv[2]);
+    }
+
+    vector<uint64_t> timestamps, cumulative_difficulties;
+    std::vector<cryptonote::difficulty_type> wide_cumulative_difficulties;
+    fstream data(argv[1], fstream::in);
+    data.exceptions(fstream::badbit);
+    data.clear(data.rdstate());
+    uint64_t timestamp;
+    uint64_t difficulty, cumulative_difficulty = 0;
+    cryptonote::difficulty_type wide_cumulative_difficulty = 0;
+    size_t n = 0;
+    while (data >> timestamp >> difficulty) {
+        size_t begin, end;
+        if (n < DIFFICULTY_WINDOW + DIFFICULTY_LAG) {
+            begin = 0;
+            end = min(n, (size_t) DIFFICULTY_WINDOW);
+        } else {
+            end = n - DIFFICULTY_LAG;
+            begin = end - DIFFICULTY_WINDOW;
+        }
+        uint64_t res = cryptonote::next_difficulty_64(
+            vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end),
+            std::vector<uint64_t>(cumulative_difficulties.begin() + begin, cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET);
+        if (res != difficulty) {
+            cerr << "Wrong difficulty for block " << n << endl
+                << "Expected: " << difficulty << endl
+                << "Found: " << res << endl;
+            return 1;
+        }
+        cryptonote::difficulty_type wide_res = cryptonote::next_difficulty(
+            std::vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end),
+            std::vector<cryptonote::difficulty_type>(wide_cumulative_difficulties.begin() + begin, wide_cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET);
+        if (wide_res.convert_to<uint64_t>() != res) {
+            cerr << "Wrong wide difficulty for block " << n << endl
+                << "Expected: " << res << endl
+                << "Found: " << wide_res << endl;
+            return 1;
+        }
+        timestamps.push_back(timestamp);
+        cumulative_difficulties.push_back(cumulative_difficulty += difficulty);
+        wide_cumulative_difficulties.push_back(wide_cumulative_difficulty += difficulty);
+        ++n;
+    }
+    if (!data.eof()) {
+        data.clear(fstream::badbit);
+    }
+    return 0;
+}

tests/difficulty/gen_wide_data.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import random
+
+DIFFICULTY_TARGET = 120
+DIFFICULTY_WINDOW = 720
+DIFFICULTY_LAG = 15
+DIFFICULTY_CUT = 60
+
+def difficulty():
+    times = []
+    diffs = []
+    while True:
+        if len(times) <= 1:
+            diff = 1
+        else:
+            begin = max(len(times) - DIFFICULTY_WINDOW - DIFFICULTY_LAG, 0)
+            end = min(begin + DIFFICULTY_WINDOW, len(times))
+            length = end - begin
+            assert length >= 2
+            if length <= DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT:
+                cut_begin = 0
+                cut_end = length
+            else:
+                excess = length - (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT)
+                cut_begin = (excess + 1) // 2
+                cut_end = length - excess // 2
+            assert cut_begin + 2 <= cut_end
+            wnd = times[begin:end]
+            wnd.sort()
+            dtime = wnd[cut_end - 1] - wnd[cut_begin]
+            dtime = max(dtime, 1)
+            ddiff = sum(diffs[begin + cut_begin + 1:begin + cut_end])
+            diff = (ddiff * DIFFICULTY_TARGET + dtime - 1) // dtime
+        times.append((yield diff))
+        diffs.append(diff)
+
+random.seed(1)
+time = 1000
+gen = difficulty()
+diff = next(gen)
+for i in range(100000):
+    power = 100 if i < 10000 else 100000000 if i < 500 else 1000000000000 if i < 1000 else 1000000000000000 if i < 2000 else 10000000000000000000 if i < 4000 else 1000000000000000000000000 
+    time += random.randint(-diff // power - 10, 3 * diff // power + 10)
+    print(time, diff)
+    diff = gen.send(time)

tests/difficulty/wide_difficulty.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import sys
+import subprocess
+
+python = sys.argv[1]
+py = sys.argv[2]
+c = sys.argv[3]
+data = sys.argv[4]
+
+first = python + " " + py + " > " + data
+second = [c, '--wide', data]
+
+try:
+  print('running: ', first)
+  subprocess.check_call(first, shell=True)
+  print('running: ', second)
+  subprocess.check_call(second)
+except:
+  sys.exit(1)
+

tests/hash-target.cpp

@@ -40,7 +40,7 @@ using cryptonote::check_hash;
 
 int main(int argc, char *argv[]) {
   crypto::hash h;
-  for (uint64_t diff = 1;; diff += 1 + (diff >> 8)) {
+  for (cryptonote::difficulty_type diff = 1;; diff += 1 + (diff >> 8)) {
     for (uint16_t b = 0; b < 256; b++) {
       memset(&h, b, sizeof(crypto::hash));
       if (check_hash(h, diff) != (b == 0 || diff <= 255 / b)) {
@@ -50,7 +50,7 @@ int main(int argc, char *argv[]) {
         memset(&h, 0, sizeof(crypto::hash));
         ((char *) &h)[31] = b;
         if (check_hash(h, diff) != (diff <= 255 / b)) {
-          return 1;
+          return 2;
         }
       }
     }
@@ -58,11 +58,11 @@ int main(int argc, char *argv[]) {
       uint64_t val = 0;
       for (int i = 31; i >= 0; i--) {
         val = val * 256 + 255;
-        ((char *) &h)[i] = static_cast<char>(val / diff);
+        ((char *) &h)[i] = static_cast<char>(static_cast<uint64_t>(val / diff));
-        val %= diff;
+        val %= diff.convert_to<uint64_t>();
       }
       if (check_hash(h, diff) != true) {
-        return 1;
+        return 3;
       }
       if (diff > 1) {
         for (int i = 0;; i++) {
@@ -74,7 +74,7 @@ int main(int argc, char *argv[]) {
           }
         }
         if (check_hash(h, diff) != false) {
-          return 1;
+          return 4;
         }
       }
     }

tests/performance_tests/CMakeLists.txt

@@ -31,6 +31,7 @@ set(performance_tests_sources
 
 set(performance_tests_headers
   check_tx_signature.h
+  check_hash.h
   cn_slow_hash.h
   construct_tx.h
   derive_public_key.h

tests/performance_tests/check_hash.h

@@ -0,0 +1,66 @@
+// Copyright (c) 2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include "string_tools.h"
+#include "cryptonote_basic/difficulty.h"
+
+template<uint64_t hash_target_high, uint64_t hash_target_low, uint64_t difficulty_high, uint64_t difficulty_low>
+class test_check_hash
+{
+public:
+  static const size_t loop_count = 100000;
+
+  bool init()
+  {
+    cryptonote::difficulty_type hash_target = hash_target_high;
+    hash_target = (hash_target << 64) | hash_target_low;
+    difficulty = difficulty_high;
+    difficulty = (difficulty << 64) | difficulty_low;
+    boost::multiprecision::uint256_t hash_value =  std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target;
+    ((uint64_t*)&hash)[0] = (hash_value << 64 >> 64).convert_to<uint64_t>();
+    hash_value >>= 64;
+    ((uint64_t*)&hash)[1] = (hash_value << 64 >> 64).convert_to<uint64_t>();
+    hash_value >>= 64;
+    ((uint64_t*)&hash)[2] = (hash_value << 64 >> 64).convert_to<uint64_t>();
+    hash_value >>= 64;
+    ((uint64_t*)&hash)[3] = (hash_value << 64 >> 64).convert_to<uint64_t>();
+    return true;
+  }
+
+  bool test()
+  {
+    cryptonote::check_hash_128(hash, difficulty);
+    return true;
+  }
+
+private:
+  crypto::hash hash;
+  cryptonote::difficulty_type difficulty;
+};

tests/performance_tests/main.cpp

@@ -38,6 +38,7 @@
 // tests
 #include "construct_tx.h"
 #include "check_tx_signature.h"
+#include "check_hash.h"
 #include "cn_slow_hash.h"
 #include "derive_public_key.h"
 #include "derive_secret_key.h"
@@ -181,6 +182,14 @@ int main(int argc, char** argv)
   TEST_PERFORMANCE4(filter, p, test_check_tx_signature_aggregated_bulletproofs, 2, 2, 56, 16);
   TEST_PERFORMANCE4(filter, p, test_check_tx_signature_aggregated_bulletproofs, 10, 2, 56, 16);
 
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 0, 1, 0, 1);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 0, 0xffffffffffffffff, 0, 0xffffffffffffffff);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 0, 0xffffffffffffffff, 0, 1);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 1, 0, 1, 0);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 1, 0, 0, 1);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 0xffffffffffffffff, 0xffffffffffffffff, 0, 1);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff);
+
   TEST_PERFORMANCE0(filter, p, test_is_out_to_acc);
   TEST_PERFORMANCE0(filter, p, test_is_out_to_acc_precomp);
   TEST_PERFORMANCE0(filter, p, test_generate_key_image_helper);

tests/unit_tests/CMakeLists.txt

@@ -43,6 +43,7 @@ set(unit_tests_sources
   crypto.cpp
   decompose_amount_into_digits.cpp
   device.cpp
+  difficulty.cpp
   dns_resolver.cpp
   epee_boosted_tcp_server.cpp
   epee_levin_protocol_handler_async.cpp

tests/unit_tests/difficulty.cpp

@@ -0,0 +1,68 @@
+// Copyright (c) 2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "gtest/gtest.h"
+#include "cryptonote_basic/difficulty.h"
+
+static cryptonote::difficulty_type MKDIFF(uint64_t high, uint64_t low)
+{
+  cryptonote::difficulty_type d = high;
+  d = (d << 64) | low;
+  return d;
+}
+
+static crypto::hash MKHASH(uint64_t high, uint64_t low)
+{
+  cryptonote::difficulty_type hash_target = high;
+  hash_target = (hash_target << 64) | low;
+  boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target;
+  crypto::hash h;
+  ((uint64_t*)&h)[0] = hash_value.convert_to<uint64_t>();
+  hash_value >>= 64;
+  ((uint64_t*)&h)[1] = hash_value.convert_to<uint64_t>();
+  hash_value >>= 64;
+  ((uint64_t*)&h)[2] = hash_value.convert_to<uint64_t>();
+  hash_value >>= 64;
+  ((uint64_t*)&h)[3] = hash_value.convert_to<uint64_t>();
+  return h;
+}
+
+TEST(difficulty, check_hash)
+{
+  ASSERT_TRUE(cryptonote::check_hash(MKHASH(0, 1), MKDIFF(0, 1)));
+  ASSERT_FALSE(cryptonote::check_hash(MKHASH(0, 1), MKDIFF(0, 2)));
+
+  ASSERT_TRUE(cryptonote::check_hash(MKHASH(0, 0xffffffffffffffff), MKDIFF(0, 0xffffffffffffffff)));
+  ASSERT_FALSE(cryptonote::check_hash(MKHASH(0, 0xffffffffffffffff), MKDIFF(1, 0)));
+
+  ASSERT_TRUE(cryptonote::check_hash(MKHASH(1, 1), MKDIFF(1, 1)));
+  ASSERT_FALSE(cryptonote::check_hash(MKHASH(1, 1), MKDIFF(1, 2)));
+
+  ASSERT_TRUE(cryptonote::check_hash(MKHASH(0xffffffffffffffff, 1), MKDIFF(0xffffffffffffffff, 1)));
+  ASSERT_FALSE(cryptonote::check_hash(MKHASH(0xffffffffffffffff, 1), MKDIFF(0xffffffffffffffff, 2)));
+}

tests/unit_tests/serialization.cpp

@@ -1187,3 +1187,26 @@ TEST(Serialization, portability_signed_tx)
   ASSERT_TRUE(epee::string_tools::pod_to_hex(ki1) == "d54cbd435a8d636ad9b01b8d4f3eb13bd0cf1ce98eddf53ab1617f9b763e66c0");
   ASSERT_TRUE(epee::string_tools::pod_to_hex(ki2) == "6c3cd6af97c4070a7aef9b1344e7463e29c7cd245076fdb65da447a34da3ca76");
 }
+
+TEST(Serialization, difficulty_type)
+{
+  std::vector<cryptonote::difficulty_type> v_original;
+
+  for(int i = 0; i != 100; i++)
+  {
+    v_original.push_back(cryptonote::difficulty_type("117868131154734361989189100"));
+    if(v_original.size() > 1)
+      v_original.back() *= v_original[v_original.size()-2];
+  }
+
+  std::stringstream ss;
+  boost::archive::portable_binary_oarchive a(ss);
+  a << v_original;
+
+  std::vector<cryptonote::difficulty_type> v_unserialized;
+
+  boost::archive::portable_binary_iarchive a2(ss);
+  a2 >> v_unserialized;
+
+  ASSERT_EQ(v_original, v_unserialized);
+}