Merge remote-tracking branch 'upstream/master'

README.md

@@ -195,7 +195,15 @@ See README.i18n
 
 While Monero isn't made to integrate with Tor, it can be used wrapped with torsocks, if you add --p2p-bind-ip 127.0.0.1 to the bitmonerod command line. You also want to set DNS requests to go over TCP, so they'll be routed through Tor, by setting DNS_PUBLIC=tcp. You may also disable IGD (UPnP port forwarding negotiation), which is pointless with Tor. To allow local connections from the wallet, add TORSOCKS_ALLOW_INBOUND=1. Example:
 
-DNS_PUBLIC=tcp TORSOCKS_ALLOW_INBOUND=1 torsocks bitmonerod --p2p-bind-ip 127.0.0.1 --no-igd
+`DNS_PUBLIC=tcp TORSOCKS_ALLOW_INBOUND=1 torsocks bitmonerod --p2p-bind-ip 127.0.0.1 --no-igd`
+
+TAILS ships with a very restrictive set of firewall rules. Therefore, you need to add a rule to allow this connection too, in addition to telling torsocks to allow inbound connections. Full example:
+
+`sudo iptables -I OUTPUT 2 -p tcp -d 127.0.0.1 -m tcp --dport 18081 -j ACCEPT`
+
+`DNS_PUBLIC=tcp TORSOCKS_ALLOW_INBOUND=1 torsocks ./bitmonerod --p2p-bind-ip 127.0.0.1 --no-igd --rpc-bind-ip 127.0.0.1 --data-dir /home/amnesia/Persistent/your/directory/to/the/blockchain`
+
+`./simplewallet`
 
 ## Using readline
 

contrib/epee/include/misc_log_ex.h

@@ -1441,8 +1441,16 @@ POP_WARNINGS
 #define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message)   do{if(!(expr)) {LOG_ERROR(message); return fail_ret_val;};}while(0)
 #endif
 
+#ifndef CHECK_AND_NO_ASSERT_MES_L
+#define CHECK_AND_NO_ASSERT_MES_L(expr, fail_ret_val, l, message)   do{if(!(expr)) {LOG_PRINT_L##l(message); /*LOCAL_ASSERT(expr);*/ return fail_ret_val;};}while(0)
+#endif
+
 #ifndef CHECK_AND_NO_ASSERT_MES
-#define CHECK_AND_NO_ASSERT_MES(expr, fail_ret_val, message)   do{if(!(expr)) {LOG_PRINT_L0(message); /*LOCAL_ASSERT(expr);*/ return fail_ret_val;};}while(0)
+#define CHECK_AND_NO_ASSERT_MES(expr, fail_ret_val, message) CHECK_AND_NO_ASSERT_MES_L(expr, fail_ret_val, 0, message)
+#endif
+
+#ifndef CHECK_AND_NO_ASSERT_MES_L1
+#define CHECK_AND_NO_ASSERT_MES_L1(expr, fail_ret_val, message) CHECK_AND_NO_ASSERT_MES_L(expr, fail_ret_val, 1, message)
 #endif
 
 

contrib/epee/include/net/http_server_handlers_map2.h

@@ -168,8 +168,8 @@
   response_info.m_header_info.m_content_type = " application/json"; \
   LOG_PRINT( query_info.m_URI << "[" << method_name << "] processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms", LOG_LEVEL_2);
 
-#define MAP_JON_RPC_WE(method_name, callback_f, command_type) \
+#define MAP_JON_RPC_WE_IF(method_name, callback_f, command_type, cond) \
-    else if(callback_name == method_name) \
+    else if((callback_name == method_name) && (cond)) \
 { \
   PREPARE_OBJECTS_FROM_JSON(command_type) \
   epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \
@@ -184,6 +184,8 @@
   return true;\
 }
 
+#define MAP_JON_RPC_WE(method_name, callback_f, command_type) MAP_JON_RPC_WE_IF(method_name, callback_f, command_type, true)
+
 #define MAP_JON_RPC_WERI(method_name, callback_f, command_type) \
     else if(callback_name == method_name) \
 { \

external/db_drivers/liblmdb/mdb.c

@@ -6771,8 +6771,8 @@ set1:
 				if (op == MDB_GET_BOTH || rc > 0)
 					return MDB_NOTFOUND;
 				rc = 0;
-				*data = olddata;
 			}
+			*data = olddata;
 
 		} else {
 			if (mc->mc_xcursor)
@@ -10501,8 +10501,11 @@ mdb_drop0(MDB_cursor *mc, int subs)
 
 		/* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves.
 		 * This also avoids any P_LEAF2 pages, which have no nodes.
+		 * Also if the DB doesn't have sub-DBs and has no overflow
+		 * pages, omit scanning leaves.
 		 */
-		if (mc->mc_flags & C_SUB)
+		if ((mc->mc_flags & C_SUB) ||
+			(!subs && !mc->mc_db->md_overflow_pages))
 			mdb_cursor_pop(mc);
 
 		mdb_cursor_copy(mc, &mx);
@@ -10529,6 +10532,9 @@ mdb_drop0(MDB_cursor *mc, int subs)
 							pg, omp->mp_pages);
 						if (rc)
 							goto done;
+						mc->mc_db->md_overflow_pages -= omp->mp_pages;
+						if (!mc->mc_db->md_overflow_pages && !subs)
+							break;
 					} else if (subs && (ni->mn_flags & F_SUBDATA)) {
 						mdb_xcursor_init1(mc, ni);
 						rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0);
@@ -10536,6 +10542,8 @@ mdb_drop0(MDB_cursor *mc, int subs)
 							goto done;
 					}
 				}
+				if (!subs && !mc->mc_db->md_overflow_pages)
+					goto pop;
 			} else {
 				if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0)
 					goto done;
@@ -10557,6 +10565,7 @@ mdb_drop0(MDB_cursor *mc, int subs)
 				/* no more siblings, go back to beginning
 				 * of previous level.
 				 */
+pop:
 				mdb_cursor_pop(mc);
 				mc->mc_ki[0] = 0;
 				for (i=1; i<mc->mc_snum; i++) {

src/blockchain_db/blockchain_db.cpp

@@ -82,14 +82,17 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti
     }
   }
 
-  add_transaction_data(blk_hash, tx, tx_hash);
+  uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash);
+
+  std::vector<uint64_t> amount_output_indices;
 
   // iterate tx.vout using indices instead of C++11 foreach syntax because
   // we need the index
   for (uint64_t i = 0; i < tx.vout.size(); ++i)
   {
-    add_output(tx_hash, tx.vout[i], i, tx.unlock_time);
+    amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time));
   }
+  add_tx_amount_output_indices(tx_id, amount_output_indices);
 }
 
 uint64_t BlockchainDB::add_block( const block& blk

src/blockchain_db/blockchain_db.h

@@ -59,6 +59,39 @@
  * Unspent transaction outputs are duplicated to quickly gather random
  * outputs to use for mixins
  *
+ * Indices and Identifiers:
+ * The word "index" is used ambiguously throughout this code. It is
+ * particularly confusing when talking about the output or transaction
+ * tables since their indexing can refer to themselves or each other.
+ * I have attempted to clarify these usages here:
+ *
+ * Blocks, transactions, and outputs are all identified by a hash.
+ * For storage efficiency, a 64-bit integer ID is used instead of the hash
+ * inside the DB. Tables exist to map between hash and ID. A block ID is
+ * also referred to as its "height". Transactions and outputs generally are
+ * not referred to by ID outside of this module, but the tx ID is returned
+ * by tx_exists() and used by get_tx_amount_output_indices(). Like their
+ * corresponding hashes, IDs are globally unique.
+ *
+ * The remaining uses of the word "index" refer to local offsets, and are
+ * not globally unique. An "amount output index" N refers to the Nth output
+ * of a specific amount. An "output local index" N refers to the Nth output
+ * of a specific tx.
+ *
+ * Exceptions:
+ *   DB_ERROR -- generic
+ *   DB_OPEN_FAILURE
+ *   DB_CREATE_FAILURE
+ *   DB_SYNC_FAILURE
+ *   BLOCK_DNE
+ *   BLOCK_PARENT_DNE
+ *   BLOCK_EXISTS
+ *   BLOCK_INVALID -- considering making this multiple errors
+ *   TX_DNE
+ *   TX_EXISTS
+ *   OUTPUT_DNE
+ *   OUTPUT_EXISTS
+ *   KEY_IMAGE_EXISTS
  */
 
 namespace cryptonote
@@ -80,6 +113,15 @@ struct output_data_t
 };
 #pragma pack(pop)
 
+#pragma pack(push, 1)
+struct tx_data_t
+{
+  uint64_t tx_id;
+  uint64_t unlock_time;
+  uint64_t block_id;
+};
+#pragma pack(pop)
+
 /***********************************
  * Exception Definitions
  ***********************************/
@@ -311,14 +353,18 @@ private:
    * and the other data passed here, not the separate outputs of the
    * transaction.
    *
+   * It returns a tx ID, which is a mapping from the tx_hash. The tx ID
+   * is used in #add_tx_amount_output_indices().
+   *
    * If any of this cannot be done, the subclass should throw the corresponding
    * subclass of DB_EXCEPTION
    *
    * @param blk_hash the hash of the block containing the transaction
    * @param tx the transaction to be added
    * @param tx_hash the hash of the transaction
+   * @return the transaction ID
    */
-  virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0;
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0;
 
   /**
    * @brief remove data about a transaction
@@ -348,6 +394,9 @@ private:
    * future, this tracking (of the number, at least) should be moved to
    * this class, as it is necessary and the same among all BlockchainDB.
    *
+   * It returns an amount output index, which is the index of the output
+   * for its specified amount.
+   *
    * This data should be stored in such a manner that the only thing needed to
    * reverse the process is the tx_out.
    *
@@ -358,25 +407,24 @@ private:
    * @param tx_output the output
    * @param local_index index of the output in its transaction
    * @param unlock_time unlock time/height of the output
+   * @return amount output index
    */
-  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0;
+  virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0;
 
   /**
-   * @brief remove an output
+   * @brief store amount output indices for a tx's outputs
    *
-   * The subclass implementing this will remove all output data it stored
+   * The subclass implementing this will add the amount output indices to its
-   * in add_output().
+   * backing store in a suitable manner. The tx_id will be the same one that
-   *
+   * was returned from #add_output().
-   * In addition, the subclass is responsible for correctly decrementing
-   * its global output counter (this may be automatic for some, such as using
-   * a database backend "count" feature).
    *
    * If any of this cannot be done, the subclass should throw the corresponding
    * subclass of DB_EXCEPTION
    *
-   * @param tx_output the output to be removed
+   * @param tx_id ID of the transaction containing these outputs
+   * @param amount_output_indices the amount output indices of the transaction
    */
-  virtual void remove_output(const tx_out& tx_output) = 0;
+  virtual void add_tx_amount_output_indices(const uint64_t tx_id, const std::vector<uint64_t>& amount_output_indices) = 0;
 
   /**
    * @brief store a spent key
@@ -414,18 +462,6 @@ private:
    */
   void pop_block();
 
-  /**
-   * @brief helper function for add_transactions, to add each individual transaction
-   *
-   * This function is called by add_transactions() for each transaction to be
-   * added.
-   *
-   * @param blk_hash hash of the block which has the transaction
-   * @param tx the transaction to add
-   * @param tx_hash_ptr the hash of the transaction, if already calculated
-   */
-  void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL);
-
   // helper function to remove transaction from blockchain
   /**
    * @brief helper function to remove transaction from the blockchain
@@ -444,6 +480,18 @@ private:
 
 protected:
 
+  /**
+   * @brief helper function for add_transactions, to add each individual transaction
+   *
+   * This function is called by add_transactions() for each transaction to be
+   * added.
+   *
+   * @param blk_hash hash of the block which has the transaction
+   * @param tx the transaction to add
+   * @param tx_hash_ptr the hash of the transaction, if already calculated
+   */
+  void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL);
+
   mutable uint64_t time_tx_exists = 0;  //!< a performance metric
   uint64_t time_commit1 = 0;  //!< a performance metric
   bool m_auto_remove_logs = true;  //!< whether or not to automatically remove old logs
@@ -930,10 +978,12 @@ public:
    * given hash and return true if so, false otherwise.
    *
    * @param h the hash to check against
+   * @param tx_id (optional) returns the tx_id for the tx hash
    *
    * @return true if the transaction exists, otherwise false
    */
   virtual bool tx_exists(const crypto::hash& h) const = 0;
+  virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_id) const = 0;
 
   // return unlock time of tx with hash <h>
   /**
@@ -1124,37 +1174,21 @@ public:
    */
   virtual bool can_thread_bulk_indices() const = 0;
 
-  /**
-   * @brief gets output indices (global) for a transaction's outputs
-   *
-   * The subclass should fetch the global output indices for each output
-   * in the transaction with the given hash.
-   *
-   * If the transaction does not exist, the subclass should throw TX_DNE.
-   *
-   * If an output cannot be found, the subclass should throw OUTPUT_DNE.
-   *
-   * @param h a transaction hash
-   *
-   * @return a list of global output indices
-   */
-  virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const = 0;
-
   /**
    * @brief gets output indices (amount-specific) for a transaction's outputs
    *
    * The subclass should fetch the amount-specific output indices for each
-   * output in the transaction with the given hash.
+   * output in the transaction with the given ID.
    *
    * If the transaction does not exist, the subclass should throw TX_DNE.
    *
    * If an output cannot be found, the subclass should throw OUTPUT_DNE.
    *
-   * @param h a transaction hash
+   * @param tx_id a transaction ID
    *
    * @return a list of amount-specific output indices
    */
-  virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const = 0;
+  virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const = 0;
 
   /**
    * @brief check if a key image is stored as spent

src/blockchain_db/lmdb/db_lmdb.h

@@ -43,20 +43,13 @@ typedef struct mdb_txn_cursors
 {
   MDB_cursor *m_txc_blocks;
   MDB_cursor *m_txc_block_heights;
-  MDB_cursor *m_txc_block_hashes;
+  MDB_cursor *m_txc_block_info;
-  MDB_cursor *m_txc_block_timestamps;
-  MDB_cursor *m_txc_block_sizes;
-  MDB_cursor *m_txc_block_diffs;
-  MDB_cursor *m_txc_block_coins;
 
   MDB_cursor *m_txc_output_txs;
-  MDB_cursor *m_txc_output_indices;
   MDB_cursor *m_txc_output_amounts;
-  MDB_cursor *m_txc_output_keys;
 
   MDB_cursor *m_txc_txs;
-  MDB_cursor *m_txc_tx_heights;
+  MDB_cursor *m_txc_tx_indices;
-  MDB_cursor *m_txc_tx_unlocks;
   MDB_cursor *m_txc_tx_outputs;
 
   MDB_cursor *m_txc_spent_keys;
@@ -66,18 +59,11 @@ typedef struct mdb_txn_cursors
 
 #define m_cur_blocks	m_cursors->m_txc_blocks
 #define m_cur_block_heights	m_cursors->m_txc_block_heights
-#define m_cur_block_hashes	m_cursors->m_txc_block_hashes
+#define m_cur_block_info	m_cursors->m_txc_block_info
-#define m_cur_block_timestamps	m_cursors->m_txc_block_timestamps
-#define m_cur_block_sizes	m_cursors->m_txc_block_sizes
-#define m_cur_block_diffs	m_cursors->m_txc_block_diffs
-#define m_cur_block_coins	m_cursors->m_txc_block_coins
 #define m_cur_output_txs	m_cursors->m_txc_output_txs
-#define m_cur_output_indices	m_cursors->m_txc_output_indices
 #define m_cur_output_amounts	m_cursors->m_txc_output_amounts
-#define m_cur_output_keys	m_cursors->m_txc_output_keys
 #define m_cur_txs	m_cursors->m_txc_txs
-#define m_cur_tx_heights	m_cursors->m_txc_tx_heights
+#define m_cur_tx_indices	m_cursors->m_txc_tx_indices
-#define m_cur_tx_unlocks	m_cursors->m_txc_tx_unlocks
 #define m_cur_tx_outputs	m_cursors->m_txc_tx_outputs
 #define m_cur_spent_keys	m_cursors->m_txc_spent_keys
 #define m_cur_hf_versions	m_cursors->m_txc_hf_versions
@@ -87,18 +73,11 @@ typedef struct mdb_rflags
   bool m_rf_txn;
   bool m_rf_blocks;
   bool m_rf_block_heights;
-  bool m_rf_block_hashes;
+  bool m_rf_block_info;
-  bool m_rf_block_timestamps;
-  bool m_rf_block_sizes;
-  bool m_rf_block_diffs;
-  bool m_rf_block_coins;
   bool m_rf_output_txs;
-  bool m_rf_output_indices;
   bool m_rf_output_amounts;
-  bool m_rf_output_keys;
   bool m_rf_txs;
-  bool m_rf_tx_heights;
+  bool m_rf_tx_indices;
-  bool m_rf_tx_unlocks;
   bool m_rf_tx_outputs;
   bool m_rf_spent_keys;
   bool m_rf_hf_versions;
@@ -223,6 +202,7 @@ public:
   virtual uint64_t height() const;
 
   virtual bool tx_exists(const crypto::hash& h) const;
+  virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const;
 
   virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const;
 
@@ -246,10 +226,8 @@ public:
 
   virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index);
   virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices);
-  virtual void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &indices);
 
-  virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const;
+  virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const;
-  virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const;
 
   virtual bool has_key_image(const crypto::key_image& img) const;
 
@@ -306,18 +284,23 @@ private:
 
   virtual void remove_block();
 
-  virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash);
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash);
 
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
 
-  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time);
+  virtual uint64_t add_output(const crypto::hash& tx_hash,
+      const tx_out& tx_output,
+      const uint64_t& local_index,
+      const uint64_t unlock_time
+      );
 
-  virtual void remove_output(const tx_out& tx_output);
+  virtual void add_tx_amount_output_indices(const uint64_t tx_id,
+      const std::vector<uint64_t>& amount_output_indices
+      );
 
-  void remove_tx_outputs(const MDB_val *tx_hash, const transaction& tx);
+  void remove_tx_outputs(const uint64_t tx_id, const transaction& tx);
 
-  void remove_output(const MDB_val *out_index, const uint64_t amount);
+  void remove_output(const uint64_t amount, const uint64_t& out_index);
-  void remove_amount_output_index(const uint64_t amount, const MDB_val *global_output_index);
 
   virtual void add_spent_key(const crypto::key_image& k_image);
 
@@ -349,16 +332,6 @@ private:
    */
   tx_out output_from_blob(const blobdata& blob) const;
 
-  /**
-   * @brief get the global index of the index-th output of the given amount
-   *
-   * @param amount the output amount
-   * @param index the index into the set of outputs of that amount
-   *
-   * @return the global index of the desired output
-   */
-  uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index);
-
   void check_open() const;
 
   virtual bool is_read_only() const;
@@ -366,25 +339,24 @@ private:
   // fix up anything that may be wrong due to past bugs
   virtual void fixup();
 
+  // migrate from older DB version to current
+  void migrate(const uint32_t oldversion);
+
+  // migrate from DB version 0 to 1
+  void migrate_0_1();
+
   MDB_env* m_env;
 
   MDB_dbi m_blocks;
   MDB_dbi m_block_heights;
-  MDB_dbi m_block_hashes;
+  MDB_dbi m_block_info;
-  MDB_dbi m_block_timestamps;
-  MDB_dbi m_block_sizes;
-  MDB_dbi m_block_diffs;
-  MDB_dbi m_block_coins;
 
   MDB_dbi m_txs;
-  MDB_dbi m_tx_unlocks;
+  MDB_dbi m_tx_indices;
-  MDB_dbi m_tx_heights;
   MDB_dbi m_tx_outputs;
 
   MDB_dbi m_output_txs;
-  MDB_dbi m_output_indices;
   MDB_dbi m_output_amounts;
-  MDB_dbi m_output_keys;
 
   MDB_dbi m_spent_keys;
 
@@ -394,6 +366,7 @@ private:
   MDB_dbi m_properties;
 
   uint64_t m_height;
+  uint64_t m_num_txs;
   uint64_t m_num_outputs;
   mutable uint64_t m_cum_size;	// used in batch size estimation
   mutable int m_cum_count;

src/blockchain_utilities/cn_deserialize.cpp

@@ -27,6 +27,7 @@
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "cryptonote_core/cryptonote_basic.h"
+#include "cryptonote_core/tx_extra.h"
 #include "cryptonote_core/blockchain.h"
 #include "blockchain_utilities.h"
 #include "common/command_line.h"
@@ -140,6 +141,7 @@ int main(int argc, char* argv[])
 
   cryptonote::block block;
   cryptonote::transaction tx;
+  std::vector<cryptonote::tx_extra_field> fields;
   if (cryptonote::parse_and_validate_block_from_blob(blob, block))
   {
     std::cout << "Parsed block:" << std::endl;
@@ -149,6 +151,25 @@ int main(int argc, char* argv[])
   {
     std::cout << "Parsed transaction:" << std::endl;
     std::cout << cryptonote::obj_to_json_str(tx) << std::endl;
+
+    if (cryptonote::parse_tx_extra(tx.extra, fields))
+    {
+      std::cout << "tx_extra has " << fields.size() << " field(s)" << std::endl;
+      for (size_t n = 0; n < fields.size(); ++n)
+      {
+        std::cout << "field " << n << ": ";
+        if (typeid(cryptonote::tx_extra_padding) == fields[n].type()) std::cout << "extra padding: " << boost::get<cryptonote::tx_extra_padding>(fields[n]).size << " bytes";
+        else if (typeid(cryptonote::tx_extra_pub_key) == fields[n].type()) std::cout << "extra pub key: " << boost::get<cryptonote::tx_extra_pub_key>(fields[n]).pub_key;
+        else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << boost::get<cryptonote::tx_extra_nonce>(fields[n]).nonce;
+        else if (typeid(cryptonote::tx_extra_merge_mining_tag) == fields[n].type()) std::cout << "extra merge mining tag: depth " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).depth << ", merkle root " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).merkle_root;
+        else std::cout << "unknown";
+        std::cout << std::endl;
+      }
+    }
+    else
+    {
+      std::cout << "Failed to parse tx_extra" << std::endl;
+    }
   }
   else
   {

src/crypto/crypto.cpp

@@ -83,7 +83,7 @@ namespace crypto {
   /* generate a random 32-byte (256-bit) integer and copy it to res */
   static inline void random_scalar(ec_scalar &res) {
     unsigned char tmp[64];
-    generate_random_bytes(64, tmp);
+    generate_random_bytes_not_thread_safe(64, tmp);
     sc_reduce(tmp);
     memcpy(&res, tmp, 32);
   }

src/crypto/crypto.h

@@ -117,13 +117,20 @@ namespace crypto {
       const public_key *const *, std::size_t, const signature *);
   };
 
+  /* Generate N random bytes
+   */
+  inline void rand(size_t N, uint8_t *bytes) {
+    boost::lock_guard<boost::mutex> lock(random_lock);
+    generate_random_bytes_not_thread_safe(N, bytes);
+  }
+
   /* Generate a value filled with random bytes.
    */
   template<typename T>
   typename std::enable_if<std::is_pod<T>::value, T>::type rand() {
     typename std::remove_cv<T>::type res;
     boost::lock_guard<boost::mutex> lock(random_lock);
-    generate_random_bytes(sizeof(T), &res);
+    generate_random_bytes_not_thread_safe(sizeof(T), &res);
     return res;
   }
 

src/crypto/random.c

@@ -113,7 +113,7 @@ INITIALIZER(init_random) {
 #endif
 }
 
-void generate_random_bytes(size_t n, void *result) {
+void generate_random_bytes_not_thread_safe(size_t n, void *result) {
 #if !defined(NDEBUG)
   assert(curstate == 1);
   curstate = 2;

src/crypto/random.h

@@ -32,4 +32,4 @@
 
 #include <stddef.h>
 
-void generate_random_bytes(size_t n, void *result);
+void generate_random_bytes_not_thread_safe(size_t n, void *result);

src/cryptonote_core/blockchain.cpp

@@ -1971,14 +1971,15 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
-  if (!m_db->tx_exists(tx_id))
+  uint64_t tx_index;
+  if (!m_db->tx_exists(tx_id, tx_index))
   {
     LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id);
     return false;
   }
 
   // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts
-  indexs = m_db->get_tx_amount_output_indices(tx_id);
+  indexs = m_db->get_tx_amount_output_indices(tx_index);
   CHECK_AND_ASSERT_MES(indexs.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty");
 
   return true;

src/cryptonote_core/cryptonote_format_utils.cpp

@@ -291,14 +291,14 @@ namespace cryptonote
     {
       tx_extra_field field;
       bool r = ::do_serialize(ar, field);
-      CHECK_AND_NO_ASSERT_MES(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size())));
+      CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size())));
       tx_extra_fields.push_back(field);
 
       std::ios_base::iostate state = iss.rdstate();
       eof = (EOF == iss.peek());
       iss.clear(state);
     }
-    CHECK_AND_NO_ASSERT_MES(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size())));
+    CHECK_AND_NO_ASSERT_MES_L1(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size())));
 
     return true;
   }
@@ -357,7 +357,7 @@ namespace cryptonote
     {
       tx_extra_field field;
       bool r = ::do_serialize(ar, field);
-      CHECK_AND_NO_ASSERT_MES(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size())));
+      CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size())));
       if (field.type() != typeid(tx_extra_nonce))
         ::do_serialize(newar, field);
 
@@ -365,7 +365,7 @@ namespace cryptonote
       eof = (EOF == iss.peek());
       iss.clear(state);
     }
-    CHECK_AND_NO_ASSERT_MES(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size())));
+    CHECK_AND_NO_ASSERT_MES_L1(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size())));
     tx_extra.clear();
     std::string s = oss.str();
     tx_extra.reserve(s.size());

src/cryptonote_core/cryptonote_format_utils.h

@@ -44,7 +44,7 @@ namespace cryptonote
   crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx);
   bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash);
   bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx);
-  bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 1, uint8_t hard_fork_version = 1);
+  bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1);
   bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key);
   bool decrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key);
 

src/rpc/core_rpc_server.h

@@ -104,12 +104,12 @@ namespace cryptonote
         MAP_JON_RPC_WE("getblockheaderbyhash",   on_get_block_header_by_hash,   COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH)
         MAP_JON_RPC_WE("getblockheaderbyheight", on_get_block_header_by_height, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT)
         MAP_JON_RPC_WE("getblock",                on_get_block,                 COMMAND_RPC_GET_BLOCK)
-        MAP_JON_RPC_WE("get_connections",        on_get_connections,            COMMAND_RPC_GET_CONNECTIONS)
+        MAP_JON_RPC_WE_IF("get_connections",     on_get_connections,            COMMAND_RPC_GET_CONNECTIONS, !m_restricted)
         MAP_JON_RPC_WE("get_info",               on_get_info_json,              COMMAND_RPC_GET_INFO)
         MAP_JON_RPC_WE("hard_fork_info",         on_hard_fork_info,             COMMAND_RPC_HARD_FORK_INFO)
-        MAP_JON_RPC_WE("setbans",                on_set_bans,                   COMMAND_RPC_SETBANS)
+        MAP_JON_RPC_WE_IF("setbans",             on_set_bans,                   COMMAND_RPC_SETBANS, !m_restricted)
-        MAP_JON_RPC_WE("getbans",                on_get_bans,                   COMMAND_RPC_GETBANS)
+        MAP_JON_RPC_WE_IF("getbans",             on_get_bans,                   COMMAND_RPC_GETBANS, !m_restricted)
-        MAP_JON_RPC_WE("flush_txpool",           on_flush_txpool,               COMMAND_RPC_FLUSH_TRANSACTION_POOL)
+        MAP_JON_RPC_WE_IF("flush_txpool",        on_flush_txpool,               COMMAND_RPC_FLUSH_TRANSACTION_POOL, !m_restricted)
         MAP_JON_RPC_WE("get_output_histogram",   on_get_output_histogram,       COMMAND_RPC_GET_OUTPUT_HISTOGRAM)
       END_JSON_RPC_MAP()
     END_URI_MAP2()

src/simplewallet/simplewallet.cpp

@@ -1010,6 +1010,7 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m
 
   m_wallet.reset(new tools::wallet2(testnet));
   m_wallet->callback(this);
+  m_wallet->set_refresh_from_block_height(field_scan_from_height);
 
   try
   {
@@ -1055,8 +1056,6 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m
     return false;
   }
 
-  m_wallet->set_refresh_from_block_height(field_scan_from_height);
-
   wallet_file = m_wallet_file;
 
   return r;
@@ -1175,7 +1174,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
         return false;
       }
     }
-    if (!m_restore_height)
+    if (!m_restore_height && m_generate_new.empty())
     {
       std::string heightstr = command_line::input_line("Restore from specific blockchain height (optional, default 0): ");
       if (std::cin.eof())
@@ -1451,6 +1450,16 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
   m_wallet->callback(this);
   m_wallet->set_seed_language(mnemonic_language);
 
+  // for a totally new account, we don't care about older blocks.
+  if (!m_generate_new.empty())
+  {
+    std::string err;
+    m_wallet->set_refresh_from_block_height(get_daemon_blockchain_height(err));
+  } else if (m_restore_height)
+  {
+    m_wallet->set_refresh_from_block_height(m_restore_height);
+  }
+
   crypto::secret_key recovery_val;
   try
   {
@@ -1466,15 +1475,6 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
   }
 
   m_wallet->init(m_daemon_address);
-  // for a totally new account, we don't care about older blocks.
-  if (!m_restore_deterministic_wallet)
-  {
-    std::string err;
-    m_wallet->set_refresh_from_block_height(get_daemon_blockchain_height(err));
-  } else if (m_restore_height)
-  {
-    m_wallet->set_refresh_from_block_height(m_restore_height);
-  }
 
   // convert rng value to electrum-style word list
   std::string electrum_words;
@@ -1507,6 +1507,8 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
 
   m_wallet.reset(new tools::wallet2(testnet));
   m_wallet->callback(this);
+  if (m_restore_height)
+    m_wallet->set_refresh_from_block_height(m_restore_height);
 
   try
   {
@@ -1522,7 +1524,6 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
   }
 
   m_wallet->init(m_daemon_address);
-  m_wallet->set_refresh_from_block_height(m_restore_height);
 
   return true;
 }
@@ -1534,6 +1535,8 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
 
   m_wallet.reset(new tools::wallet2(testnet));
   m_wallet->callback(this);
+  if (m_restore_height)
+    m_wallet->set_refresh_from_block_height(m_restore_height);
 
   try
   {
@@ -1548,7 +1551,6 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
   }
 
   m_wallet->init(m_daemon_address);
-  m_wallet->set_refresh_from_block_height(m_restore_height);
 
   return true;
 }

src/wallet/wallet2.cpp

@@ -519,7 +519,8 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry
     LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " <<  miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms");
   }else
   {
-    LOG_PRINT_L2( "Skipped block by timestamp, height: " << height << ", block time " << b.timestamp << ", account time " << m_account.get_createtime());
+    if (!(height % 100))
+      LOG_PRINT_L2( "Skipped block by timestamp, height: " << height << ", block time " << b.timestamp << ", account time " << m_account.get_createtime());
   }
   m_blockchain.push_back(bl_id);
   ++m_local_bc_height;
@@ -796,7 +797,8 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height,
 {
   std::list<crypto::hash> hashes;
   size_t current_index = m_blockchain.size();
-  while(current_index < stop_height)
+
+  while(m_run.load(std::memory_order_relaxed) && current_index < stop_height)
   {
     pull_hashes(0, blocks_start_height, short_chain_history, hashes);
     if (hashes.size() < 3)
@@ -823,7 +825,8 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height,
     {
       if(current_index >= m_blockchain.size())
       {
-        LOG_PRINT_L2( "Skipped block by height: " << current_index);
+        if (!(current_index % 1000))
+          LOG_PRINT_L2( "Skipped block by height: " << current_index);
         m_blockchain.push_back(bl_id);
         ++m_local_bc_height;
 
@@ -860,6 +863,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re
 
   // pull the first set of blocks
   get_short_chain_history(short_chain_history);
+  m_run.store(true, std::memory_order_relaxed);
   if (start_height > m_blockchain.size() || m_refresh_from_block_height > m_blockchain.size()) {
     if (!start_height)
       start_height = m_refresh_from_block_height;
@@ -877,7 +881,6 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re
   // subsequent pulls in this refresh.
   start_height = 0;
 
-  m_run.store(true, std::memory_order_relaxed);
   while(m_run.load(std::memory_order_relaxed))
   {
     try
@@ -1057,6 +1060,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const std::string& p
   value2.SetInt(m_refresh_type);
   json.AddMember("refresh_type", value2, json.GetAllocator());
 
+  value2.SetUint64(m_refresh_from_block_height);
+  json.AddMember("refresh_height", value2, json.GetAllocator());
+
   // Serialize the JSON object
   rapidjson::StringBuffer buffer;
   rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
@@ -1163,6 +1169,9 @@ bool wallet2::load_keys(const std::string& keys_file_name, const std::string& pa
       else
         LOG_PRINT_L0("Unknown refresh-type value (" << field_refresh_type << "), using default");
     }
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, refresh_height, uint64_t, Uint64, false);
+    if (field_refresh_height_found)
+      m_refresh_from_block_height = field_refresh_height;
   }
 
   const cryptonote::account_keys& keys = m_account.get_keys();

src/wallet/wallet2.h

@@ -309,6 +309,7 @@ namespace tools
     template <class t_archive>
     inline void serialize(t_archive &a, const unsigned int ver)
     {
+      uint64_t dummy_refresh_height = 0; // moved to keys file
       if(ver < 5)
         return;
       a & m_blockchain;
@@ -329,7 +330,7 @@ namespace tools
       a & m_confirmed_txs;
       if(ver < 11)
         return;
-      a & m_refresh_from_block_height;
+      a & dummy_refresh_height;
       if(ver < 12)
         return;
       a & m_tx_notes;

tests/core_tests/chaingen.cpp

@@ -222,7 +222,7 @@ bool test_generator::construct_block_manually(block& blk, const block& prev_bloc
   blk.timestamp     = actual_params & bf_timestamp ? timestamp : prev_block.timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN; // Keep difficulty unchanged
   blk.prev_id       = actual_params & bf_prev_id   ? prev_id   : get_block_hash(prev_block);
   blk.tx_hashes     = actual_params & bf_tx_hashes ? tx_hashes : std::vector<crypto::hash>();
-  max_outs          = actual_params & bf_max_outs ? max_outs : 1;
+  max_outs          = actual_params & bf_max_outs ? max_outs : 9999;
 
   size_t height = get_block_height(prev_block) + 1;
   uint64_t already_generated_coins = get_already_generated_coins(prev_block);

tests/core_tests/chaingen.h

@@ -241,7 +241,7 @@ public:
     const cryptonote::account_base& miner_acc, int actual_params = bf_none, uint8_t major_ver = 0,
     uint8_t minor_ver = 0, uint64_t timestamp = 0, const crypto::hash& prev_id = crypto::hash(),
     const cryptonote::difficulty_type& diffic = 1, const cryptonote::transaction& miner_tx = cryptonote::transaction(),
-    const std::vector<crypto::hash>& tx_hashes = std::vector<crypto::hash>(), size_t txs_sizes = 0, size_t max_outs = 0);
+    const std::vector<crypto::hash>& tx_hashes = std::vector<crypto::hash>(), size_t txs_sizes = 0, size_t max_outs = 999);
   bool construct_block_manually_tx(cryptonote::block& blk, const cryptonote::block& prev_block,
     const cryptonote::account_base& miner_acc, const std::vector<crypto::hash>& tx_hashes, size_t txs_size);
 

tests/core_tests/chaingen_main.cpp

@@ -168,9 +168,9 @@ int main(int argc, char* argv[])
 
     GENERATE_AND_PLAY(gen_v2_tx_mixable_0_mixin);
     GENERATE_AND_PLAY(gen_v2_tx_mixable_low_mixin);
-    GENERATE_AND_PLAY(gen_v2_tx_unmixable_only);
+//    GENERATE_AND_PLAY(gen_v2_tx_unmixable_only);
-    GENERATE_AND_PLAY(gen_v2_tx_unmixable_one);
+//    GENERATE_AND_PLAY(gen_v2_tx_unmixable_one);
-    GENERATE_AND_PLAY(gen_v2_tx_unmixable_two);
+//    GENERATE_AND_PLAY(gen_v2_tx_unmixable_two);
     GENERATE_AND_PLAY(gen_v2_tx_dust);
 
     std::cout << (failed_tests.empty() ? concolor::green : concolor::magenta);

tests/core_tests/v2_tests.cpp

@@ -38,7 +38,7 @@ using namespace cryptonote;
 //----------------------------------------------------------------------------------------------------------------------
 // Tests
 
-bool gen_v2_tx_validation_base::generate_with(std::vector<test_event_entry>& events, const int *out_idx, int mixin, uint64_t amount_paid, size_t max_outs, bool valid) const
+bool gen_v2_tx_validation_base::generate_with(std::vector<test_event_entry>& events, const int *out_idx, int mixin, uint64_t amount_paid, bool valid) const
 {
   uint64_t ts_start = 1338224400;
 
@@ -52,9 +52,9 @@ bool gen_v2_tx_validation_base::generate_with(std::vector<test_event_entry>& eve
   for (size_t n = 0; n < 4; ++n) {
     miner_accounts[n].generate();
     CHECK_AND_ASSERT_MES(generator.construct_block_manually(blocks[n], *prev_block, miner_accounts[n],
-        test_generator::bf_major_ver | test_generator::bf_minor_ver | test_generator::bf_timestamp | test_generator::bf_max_outs,
+        test_generator::bf_major_ver | test_generator::bf_minor_ver | test_generator::bf_timestamp,
         2, 2, prev_block->timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN * 2, // v2 has blocks twice as long
-          crypto::hash(), 0, transaction(), std::vector<crypto::hash>(), 0, max_outs),
+          crypto::hash(), 0, transaction(), std::vector<crypto::hash>(), 0, 0),
         false, "Failed to generate block");
     events.push_back(blocks[n]);
     prev_block = blocks + n;
@@ -68,9 +68,9 @@ bool gen_v2_tx_validation_base::generate_with(std::vector<test_event_entry>& eve
     {
       cryptonote::block blk;
       CHECK_AND_ASSERT_MES(generator.construct_block_manually(blk, blk_last, miner_account,
-          test_generator::bf_major_ver | test_generator::bf_minor_ver | test_generator::bf_timestamp | test_generator::bf_max_outs,
+          test_generator::bf_major_ver | test_generator::bf_minor_ver | test_generator::bf_timestamp,
           2, 2, blk_last.timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN * 2, // v2 has blocks twice as long
-          crypto::hash(), 0, transaction(), std::vector<crypto::hash>(), 0, max_outs),
+          crypto::hash(), 0, transaction(), std::vector<crypto::hash>(), 0, 0),
           false, "Failed to generate block");
       events.push_back(blk);
       blk_last = blk;
@@ -123,8 +123,7 @@ bool gen_v2_tx_mixable_0_mixin::generate(std::vector<test_event_entry>& events)
   const int mixin = 0;
   const int out_idx[] = {1, -1};
   const uint64_t amount_paid = 10000;
-  const size_t max_outs = 11;
+  return generate_with(events, out_idx, mixin, amount_paid, false);
-  return generate_with(events, out_idx, mixin, amount_paid, max_outs, false);
 }
 
 bool gen_v2_tx_mixable_low_mixin::generate(std::vector<test_event_entry>& events) const
@@ -132,8 +131,7 @@ bool gen_v2_tx_mixable_low_mixin::generate(std::vector<test_event_entry>& events
   const int mixin = 1;
   const int out_idx[] = {1, -1};
   const uint64_t amount_paid = 10000;
-  const size_t max_outs = 11;
+  return generate_with(events, out_idx, mixin, amount_paid, false);
-  return generate_with(events, out_idx, mixin, amount_paid, max_outs, false);
 }
 
 bool gen_v2_tx_unmixable_only::generate(std::vector<test_event_entry>& events) const
@@ -141,8 +139,7 @@ bool gen_v2_tx_unmixable_only::generate(std::vector<test_event_entry>& events) c
   const int mixin = 0;
   const int out_idx[] = {0, -1};
   const uint64_t amount_paid = 10000;
-  const size_t max_outs = 1;
+  return generate_with(events, out_idx, mixin, amount_paid, true);
-  return generate_with(events, out_idx, mixin, amount_paid, max_outs, true);
 }
 
 bool gen_v2_tx_unmixable_one::generate(std::vector<test_event_entry>& events) const
@@ -150,8 +147,7 @@ bool gen_v2_tx_unmixable_one::generate(std::vector<test_event_entry>& events) co
   const int mixin = 0;
   const int out_idx[] = {0, 1, -1};
   const uint64_t amount_paid = 10000;
-  const size_t max_outs = 11;
+  return generate_with(events, out_idx, mixin, amount_paid, true);
-  return generate_with(events, out_idx, mixin, amount_paid, max_outs, true);
 }
 
 bool gen_v2_tx_unmixable_two::generate(std::vector<test_event_entry>& events) const
@@ -159,8 +155,7 @@ bool gen_v2_tx_unmixable_two::generate(std::vector<test_event_entry>& events) co
   const int mixin = 0;
   const int out_idx[] = {0, 1, 2, -1};
   const uint64_t amount_paid = 10000;
-  const size_t max_outs = 11;
+  return generate_with(events, out_idx, mixin, amount_paid, false);
-  return generate_with(events, out_idx, mixin, amount_paid, max_outs, false);
 }
 
 bool gen_v2_tx_dust::generate(std::vector<test_event_entry>& events) const
@@ -168,6 +163,5 @@ bool gen_v2_tx_dust::generate(std::vector<test_event_entry>& events) const
   const int mixin = 2;
   const int out_idx[] = {1, -1};
   const uint64_t amount_paid = 10001;
-  const size_t max_outs = 11;
+  return generate_with(events, out_idx, mixin, amount_paid, false);
-  return generate_with(events, out_idx, mixin, amount_paid, max_outs, false);
 }

tests/core_tests/v2_tests.h

@@ -70,7 +70,7 @@ struct gen_v2_tx_validation_base : public test_chain_unit_base
   }
 
   bool generate_with(std::vector<test_event_entry>& events, const int *out_idx, int mixin,
-      uint64_t amount_paid, size_t max_outs, bool valid) const;
+      uint64_t amount_paid, bool valid) const;
 
 private:
   size_t m_invalid_tx_index;

tests/unit_tests/CMakeLists.txt

@@ -50,7 +50,8 @@ set(unit_tests_sources
   test_peerlist.cpp
   test_protocol_pack.cpp
   hardfork.cpp
-  unbound.cpp)
+  unbound.cpp
+  varint.cpp)
 
 set(unit_tests_headers
   unit_tests_utils.h)
@@ -70,7 +71,7 @@ target_link_libraries(unit_tests
     ${Boost_REGEX_LIBRARY}
     ${Boost_SYSTEM_LIBRARY}
     ${Boost_THREAD_LIBRARY}
-    ${UNBOUND_LIBRARIES}
+    ${UNBOUND_LIBRARY}
     ${EXTRA_LIBRARIES})
 set_property(TARGET unit_tests
   PROPERTY

tests/unit_tests/hardfork.cpp

@@ -78,6 +78,7 @@ public:
   virtual block get_top_block() const { return block(); }
   virtual uint64_t height() const { return blocks.size(); }
   virtual bool tx_exists(const crypto::hash& h) const { return false; }
+  virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const { return false; }
   virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const { return 0; }
   virtual transaction get_tx(const crypto::hash& h) const { return transaction(); }
   virtual uint64_t get_tx_count() const { return 0; }
@@ -93,13 +94,13 @@ public:
   virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) {}
   virtual bool can_thread_bulk_indices() const { return false; }
   virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); }
-  virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); }
+  virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector<uint64_t>(); }
   virtual bool has_key_image(const crypto::key_image& img) const { return false; }
   virtual void remove_block() { blocks.pop_back(); }
-  virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {}
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {return 0;}
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) {}
-  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {}
+  virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {return 0;}
-  virtual void remove_output(const tx_out& tx_output) {}
+  virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) {}
   virtual void add_spent_key(const crypto::key_image& k_image) {}
   virtual void remove_spent_key(const crypto::key_image& k_image) {}
 

tests/unit_tests/varint.cpp

@@ -0,0 +1,66 @@
+// Copyright (c) 2014-2016, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#include <cstring>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <vector>
+#include <boost/foreach.hpp>
+#include "cryptonote_core/cryptonote_basic.h"
+#include "cryptonote_core/cryptonote_basic_impl.h"
+#include "serialization/serialization.h"
+#include "serialization/binary_archive.h"
+#include "serialization/json_archive.h"
+#include "serialization/debug_archive.h"
+#include "serialization/variant.h"
+#include "serialization/vector.h"
+#include "serialization/binary_utils.h"
+#include "gtest/gtest.h"
+using namespace std;
+
+TEST(varint, equal)
+{
+  for (uint64_t idx = 0; idx < 65537; ++idx)
+  {
+    char buf[12];
+    char *bufptr = buf;
+    tools::write_varint(bufptr, idx);
+    uint64_t bytes = bufptr - buf;
+    ASSERT_TRUE (bytes > 0 && bytes <= sizeof(buf));
+
+    uint64_t idx2;
+    bufptr = buf;
+    std::string s(buf, bytes);
+    int read = tools::read_varint(s.begin(), s.end(), idx2);
+    ASSERT_EQ (read, bytes);
+    ASSERT_TRUE(idx2 == idx);
+  }
+}