Merge remote-tracking branch 'upstream/master' into develop

README.md

@@ -193,9 +193,9 @@ See README.i18n
 
 ## Using Tor
 
-While Monero isn't made to integrate with Tor, it can be used wrapped with torsocks, if you add --p2p-bind-ip 127.0.0.1 to the bitmonerod command line. You also want to set DNS requests to go over TCP, so they'll be routed through Tor, by setting DNS_PUBLIC=tcp. You may also disable IGD (UPnP port forwarding negotiation), which is pointless with Tor. Example:
+While Monero isn't made to integrate with Tor, it can be used wrapped with torsocks, if you add --p2p-bind-ip 127.0.0.1 to the bitmonerod command line. You also want to set DNS requests to go over TCP, so they'll be routed through Tor, by setting DNS_PUBLIC=tcp. You may also disable IGD (UPnP port forwarding negotiation), which is pointless with Tor. To allow local connections from the wallet, add TORSOCKS_ALLOW_INBOUND=1. Example:
 
-DNS_PUBLIC=tcp torsocks bitmonerod --p2p-bind-ip 127.0.0.1 --no-igd
+DNS_PUBLIC=tcp TORSOCKS_ALLOW_INBOUND=1 torsocks bitmonerod --p2p-bind-ip 127.0.0.1 --no-igd
 
 ## Using readline
 

contrib/epee/include/misc_log_ex.h

@@ -424,6 +424,7 @@ namespace log_space
       }
 
       std::cout << buf;
+      std::cout << std::flush;
 #endif
       reset_console_color();
       return  true;

contrib/epee/include/misc_os_dependent.h

@@ -53,11 +53,13 @@ namespace misc_utils
 #if defined(_MSC_VER)
                 return ::GetTickCount64();
 #elif defined(WIN32)
-# if defined(WIN64)
+                static LARGE_INTEGER pcfreq = {0};
-                return GetTickCount64();
+                LARGE_INTEGER ticks;
-# else
+                if (!pcfreq.QuadPart)
-                return GetTickCount();
+                    QueryPerformanceFrequency(&pcfreq);
-# endif
+                QueryPerformanceCounter(&ticks);
+                ticks.QuadPart *= 1000; /* we want msec */
+                return ticks.QuadPart / pcfreq.QuadPart;
 #elif defined(__MACH__)
                 clock_serv_t cclock;
                 mach_timespec_t mts;

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -252,11 +252,24 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   template<class t_protocol_handler>
   void connection<t_protocol_handler>::save_dbg_log()
   {
+    std::string address, port;
+    boost::system::error_code e;
+
+    boost::asio::ip::tcp::endpoint endpoint = socket_.remote_endpoint(e);
+    if (e)
+    {
+      address = "<not connected>";
+      port = "<not connected>";
+    }
+    else
+    {
+      address = endpoint.address().to_string();
+      port = boost::lexical_cast<std::string>(endpoint.port());
+    }
     _mark_c("net/kind" ,
-			 " connection type " << to_string( m_connection_type ) << " "
+        " connection type " << to_string( m_connection_type ) << " "
-	    << socket_.local_endpoint().address().to_string() << ":" << socket_.local_endpoint().port()
+        << socket_.local_endpoint().address().to_string() << ":" << socket_.local_endpoint().port()
-			<<	" <--> " << socket_.remote_endpoint().address().to_string() << ":" << socket_.remote_endpoint().port()
+        << " <--> " << address << ":" << port);
-		);
   }
   //---------------------------------------------------------------------------------
   template<class t_protocol_handler>

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -377,7 +377,9 @@ void BlockchainLMDB::do_resize(uint64_t increase_size)
 
   mdb_txn_safe::wait_no_active_txns();
 
-  mdb_env_set_mapsize(m_env, new_mapsize);
+  int result = mdb_env_set_mapsize(m_env, new_mapsize);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Failed to set new mapsize: ", result).c_str()));
 
   LOG_PRINT_GREEN("LMDB Mapsize increased." << "  Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB", LOG_LEVEL_0);
 

src/common/json_util.h

@@ -0,0 +1,53 @@
+// Copyright (c) 2016, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once 
+
+#define GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, name, type, jtype, mandatory) \
+  type field_##name; \
+  bool field_##name##_found = false; \
+  (void)field_##name##_found; \
+  do if (json.HasMember(#name)) \
+  { \
+    if (json[#name].Is##jtype()) \
+    { \
+      field_##name = json[#name].Get##jtype(); \
+      field_##name##_found = true; \
+    } \
+    else \
+    { \
+      LOG_ERROR("Field " << #name << " found in JSON, but not " << #jtype); \
+      return false; \
+    } \
+  } \
+  else if (mandatory) \
+  { \
+    LOG_ERROR("Field " << #name << " not found in JSON"); \
+    return false; \
+  } while(0)
+

src/common/util.cpp

@@ -413,4 +413,13 @@ std::string get_nix_version_display_string()
     }
     return false;
   }
+  void set_strict_default_file_permissions(bool strict)
+  {
+#if defined(__MINGW32__) || defined(__MINGW__)
+    // no clue about the odd one out
+#else
+    mode_t mode = strict ? 077 : 0;
+    umask(mode);
+#endif
+  }
 }

src/common/util.h

@@ -158,4 +158,6 @@ namespace tools
     /*! \brief where the installed handler is stored */
     static std::function<void(int)> m_handler;
   };
+
+  void set_strict_default_file_permissions(bool strict);
 }

src/cryptonote_core/blockchain.cpp

@@ -1991,7 +1991,7 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u
 // This function overloads its sister function with
 // an extra value (hash of highest block that holds an output used as input)
 // as a return-by-reference.
-bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id, bool kept_by_block)
+bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
@@ -2013,26 +2013,20 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t& max_used_block
 #endif
 
   TIME_MEASURE_START(a);
-  bool res = check_tx_inputs(tx, &max_used_block_height);
+  bool res = check_tx_inputs(tx, tvc, &max_used_block_height);
   TIME_MEASURE_FINISH(a);
-  crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx);
   if(m_show_time_stats)
     LOG_PRINT_L0("HASH: " << "+" << " VIN/VOUT: " << tx.vin.size() << "/" << tx.vout.size() << " H: " << max_used_block_height << " chcktx: " << a + m_fake_scan_time);
 
   if (!res)
     return false;
 
-  // ND: Speedup:
-  // 1. keep a list of verified transactions, when the Blockchain tries to check a tx again,
-  //    verify against list and skip if already verified to be correct.
-  m_check_tx_inputs_table.emplace(tx_prefix_hash, std::make_pair(res, max_used_block_height));
-
   CHECK_AND_ASSERT_MES(max_used_block_height < m_db->height(), false,  "internal error: max used block index=" << max_used_block_height << " is not less then blockchain size = " << m_db->height());
   max_used_block_id = m_db->get_block_hash_from_height(max_used_block_height);
   return true;
 }
 //------------------------------------------------------------------
-bool Blockchain::check_tx_outputs(const transaction& tx)
+bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context &tvc)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
@@ -2041,6 +2035,7 @@ bool Blockchain::check_tx_outputs(const transaction& tx)
   if (m_hardfork->get_current_version() >= 2) {
     for (auto &o: tx.vout) {
       if (!is_valid_decomposed_amount(o.amount)) {
+        tvc.m_invalid_output = true;
         return false;
       }
     }
@@ -2066,7 +2061,7 @@ bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const
 //        check_tx_input() rather than here, and use this function simply
 //        to iterate the inputs as necessary (splitting the task
 //        using threads, etc.)
-bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height)
+bool Blockchain::check_tx_inputs(const transaction& tx, tx_verification_context &tvc, uint64_t* pmax_used_block_height)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   size_t sig_index = 0;
@@ -2075,16 +2070,6 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc
 
   crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx);
 
-  auto its = m_check_tx_inputs_table.find(tx_prefix_hash);
-  if (its != m_check_tx_inputs_table.end())
-  {
-    if (!its->second.first)
-      return false;
-    if (pmax_used_block_height)
-      *pmax_used_block_height = its->second.second;
-    return true;
-  }
-
   // from hard fork 2, we require mixin at least 2 unless one output cannot mix with 2 others
   // if one output cannot mix with 2 others, we accept at most 1 output that can mix
   if (m_hardfork->get_current_version() >= 2)
@@ -2113,11 +2098,13 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc
       if (n_unmixable == 0)
       {
         LOG_PRINT_L1("Tx " << get_transaction_hash(tx) << " has too low mixin (" << mixin << "), and no unmixable inputs");
+        tvc.m_low_mixin = true;
         return false;
       }
       if (n_mixable > 1)
       {
         LOG_PRINT_L1("Tx " << get_transaction_hash(tx) << " has too low mixin (" << mixin << "), and more than one mixable input with unmixable inputs");
+        tvc.m_low_mixin = true;
         return false;
       }
     }
@@ -2156,6 +2143,7 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc
         if(ioservice_active) \
         { \
             work.reset(); \
+            while (!ioservice.stopped()) ioservice.poll(); \
             threadpool.join_all(); \
             ioservice.stop(); \
             ioservice_active = false; \
@@ -2176,6 +2164,7 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc
     if(have_tx_keyimg_as_spent(in_to_key.k_image))
     {
       LOG_PRINT_L1("Key image already spent in blockchain: " << epee::string_tools::pod_to_hex(in_to_key.k_image));
+      tvc.m_double_spend = true;
       return false;
     }
 
@@ -2667,7 +2656,8 @@ leave:
 #endif
     {
       // validate that transaction inputs and the keys spending them are correct.
-      if(!check_tx_inputs(tx))
+      tx_verification_context tvc;
+      if(!check_tx_inputs(tx, tvc))
       {
         LOG_PRINT_L1("Block with id: " << id  << " has at least one transaction (id: " << tx_id << ") with wrong inputs.");
 
@@ -2962,7 +2952,6 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
   TIME_MEASURE_FINISH(t1);
   m_blocks_longhash_table.clear();
   m_scan_table.clear();
-  m_check_tx_inputs_table.clear();
   m_blocks_txs_check.clear();
   m_check_txin_table.clear();
 
@@ -3110,7 +3099,6 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e
   m_fake_pow_calc_time = 0;
 
   m_scan_table.clear();
-  m_check_tx_inputs_table.clear();
   m_check_txin_table.clear();
 
   TIME_MEASURE_FINISH(prepare);

src/cryptonote_core/blockchain.h

@@ -472,11 +472,12 @@ namespace cryptonote
      * @param tx the transaction to validate
      * @param pmax_used_block_height return-by-reference block height of most recent input
      * @param max_used_block_id return-by-reference block hash of most recent input
+     * @param tvc returned information about tx verification
      * @param kept_by_block whether or not the transaction is from a previously-verified block
      *
      * @return false if any input is invalid, otherwise true
      */
-    bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, bool kept_by_block = false);
+    bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false);
 
     /**
      * @brief check that a transaction's outputs conform to current standards
@@ -486,10 +487,11 @@ namespace cryptonote
      * written out would have only one non-zero digit in base 10).
      *
      * @param tx the transaction to check the outputs of
+     * @param tvc returned info about tx verification
      *
      * @return false if any outputs do not conform, otherwise true
      */
-    bool check_tx_outputs(const transaction& tx);
+    bool check_tx_outputs(const transaction& tx, tx_verification_context &tvc);
 
     /**
      * @brief gets the blocksize limit based on recent blocks
@@ -788,7 +790,6 @@ namespace cryptonote
 
     // metadata containers
     std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, std::vector<output_data_t>>> m_scan_table;
-    std::unordered_map<crypto::hash, std::pair<bool, uint64_t>> m_check_tx_inputs_table;
     std::unordered_map<crypto::hash, crypto::hash> m_blocks_longhash_table;
     std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, bool>> m_check_txin_table;
 
@@ -883,11 +884,12 @@ namespace cryptonote
      * transaction.
      *
      * @param tx the transaction to validate
+     * @param tvc returned information about tx verification
      * @param pmax_related_block_height return-by-pointer the height of the most recent block in the input set
      *
      * @return false if any validation step fails, otherwise true
      */
-    bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL);
+    bool check_tx_inputs(const transaction& tx, tx_verification_context &tvc, uint64_t* pmax_used_block_height = NULL);
 
     /**
      * @brief performs a blockchain reorganization according to the longest chain rule

src/cryptonote_core/cryptonote_core.cpp

@@ -489,6 +489,7 @@ namespace cryptonote
     {
       LOG_PRINT_L1("WRONG TRANSACTION BLOB, too big size " << tx_blob.size() << ", rejected");
       tvc.m_verifivation_failed = true;
+      tvc.m_too_big = true;
       return false;
     }
 

src/cryptonote_core/cryptonote_format_utils.cpp

@@ -145,10 +145,18 @@ namespace cryptonote
       [&out_amounts](uint64_t a_dust) { out_amounts.push_back(a_dust); });
 
     CHECK_AND_ASSERT_MES(1 <= max_outs, false, "max_out must be non-zero");
-    while (max_outs < out_amounts.size())
+    if (height == 0)
     {
-      out_amounts[out_amounts.size() - 2] += out_amounts.back();
+      // the genesis block was not decomposed, for unknown reasons
-      out_amounts.resize(out_amounts.size() - 1);
+      while (max_outs < out_amounts.size())
+      {
+        out_amounts[out_amounts.size() - 2] += out_amounts.back();
+        out_amounts.resize(out_amounts.size() - 1);
+      }
+    }
+    else
+    {
+      CHECK_AND_ASSERT_MES(max_outs >= out_amounts.size(), false, "max_out exceeded");
     }
 
     uint64_t summary_amounts = 0;

src/cryptonote_core/tx_pool.cpp

@@ -54,6 +54,11 @@ namespace cryptonote
 {
   namespace
   {
+    //TODO: constants such as these should at least be in the header,
+    //      but probably somewhere more accessible to the rest of the
+    //      codebase.  As it stands, it is at best nontrivial to test
+    //      whether or not changing these parameters (or adding new)
+    //      will work correctly.
     size_t const TRANSACTION_SIZE_LIMIT_V1 = (((CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1 * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE);
     size_t const TRANSACTION_SIZE_LIMIT_V2 = (((CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE);
     time_t const MIN_RELAY_TIME = (60 * 5); // only start re-relaying transactions after that many seconds
@@ -97,6 +102,7 @@ namespace cryptonote
     if(!check_inputs_types_supported(tx))
     {
       tvc.m_verifivation_failed = true;
+      tvc.m_invalid_input = true;
       return false;
     }
 
@@ -113,17 +119,20 @@ namespace cryptonote
     {
       LOG_PRINT_L1("transaction use more money then it has: use " << print_money(outputs_amount) << ", have " << print_money(inputs_amount));
       tvc.m_verifivation_failed = true;
+      tvc.m_overspend = true;
       return false;
     }
 
+    // fee per kilobyte, size rounded up.
     uint64_t fee = inputs_amount - outputs_amount;
     uint64_t needed_fee = blob_size / 1024;
     needed_fee += (blob_size % 1024) ? 1 : 0;
     needed_fee *= FEE_PER_KB;
-    if (!kept_by_block && fee < needed_fee /*&& fee < MINING_ALLOWED_LEGACY_FEE*/)
+    if (!kept_by_block && fee < needed_fee)
     {
       LOG_PRINT_L1("transaction fee is not enough: " << print_money(fee) << ", minimum fee: " << print_money(needed_fee));
       tvc.m_verifivation_failed = true;
+      tvc.m_fee_too_low = true;
       return false;
     }
 
@@ -132,40 +141,46 @@ namespace cryptonote
     {
       LOG_PRINT_L1("transaction is too big: " << blob_size << " bytes, maximum size: " << tx_size_limit);
       tvc.m_verifivation_failed = true;
+      tvc.m_too_big = true;
       return false;
     }
 
-    //check key images for transaction if it is not kept by block
+    // if the transaction came from a block popped from the chain,
+    // don't check if we have its key images as spent.
+    // TODO: Investigate why not?
     if(!kept_by_block)
     {
       if(have_tx_keyimges_as_spent(tx))
       {
         LOG_PRINT_L1("Transaction with id= "<< id << " used already spent key images");
         tvc.m_verifivation_failed = true;
+        tvc.m_double_spend = true;
         return false;
       }
     }
 
-    if (!m_blockchain.check_tx_outputs(tx))
+    if (!m_blockchain.check_tx_outputs(tx, tvc))
     {
       LOG_PRINT_L1("Transaction with id= "<< id << " has at least one invalid outout");
       tvc.m_verifivation_failed = true;
+      tvc.m_invalid_output = true;
       return false;
     }
 
     crypto::hash max_used_block_id = null_hash;
     uint64_t max_used_block_height = 0;
 #if BLOCKCHAIN_DB == DB_LMDB
-    bool ch_inp_res = m_blockchain.check_tx_inputs(tx, max_used_block_height, max_used_block_id, kept_by_block);
+    bool ch_inp_res = m_blockchain.check_tx_inputs(tx, max_used_block_height, max_used_block_id, tvc, kept_by_block);
 #else
     bool ch_inp_res = m_blockchain.check_tx_inputs(tx, max_used_block_height, max_used_block_id);
 #endif
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     if(!ch_inp_res)
     {
+      // if the transaction was valid before (kept_by_block), then it
+      // may become valid again, so ignore the failed inputs check.
       if(kept_by_block)
       {
-        //anyway add this transaction to pool, because it related to block
         auto txd_p = m_transactions.insert(transactions_container::value_type(id, tx_details()));
         CHECK_AND_ASSERT_MES(txd_p.second, false, "transaction already exists at inserting in memory pool");
         txd_p.first->second.blob_size = blob_size;
@@ -207,13 +222,14 @@ namespace cryptonote
         tvc.m_should_be_relayed = true;
     }
 
+    // assume failure during verification steps until success is certain
     tvc.m_verifivation_failed = true;
-    //update image_keys container, here should everything goes ok.
+
     BOOST_FOREACH(const auto& in, tx.vin)
     {
       CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, txin, false);
       std::unordered_set<crypto::hash>& kei_image_set = m_spent_key_images[txin.k_image];
-      CHECK_AND_ASSERT_MES(kept_by_block || kei_image_set.size() == 0, false, "internal error: keeped_by_block=" << kept_by_block
+      CHECK_AND_ASSERT_MES(kept_by_block || kei_image_set.size() == 0, false, "internal error: kept_by_block=" << kept_by_block
                                           << ",  kei_image_set.size()=" << kei_image_set.size() << ENDL << "txin.k_image=" << txin.k_image << ENDL
                                           << "tx_id=" << id );
       auto ins_res = kei_image_set.insert(id);
@@ -223,7 +239,7 @@ namespace cryptonote
     tvc.m_verifivation_failed = false;
 
     m_txs_by_fee.emplace((double)blob_size / fee, id);
-    //succeed
+
     return true;
   }
   //---------------------------------------------------------------------------------
@@ -235,6 +251,9 @@ namespace cryptonote
     return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, version);
   }
   //---------------------------------------------------------------------------------
+  //FIXME: Can return early before removal of all of the key images.
+  //       At the least, need to make sure that a false return here
+  //       is treated properly.  Should probably not return early, however.
   bool tx_memory_pool::remove_transaction_keyimages(const transaction& tx)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -301,7 +320,7 @@ namespace cryptonote
     );
   }
   //---------------------------------------------------------------------------------
-  //proper tx_pool handling courtesy of CryptoZoidberg and Boolberry
+  //TODO: investigate whether boolean return is appropriate
   bool tx_memory_pool::remove_stuck_transactions()
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -332,6 +351,7 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------------------------
+  //TODO: investigate whether boolean return is appropriate
   bool tx_memory_pool::get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::transaction>> &txs) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -380,6 +400,7 @@ namespace cryptonote
       txs.push_back(tx_vt.second.tx);
   }
   //------------------------------------------------------------------
+  //TODO: investigate whether boolean return is appropriate
   bool tx_memory_pool::get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -480,7 +501,8 @@ namespace cryptonote
       if(txd.last_failed_id != null_hash && m_blockchain.get_current_blockchain_height() > txd.last_failed_height && txd.last_failed_id == m_blockchain.get_block_id_by_height(txd.last_failed_height))
         return false;//we already sure that this tx is broken for this height
 
-      if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id))
+      tx_verification_context tvc;
+      if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id, tvc))
       {
         txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1;
         txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height);
@@ -490,13 +512,18 @@ namespace cryptonote
     {
       if(txd.max_used_block_height >= m_blockchain.get_current_blockchain_height())
         return false;
-      if(m_blockchain.get_block_id_by_height(txd.max_used_block_height) != txd.max_used_block_id)
+      if(true)
       {
         //if we already failed on this height and id, skip actual ring signature check
         if(txd.last_failed_id == m_blockchain.get_block_id_by_height(txd.last_failed_height))
           return false;
         //check ring signature again, it is possible (with very small chance) that this transaction become again valid
+#if BLOCKCHAIN_DB == DB_LMDB
+        tx_verification_context tvc;
+        if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id, tvc))
+#else
         if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id))
+#endif
         {
           txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1;
           txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height);
@@ -556,6 +583,7 @@ namespace cryptonote
     return ss.str();
   }
   //---------------------------------------------------------------------------------
+  //TODO: investigate whether boolean return is appropriate
   bool tx_memory_pool::fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee)
   {
     // Warning: This function takes already_generated_
@@ -646,6 +674,7 @@ namespace cryptonote
     return n_removed;
   }
   //---------------------------------------------------------------------------------
+  //TODO: investigate whether only ever returning true is correct
   bool tx_memory_pool::init(const std::string& config_folder)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -679,6 +708,7 @@ namespace cryptonote
   }
 
   //---------------------------------------------------------------------------------
+  //TODO: investigate whether only ever returning true is correct
   bool tx_memory_pool::deinit()
   {
     if (m_config_folder.empty())

src/cryptonote_core/tx_pool.h

@@ -57,7 +57,9 @@ namespace cryptonote
   /*                                                                      */
   /************************************************************************/
 
+  //! pair of <transaction fee, transaction hash> for organization
   typedef std::pair<double, crypto::hash> tx_by_fee_entry;
+
   class txCompare
   {
   public:
@@ -71,47 +73,256 @@ namespace cryptonote
     }
   };
 
+  //! container for sorting transactions by fee per unit size
   typedef std::set<tx_by_fee_entry, txCompare> sorted_tx_container;
 
+  /**
+   * @brief Transaction pool, handles transactions which are not part of a block
+   *
+   * This class handles all transactions which have been received, but not as
+   * part of a block.
+   *
+   * This handling includes:
+   *   storing the transactions
+   *   organizing the transactions by fee per size
+   *   taking/giving transactions to and from various other components
+   *   saving the transactions to disk on shutdown
+   *   helping create a new block template by choosing transactions for it
+   *
+   */
   class tx_memory_pool: boost::noncopyable
   {
   public:
 #if BLOCKCHAIN_DB == DB_LMDB
+    /**
+     * @brief Constructor
+     *
+     * @param bchs a Blockchain class instance, for getting chain info
+     */
     tx_memory_pool(Blockchain& bchs);
 #else
     tx_memory_pool(blockchain_storage& bchs);
 #endif
-    bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, uint8_t version);
+
-    bool add_tx(const transaction &tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, uint8_t version);
+
-    //gets tx and remove it from pool
+    /**
+     * @copydoc add_tx(const transaction&, tx_verification_context&, bool, bool, uint8_t)
+     *
+     * @param id the transaction's hash
+     * @param blob_size the transaction's size
+     */
+    bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, uint8_t version);
+
+    /**
+     * @brief add a transaction to the transaction pool
+     *
+     * Most likely the transaction will come from the network, but it is
+     * also possible for transactions to come from popped blocks during
+     * a reorg, or from local clients creating a transaction and
+     * submitting it to the network
+     *
+     * @param tx the transaction to be added
+     * @param tvc return-by-reference status about the transaction verification
+     * @param kept_by_block has this transaction been in a block?
+     * @param relayed was this transaction from the network or a local client?
+     * @param version the version used to create the transaction
+     *
+     * @return true if the transaction passes validations, otherwise false
+     */
+    bool add_tx(const transaction &tx, tx_verification_context& tvc, bool kept_by_block, bool relayed, uint8_t version);
+
+    /**
+     * @brief takes a transaction with the given hash from the pool
+     *
+     * @param id the hash of the transaction
+     * @param tx return-by-reference the transaction taken
+     * @param blob_size return-by-reference the transaction's size
+     * @param fee the transaction fee
+     * @param relayed return-by-reference was transaction relayed to us by the network?
+     *
+     * @return true unless the transaction cannot be found in the pool
+     */
     bool take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed);
 
+    /**
+     * @brief checks if the pool has a transaction with the given hash
+     *
+     * @param id the hash to look for
+     *
+     * @return true if the transaction is in the pool, otherwise false
+     */
     bool have_tx(const crypto::hash &id) const;
+
+    /**
+     * @brief action to take when notified of a block added to the blockchain
+     *
+     * Currently does nothing
+     *
+     * @param new_block_height the height of the blockchain after the change
+     * @param top_block_id the hash of the new top block
+     *
+     * @return true
+     */
     bool on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id);
+
+    /**
+     * @brief action to take when notified of a block removed from the blockchain
+     *
+     * Currently does nothing
+     *
+     * @param new_block_height the height of the blockchain after the change
+     * @param top_block_id the hash of the new top block
+     *
+     * @return true
+     */
     bool on_blockchain_dec(uint64_t new_block_height, const crypto::hash& top_block_id);
+
+    /**
+     * @brief action to take periodically
+     *
+     * Currently checks transaction pool for stale ("stuck") transactions
+     */
     void on_idle();
 
+    /**
+     * @brief locks the transaction pool
+     */
     void lock() const;
+
+    /**
+     * @brief unlocks the transaction pool
+     */
     void unlock() const;
 
     // load/store operations
+
+    /**
+     * @brief loads pool state (if any) from disk, and initializes pool
+     *
+     * @param config_folder folder name where pool state will be
+     *
+     * @return true
+     */
     bool init(const std::string& config_folder);
+
+    /**
+     * @brief attempts to save the transaction pool state to disk
+     *
+     * Currently fails (returns false) if the data directory from init()
+     * does not exist and cannot be created, but returns true even if
+     * saving to disk is unsuccessful.
+     *
+     * @return true in most cases (see above)
+     */
     bool deinit();
+
+    /**
+     * @brief Chooses transactions for a block to include
+     *
+     * @param bl return-by-reference the block to fill in with transactions
+     * @param median_size the current median block size
+     * @param already_generated_coins the current total number of coins "minted"
+     * @param total_size return-by-reference the total size of the new block
+     * @param fee return-by-reference the total of fees from the included transactions
+     *
+     * @return true
+     */
     bool fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee);
+
+    /**
+     * @brief get a list of all transactions in the pool
+     *
+     * @param txs return-by-reference the list of transactions
+     */
     void get_transactions(std::list<transaction>& txs) const;
+
+    /**
+     * @brief get information about all transactions and key images in the pool
+     *
+     * see documentation on tx_info and spent_key_image_info for more details
+     *
+     * @param tx_infos return-by-reference the transactions' information
+     * @param key_image_infos return-by-reference the spent key images' information
+     *
+     * @return true
+     */
     bool get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const;
+
+    /**
+     * @brief get a specific transaction from the pool
+     *
+     * @param h the hash of the transaction to get
+     * @param tx return-by-reference the transaction requested
+     *
+     * @return true if the transaction is found, otherwise false
+     */
     bool get_transaction(const crypto::hash& h, transaction& tx) const;
+
+    /**
+     * @brief get a list of all relayable transactions and their hashes
+     *
+     * "relayable" in this case means:
+     *   nonzero fee
+     *   hasn't been relayed too recently
+     *   isn't old enough that relaying it is considered harmful
+     *
+     * @param txs return-by-reference the transactions and their hashes
+     *
+     * @return true
+     */
     bool get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::transaction>>& txs) const;
+
+    /**
+     * @brief tell the pool that certain transactions were just relayed
+     *
+     * @param txs the list of transactions (and their hashes)
+     */
     void set_relayed(const std::list<std::pair<crypto::hash, cryptonote::transaction>>& txs);
+
+    /**
+     * @brief get the total number of transactions in the pool
+     *
+     * @return the number of transactions in the pool
+     */
     size_t get_transactions_count() const;
+
+    /**
+     * @brief get a string containing human-readable pool information
+     *
+     * @param short_format whether to use a shortened format for the info
+     *
+     * @return the string
+     */
     std::string print_pool(bool short_format) const;
+
+    /**
+     * @brief remove transactions from the pool which are no longer valid
+     *
+     * With new versions of the currency, what conditions render a transaction
+     * invalid may change.  This function clears those which were received
+     * before a version change and no longer conform to requirements.
+     *
+     * @param version the version the transactions must conform to
+     *
+     * @return the number of transactions removed
+     */
     size_t validate(uint8_t version);
 
-    /*bool flush_pool(const std::strig& folder);
-    bool inflate_pool(const std::strig& folder);*/
 
 #define CURRENT_MEMPOOL_ARCHIVE_VER    11
+#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER    11
 
+    /**
+     * @brief serialize the transaction pool to/from disk
+     *
+     * If the archive version passed is older than the version compiled
+     * in, this function does nothing, as it cannot deserialize after a
+     * format change.
+     *
+     * @tparam archive_t the archive class
+     * @param a the archive to serialize to/from
+     * @param version the archive version
+     */
     template<class archive_t>
     void serialize(archive_t & a, const unsigned int version)
     {
@@ -123,97 +334,169 @@ namespace cryptonote
       a & m_timed_out_transactions;
     }
 
+    /**
+     * @brief information about a single transaction
+     */
     struct tx_details
     {
-      transaction tx;
+      transaction tx;  //!< the transaction
-      size_t blob_size;
+      size_t blob_size;  //!< the transaction's size
-      uint64_t fee;
+      uint64_t fee;  //!< the transaction's fee amount
-      crypto::hash max_used_block_id;
+      crypto::hash max_used_block_id;  //!< the hash of the highest block referenced by an input
-      uint64_t max_used_block_height;
+      uint64_t max_used_block_height;  //!< the height of the highest block referenced by an input
-      bool kept_by_block;
-      //
-      uint64_t last_failed_height;
-      crypto::hash last_failed_id;
-      time_t receive_time;
 
-      time_t last_relayed_time;
+      //! whether or not the transaction has been in a block before
-      bool relayed;
+      /*! if the transaction was returned to the pool from the blockchain
+       *  due to a reorg, then this will be true
+       */
+      bool kept_by_block;  
+
+      //! the highest block the transaction referenced when last checking it failed
+      /*! if verifying a transaction's inputs fails, it's possible this is due
+       *  to a reorg since it was created (if it used recently created outputs
+       *  as inputs).
+       */
+      uint64_t last_failed_height;  
+
+      //! the hash of the highest block the transaction referenced when last checking it failed
+      /*! if verifying a transaction's inputs fails, it's possible this is due
+       *  to a reorg since it was created (if it used recently created outputs
+       *  as inputs).
+       */
+      crypto::hash last_failed_id;
+
+      time_t receive_time;  //!< the time when the transaction entered the pool
+
+      time_t last_relayed_time;  //!< the last time the transaction was relayed to the network
+      bool relayed;  //!< whether or not the transaction has been relayed to the network
     };
 
   private:
+
+    /**
+     * @brief remove old transactions from the pool
+     *
+     * After a certain time, it is assumed that a transaction which has not
+     * yet been mined will likely not be mined.  These transactions are removed
+     * from the pool to avoid buildup.
+     *
+     * @return true
+     */
     bool remove_stuck_transactions();
+
+    /**
+     * @brief check if a transaction in the pool has a given spent key image
+     *
+     * @param key_im the spent key image to look for
+     *
+     * @return true if the spent key image is present, otherwise false
+     */
     bool have_tx_keyimg_as_spent(const crypto::key_image& key_im) const;
+
+    /**
+     * @brief check if any spent key image in a transaction is in the pool
+     *
+     * Checks if any of the spent key images in a given transaction are present
+     * in any of the transactions in the transaction pool.
+     *
+     * @note see tx_pool::have_tx_keyimg_as_spent
+     *
+     * @param tx the transaction to check spent key images of
+     *
+     * @return true if any spent key images are present in the pool, otherwise false
+     */
     bool have_tx_keyimges_as_spent(const transaction& tx) const;
+
+    /**
+     * @brief forget a transaction's spent key images
+     *
+     * Spent key images are stored separately from transactions for
+     * convenience/speed, so this is part of the process of removing
+     * a transaction from the pool.
+     *
+     * @param tx the transaction
+     *
+     * @return false if any key images to be removed cannot be found, otherwise true
+     */
     bool remove_transaction_keyimages(const transaction& tx);
+
+    /**
+     * @brief check if any of a transaction's spent key images are present in a given set
+     *
+     * @param kic the set of key images to check against
+     * @param tx the transaction to check
+     *
+     * @return true if any key images present in the set, otherwise false
+     */
     static bool have_key_images(const std::unordered_set<crypto::key_image>& kic, const transaction& tx);
+
+    /**
+     * @brief append the key images from a transaction to the given set
+     *
+     * @param kic the set of key images to append to
+     * @param tx the transaction
+     *
+     * @return false if any append fails, otherwise true
+     */
     static bool append_key_images(std::unordered_set<crypto::key_image>& kic, const transaction& tx);
 
+    /**
+     * @brief check if a transaction is a valid candidate for inclusion in a block
+     *
+     * @param txd the transaction to check (and info about it)
+     *
+     * @return true if the transaction is good to go, otherwise false
+     */
     bool is_transaction_ready_to_go(tx_details& txd) const;
+
+    //! map transactions (and related info) by their hashes
     typedef std::unordered_map<crypto::hash, tx_details > transactions_container;
+
+    //TODO: confirm the below comments and investigate whether or not this
+    //      is the desired behavior
+    //! map key images to transactions which spent them
+    /*! this seems odd, but it seems that multiple transactions can exist
+     *  in the pool which both have the same spent key.  This would happen
+     *  in the event of a reorg where someone creates a new/different
+     *  transaction on the assumption that the original will not be in a
+     *  block again.
+     */
     typedef std::unordered_map<crypto::key_image, std::unordered_set<crypto::hash> > key_images_container;
 
-    mutable epee::critical_section m_transactions_lock;
+    mutable epee::critical_section m_transactions_lock;  //!< lock for the pool
-    transactions_container m_transactions;
+    transactions_container m_transactions;  //!< container for transactions in the pool
-    key_images_container m_spent_key_images;
+
+    //! container for spent key images from the transactions in the pool
+    key_images_container m_spent_key_images;  
+
+    //TODO: this time should be a named constant somewhere, not hard-coded
+    //! interval on which to check for stale/"stuck" transactions
     epee::math_helper::once_a_time_seconds<30> m_remove_stuck_tx_interval;
 
-    //TODO: add fee_per_kb element to type tx_details and replace this
+    //TODO: look into doing this better
-    //functionality by just making m_transactions a std::set
+    sorted_tx_container m_txs_by_fee;  //!< container for transactions organized by fee per size
-    sorted_tx_container m_txs_by_fee;
 
+    /**
+     * @brief get an iterator to a transaction in the sorted container
+     *
+     * @param id the hash of the transaction to look for
+     *
+     * @return an iterator, possibly to the end of the container if not found
+     */
     sorted_tx_container::iterator find_tx_in_sorted_container(const crypto::hash& id) const;
 
+    //! transactions which are unlikely to be included in blocks
+    /*! These transactions are kept in RAM in case they *are* included
+     *  in a block eventually, but this container is not saved to disk.
+     */
     std::unordered_set<crypto::hash> m_timed_out_transactions;
 
-    //transactions_container m_alternative_transactions;
+    std::string m_config_folder;  //!< the folder to save state to
-
-    std::string m_config_folder;
 #if BLOCKCHAIN_DB == DB_LMDB
-    Blockchain& m_blockchain;
+    Blockchain& m_blockchain;  //!< reference to the Blockchain object
 #else
     blockchain_storage& m_blockchain;
 #endif
-    /************************************************************************/
-    /*                                                                      */
-    /************************************************************************/
-    /*class inputs_visitor: public boost::static_visitor<bool>
-    {
-      key_images_container& m_spent_keys;
-    public:
-      inputs_visitor(key_images_container& spent_keys): m_spent_keys(spent_keys)
-      {}
-      bool operator()(const txin_to_key& tx) const
-      {
-        auto pr = m_spent_keys.insert(tx.k_image);
-        CHECK_AND_ASSERT_MES(pr.second, false, "Tried to insert transaction with input seems already spent, input: " << epee::string_tools::pod_to_hex(tx.k_image));
-        return true;
-      }
-      bool operator()(const txin_gen& tx) const
-      {
-        CHECK_AND_ASSERT_MES(false, false, "coinbase transaction in memory pool");
-        return false;
-      }
-      bool operator()(const txin_to_script& tx) const {return false;}
-      bool operator()(const txin_to_scripthash& tx) const {return false;}
-    }; */
-    /************************************************************************/
-    /*                                                                      */
-    /************************************************************************/
-    class amount_visitor: public boost::static_visitor<uint64_t>
-    {
-    public:
-      uint64_t operator()(const txin_to_key& tx) const
-      {
-        return tx.amount;
-      }
-      uint64_t operator()(const txin_gen& tx) const
-      {
-        CHECK_AND_ASSERT_MES(false, false, "coinbase transaction in memory pool");
-        return 0;
-      }
-      uint64_t operator()(const txin_to_script& tx) const {return 0;}
-      uint64_t operator()(const txin_to_scripthash& tx) const {return 0;}
-    };
 
 #if BLOCKCHAIN_DB == DB_LMDB
 #else
@@ -248,6 +531,7 @@ namespace boost
   }
 }
 BOOST_CLASS_VERSION(cryptonote::tx_memory_pool, CURRENT_MEMPOOL_ARCHIVE_VER)
+BOOST_CLASS_VERSION(cryptonote::tx_memory_pool::tx_details, CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER)
 
 
 

src/cryptonote_core/verification_context.h

@@ -40,6 +40,13 @@ namespace cryptonote
     bool m_verifivation_failed; //bad tx, should drop connection
     bool m_verifivation_impossible; //the transaction is related with an alternative blockchain
     bool m_added_to_pool; 
+    bool m_low_mixin;
+    bool m_double_spend;
+    bool m_invalid_input;
+    bool m_invalid_output;
+    bool m_too_big;
+    bool m_overspend;
+    bool m_fee_too_low;
   };
 
   struct block_verification_context

src/daemon/rpc_command_executor.cpp

@@ -70,6 +70,23 @@ namespace {
       << "difficulty: " << boost::lexical_cast<std::string>(header.difficulty) << std::endl
       << "reward: " << boost::lexical_cast<std::string>(header.reward);
   }
+
+  std::string get_human_time_ago(time_t t, time_t now)
+  {
+    if (t == now)
+      return "now";
+    time_t dt = t > now ? t - now : now - t;
+    std::string s;
+    if (dt < 90)
+      s = boost::lexical_cast<std::string>(dt) + " seconds";
+    else if (dt < 90 * 60)
+      s = boost::lexical_cast<std::string>(dt/60) + " minutes";
+    else if (dt < 36 * 3600)
+      s = boost::lexical_cast<std::string>(dt/3600) + " hours";
+    else
+      s = boost::lexical_cast<std::string>(dt/(3600*24)) + " days";
+    return s + " " + (t > now ? "in the future" : "ago");
+  }
 }
 
 t_rpc_command_executor::t_rpc_command_executor(
@@ -575,16 +592,26 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash) {
     }
   }
 
-  if (1 == res.txs_as_hex.size())
+  if (1 == res.txs.size() || 1 == res.txs_as_hex.size())
   {
+    if (1 == res.txs.size())
+    {
+      // only available for new style answers
+      if (res.txs.front().in_pool)
+        tools::success_msg_writer() << "Found in pool";
+      else
+        tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height;
+    }
+
     // first as hex
-    tools::success_msg_writer() << res.txs_as_hex.front();
+    const std::string &as_hex = (1 == res.txs.size()) ? res.txs.front().as_hex : res.txs_as_hex.front();
+    tools::success_msg_writer() << as_hex;
 
     // then as json
     crypto::hash tx_hash, tx_prefix_hash;
     cryptonote::transaction tx;
     cryptonote::blobdata blob;
-    if (!string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), blob))
+    if (!string_tools::parse_hexstr_to_binbuff(as_hex, blob))
     {
       tools::fail_msg_writer() << "Failed to parse tx";
     }
@@ -669,6 +696,7 @@ bool t_rpc_command_executor::print_transaction_pool_long() {
   }
   if (! res.transactions.empty())
   {
+    const time_t now = time(NULL);
     tools::msg_writer() << "Transactions: ";
     for (auto & tx_info : res.transactions)
     {
@@ -676,7 +704,7 @@ bool t_rpc_command_executor::print_transaction_pool_long() {
                           << tx_info.tx_json << std::endl
                           << "blob_size: " << tx_info.blob_size << std::endl
                           << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl
-                          << "receive_time: " << tx_info.receive_time << std::endl
+                          << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl
                           << "kept_by_block: " << (tx_info.kept_by_block ? 'T' : 'F') << std::endl
                           << "max_used_block_height: " << tx_info.max_used_block_height << std::endl
                           << "max_used_block_id: " << tx_info.max_used_block_id_hash << std::endl
@@ -747,17 +775,21 @@ bool t_rpc_command_executor::print_transaction_pool_short() {
   {
     tools::msg_writer() << "Pool is empty" << std::endl;
   }
-  for (auto & tx_info : res.transactions)
+  else
   {
-    tools::msg_writer() << "id: " << tx_info.id_hash << std::endl
+    const time_t now = time(NULL);
-                        << "blob_size: " << tx_info.blob_size << std::endl
+    for (auto & tx_info : res.transactions)
-                        << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl
+    {
-                        << "receive_time: " << tx_info.receive_time << std::endl
+      tools::msg_writer() << "id: " << tx_info.id_hash << std::endl
-                        << "kept_by_block: " << (tx_info.kept_by_block ? 'T' : 'F') << std::endl
+                          << "blob_size: " << tx_info.blob_size << std::endl
-                        << "max_used_block_height: " << tx_info.max_used_block_height << std::endl
+                          << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl
-                        << "max_used_block_id: " << tx_info.max_used_block_id_hash << std::endl
+                          << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl
-                        << "last_failed_height: " << tx_info.last_failed_height << std::endl
+                          << "kept_by_block: " << (tx_info.kept_by_block ? 'T' : 'F') << std::endl
-                        << "last_failed_id: " << tx_info.last_failed_id_hash << std::endl;
+                          << "max_used_block_height: " << tx_info.max_used_block_height << std::endl
+                          << "max_used_block_id: " << tx_info.max_used_block_id_hash << std::endl
+                          << "last_failed_height: " << tx_info.last_failed_height << std::endl
+                          << "last_failed_id: " << tx_info.last_failed_id_hash << std::endl;
+    }
   }
 
   return true;

src/rpc/core_rpc_server.cpp

@@ -240,6 +240,7 @@ namespace cryptonote
 
     // try the pool for any missing txes
     size_t found_in_pool = 0;
+    std::unordered_set<crypto::hash> pool_tx_hashes;
     if (!missed_txs.empty())
     {
       std::list<transaction> pool_txs;
@@ -248,9 +249,11 @@ namespace cryptonote
       {
         for (std::list<transaction>::const_iterator i = pool_txs.begin(); i != pool_txs.end(); ++i)
         {
-          std::list<crypto::hash>::iterator mi = std::find(missed_txs.begin(), missed_txs.end(), get_transaction_hash(*i));
+          crypto::hash tx_hash = get_transaction_hash(*i);
+          std::list<crypto::hash>::iterator mi = std::find(missed_txs.begin(), missed_txs.end(), tx_hash);
           if (mi != missed_txs.end())
           {
+            pool_tx_hashes.insert(tx_hash);
             missed_txs.erase(mi);
             txs.push_back(*i);
             ++found_in_pool;
@@ -260,12 +263,33 @@ namespace cryptonote
       LOG_PRINT_L2("Found " << found_in_pool << "/" << vh.size() << " transactions in the pool");
     }
 
+    std::list<std::string>::const_iterator txhi = req.txs_hashes.begin();
+    std::vector<crypto::hash>::const_iterator vhi = vh.begin();
     BOOST_FOREACH(auto& tx, txs)
     {
+      res.txs.push_back(COMMAND_RPC_GET_TRANSACTIONS::entry());
+      COMMAND_RPC_GET_TRANSACTIONS::entry &e = res.txs.back();
+
+      crypto::hash tx_hash = *vhi++;
+      e.tx_hash = *txhi++;
       blobdata blob = t_serializable_object_to_blob(tx);
-      res.txs_as_hex.push_back(string_tools::buff_to_hex_nodelimer(blob));
+      e.as_hex = string_tools::buff_to_hex_nodelimer(blob);
       if (req.decode_as_json)
-        res.txs_as_json.push_back(obj_to_json_str(tx));
+        e.as_json = obj_to_json_str(tx);
+      e.in_pool = pool_tx_hashes.find(tx_hash) != pool_tx_hashes.end();
+      if (e.in_pool)
+      {
+        e.block_height = std::numeric_limits<uint64_t>::max();
+      }
+      else
+      {
+        e.block_height = m_core.get_blockchain_storage().get_db().get_tx_block_height(tx_hash);
+      }
+
+      // fill up old style responses too, in case an old wallet asks
+      res.txs_as_hex.push_back(e.as_hex);
+      if (req.decode_as_json)
+        res.txs_as_json.push_back(e.as_json);
     }
 
     BOOST_FOREACH(const auto& miss_tx, missed_txs)
@@ -273,7 +297,7 @@ namespace cryptonote
       res.missed_tx.push_back(string_tools::pod_to_hex(miss_tx));
     }
 
-    LOG_PRINT_L2(res.txs_as_hex.size() << " transactions found, " << res.missed_tx.size() << " not found");
+    LOG_PRINT_L2(res.txs.size() << " transactions found, " << res.missed_tx.size() << " not found");
     res.status = CORE_RPC_STATUS_OK;
     return true;
   }
@@ -355,24 +379,40 @@ namespace cryptonote
 
     cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context);
     tx_verification_context tvc = AUTO_VAL_INIT(tvc);
-    if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false))
+    if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false) || tvc.m_verifivation_failed)
     {
-      LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx");
+      if (tvc.m_verifivation_failed)
+      {
+        LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed");
+      }
+      else
+      {
+        LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx");
+      }
       res.status = "Failed";
+      if ((res.low_mixin = tvc.m_low_mixin))
+        res.reason = "mixin too low";
+      if ((res.double_spend = tvc.m_double_spend))
+        res.reason = "double spend";
+      if ((res.invalid_input = tvc.m_invalid_input))
+        res.reason = "invalid input";
+      if ((res.invalid_output = tvc.m_invalid_output))
+        res.reason = "invalid output";
+      if ((res.too_big = tvc.m_too_big))
+        res.reason = "too big";
+      if ((res.overspend = tvc.m_overspend))
+        res.reason = "overspend";
+      if ((res.fee_too_low = tvc.m_fee_too_low))
+        res.reason = "fee too low";
       return true;
     }
 
-    if(tvc.m_verifivation_failed)
+    if(!tvc.m_should_be_relayed || req.do_not_relay)
-    {
-      LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed");
-      res.status = "Failed";
-      return true;
-    }
-
-    if(!tvc.m_should_be_relayed)
     {
       LOG_PRINT_L0("[on_send_raw_tx]: tx accepted, but not relayed");
-      res.status = "Not relayed";
+      res.reason = "Not relayed";
+      res.not_relayed = true;
+      res.status = CORE_RPC_STATUS_OK;
       return true;
     }
 
@@ -627,8 +667,10 @@ namespace cryptonote
       LOG_ERROR("Failed to calculate offset for ");
       return false;
     }
+    blobdata hashing_blob = get_block_hashing_blob(b);
     res.prev_hash = string_tools::pod_to_hex(b.prev_id);
     res.blocktemplate_blob = string_tools::buff_to_hex_nodelimer(block_blob);
+    res.blockhashing_blob =  string_tools::buff_to_hex_nodelimer(hashing_blob);
     res.status = CORE_RPC_STATUS_OK;
     return true;
   }

src/rpc/core_rpc_server_commands_defs.h

@@ -103,18 +103,41 @@ namespace cryptonote
       END_KV_SERIALIZE_MAP()
     };
 
+    struct entry
+    {
+      std::string tx_hash;
+      std::string as_hex;
+      std::string as_json;
+      bool in_pool;
+      uint64_t block_height;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(tx_hash)
+        KV_SERIALIZE(as_hex)
+        KV_SERIALIZE(as_json)
+        KV_SERIALIZE(in_pool)
+        KV_SERIALIZE(block_height)
+      END_KV_SERIALIZE_MAP()
+    };
 
     struct response
     {
-      std::list<std::string> txs_as_hex;  //transactions blobs as hex
+      // older compatibility stuff
+      std::list<std::string> txs_as_hex;  //transactions blobs as hex (old compat)
+      std::list<std::string> txs_as_json; //transactions decoded as json (old compat)
+
+      // in both old and new
       std::list<std::string> missed_tx;   //not found transactions
-      std::list<std::string> txs_as_json; //transactions decoded as json
+
+      // new style
+      std::vector<entry> txs;
       std::string status;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(txs_as_hex)
-        KV_SERIALIZE(missed_tx)
         KV_SERIALIZE(txs_as_json)
+        KV_SERIALIZE(txs)
+        KV_SERIALIZE(missed_tx)
         KV_SERIALIZE(status)
       END_KV_SERIALIZE_MAP()
     };
@@ -221,12 +244,14 @@ namespace cryptonote
     struct request
     {
       std::string tx_as_hex;
+      bool do_not_relay;
 
       request() {}
       explicit request(const transaction &);
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(tx_as_hex)
+        KV_SERIALIZE(do_not_relay)
       END_KV_SERIALIZE_MAP()
     };
 
@@ -234,9 +259,27 @@ namespace cryptonote
     struct response
     {
       std::string status;
+      std::string reason;
+      bool not_relayed;
+      bool low_mixin;
+      bool double_spend;
+      bool invalid_input;
+      bool invalid_output;
+      bool too_big;
+      bool overspend;
+      bool fee_too_low;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(status)
+        KV_SERIALIZE(reason)
+        KV_SERIALIZE(not_relayed)
+        KV_SERIALIZE(low_mixin)
+        KV_SERIALIZE(double_spend)
+        KV_SERIALIZE(invalid_input)
+        KV_SERIALIZE(invalid_output)
+        KV_SERIALIZE(too_big)
+        KV_SERIALIZE(overspend)
+        KV_SERIALIZE(fee_too_low)
       END_KV_SERIALIZE_MAP()
     };
   };
@@ -427,6 +470,7 @@ namespace cryptonote
       uint64_t reserved_offset;
       std::string prev_hash;
       blobdata blocktemplate_blob;
+      blobdata blockhashing_blob;
       std::string status;
 
       BEGIN_KV_SERIALIZE_MAP()
@@ -435,6 +479,7 @@ namespace cryptonote
         KV_SERIALIZE(reserved_offset)
         KV_SERIALIZE(prev_hash)
         KV_SERIALIZE(blocktemplate_blob)
+        KV_SERIALIZE(blockhashing_blob)
         KV_SERIALIZE(status)
       END_KV_SERIALIZE_MAP()
     };

src/simplewallet/simplewallet.cpp

@@ -58,6 +58,7 @@
 #include "crypto/crypto.h"  // for crypto::secret_key definition
 #include "mnemonics/electrum-words.h"
 #include "rapidjson/document.h"
+#include "common/json_util.h"
 #include <stdexcept>
 
 #if defined(WIN32)
@@ -257,6 +258,8 @@ bool simple_wallet::seed(const std::vector<std::string> &args/* = std::vector<st
     if (m_wallet->get_seed_language().empty())
     {
       std::string mnemonic_language = get_mnemonic_language();
+      if (mnemonic_language.empty())
+        return true;
       m_wallet->set_seed_language(mnemonic_language);
     }
 
@@ -304,6 +307,8 @@ bool simple_wallet::seed_set_language(const std::vector<std::string> &args/* = s
   }
 
   std::string mnemonic_language = get_mnemonic_language();
+  if (mnemonic_language.empty())
+    return true;
   m_wallet->set_seed_language(mnemonic_language);
   m_wallet->rewrite(m_wallet_file, pwd_container.password());
   return true;
@@ -698,6 +703,10 @@ bool simple_wallet::ask_wallet_create_if_needed()
         tr("Specify wallet file name (e.g., MyWallet). If the wallet doesn't exist, it will be created.\n"
         "Wallet file name: ")
     );
+    if (std::cin.eof())
+    {
+      return false;
+    }
     valid_path = tools::wallet2::wallet_valid_path_format(wallet_path);
     if (!valid_path)
     {
@@ -820,6 +829,8 @@ static bool get_password(const boost::program_options::variables_map& vm, bool a
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::generate_from_json(const boost::program_options::variables_map& vm, std::string &wallet_file, std::string &password)
 {
+  bool testnet = command_line::get_arg(vm, arg_testnet);
+
   std::string buf;
   bool r = epee::file_io_utils::load_file_to_string(m_generate_from_json, buf);
   if (!r) {
@@ -833,84 +844,123 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m
     return false;
   }
 
-  if (!json.HasMember("version")) {
+  GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, version, unsigned, Uint, true);
-    fail_msg_writer() << tr("Version not found in JSON");
-    return false;
-  }
-  unsigned int version = json["version"].GetUint();
   const int current_version = 1;
-  if (version > current_version) {
+  if (field_version > current_version) {
-    fail_msg_writer() << boost::format(tr("Version %u too new, we can only grok up to %u")) % version % current_version;
+    fail_msg_writer() << boost::format(tr("Version %u too new, we can only grok up to %u")) % field_version % current_version;
     return false;
   }
-  if (!json.HasMember("filename")) {
-    fail_msg_writer() << tr("Filename not found in JSON");
-    return false;
-  }
-  std::string filename = json["filename"].GetString();
 
-  bool recover = false;
+  GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, filename, std::string, String, true);
-  uint64_t scan_from_height = 0;
-  if (json.HasMember("scan_from_height")) {
-    scan_from_height = json["scan_from_height"].GetUint64();
-    recover = true;
-  }
 
-  password = "";
+  GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, scan_from_height, uint64_t, Uint64, false);
-  if (json.HasMember("password")) {
+  bool recover = field_scan_from_height_found;
-    password = json["password"].GetString();
-  }
 
-  std::string viewkey_string("");
+  GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, password, std::string, String, false);
+  password = field_password;
+
+  GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, viewkey, std::string, String, false);
   crypto::secret_key viewkey;
-  if (json.HasMember("viewkey")) {
+  if (field_viewkey_found)
-    viewkey_string = json["viewkey"].GetString();
+  {
     cryptonote::blobdata viewkey_data;
-    if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data))
+    if(!epee::string_tools::parse_hexstr_to_binbuff(field_viewkey, viewkey_data))
     {
       fail_msg_writer() << tr("failed to parse view key secret key");
       return false;
     }
     viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data());
+    crypto::public_key pkey;
+    if (!crypto::secret_key_to_public_key(viewkey, pkey)) {
+      fail_msg_writer() << tr("failed to verify view key secret key");
+      return false;
+    }
   }
 
-  std::string spendkey_string("");
+  GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, spendkey, std::string, String, false);
   crypto::secret_key spendkey;
-  if (json.HasMember("spendkey")) {
+  if (field_spendkey_found)
-    spendkey_string = json["spendkey"].GetString();
+  {
     cryptonote::blobdata spendkey_data;
-    if(!epee::string_tools::parse_hexstr_to_binbuff(spendkey_string, spendkey_data))
+    if(!epee::string_tools::parse_hexstr_to_binbuff(field_spendkey, spendkey_data))
     {
       fail_msg_writer() << tr("failed to parse spend key secret key");
       return false;
     }
     spendkey = *reinterpret_cast<const crypto::secret_key*>(spendkey_data.data());
+    crypto::public_key pkey;
+    if (!crypto::secret_key_to_public_key(spendkey, pkey)) {
+      fail_msg_writer() << tr("failed to verify spend key secret key");
+      return false;
+    }
   }
 
-  std::string seed("");
+  GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, seed, std::string, String, false);
   std::string old_language;
-  if (json.HasMember("seed")) {
+  if (field_seed_found)
-    seed = json["seed"].GetString();
+  {
-    if (!crypto::ElectrumWords::words_to_bytes(seed, m_recovery_key, old_language))
+    if (!crypto::ElectrumWords::words_to_bytes(field_seed, m_recovery_key, old_language))
     {
       fail_msg_writer() << tr("Electrum-style word list failed verification");
       return false;
     }
-    m_electrum_seed = seed;
+    m_electrum_seed = field_seed;
     m_restore_deterministic_wallet = true;
   }
 
+  GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, address, std::string, String, false);
+
   // compatibility checks
-  if (seed.empty() && viewkey_string.empty()) {
+  if (!field_seed_found && !field_viewkey_found)
+  {
     fail_msg_writer() << tr("At least one of Electrum-style word list and private view key must be specified");
     return false;
   }
-  if (!seed.empty() && (!viewkey_string.empty() || !spendkey_string.empty())) {
+  if (field_seed_found && (field_viewkey_found || field_spendkey_found))
+  {
     fail_msg_writer() << tr("Both Electrum-style word list and private key(s) specified");
     return false;
   }
 
-  m_wallet_file = filename;
+  // if an address was given, we check keys against it, and deduce the spend
+  // public key if it was not given
+  if (field_address_found)
+  {
+    cryptonote::account_public_address address;
+    bool has_payment_id;
+    crypto::hash8 new_payment_id;
+    if(!get_account_integrated_address_from_str(address, has_payment_id, new_payment_id, testnet, field_address))
+    {
+      fail_msg_writer() << tr("invalid address");
+      return false;
+    }
+    if (field_viewkey_found)
+    {
+      crypto::public_key pkey;
+      if (!crypto::secret_key_to_public_key(viewkey, pkey)) {
+        fail_msg_writer() << tr("failed to verify view key secret key");
+        return false;
+      }
+      if (address.m_view_public_key != pkey) {
+        fail_msg_writer() << tr("view key does not match standard address");
+        return false;
+      }
+    }
+    if (field_spendkey_found)
+    {
+      crypto::public_key pkey;
+      if (!crypto::secret_key_to_public_key(spendkey, pkey)) {
+        fail_msg_writer() << tr("failed to verify spend key secret key");
+        return false;
+      }
+      if (address.m_spend_public_key != pkey) {
+        fail_msg_writer() << tr("spend key does not match standard address");
+        return false;
+      }
+    }
+  }
+
+  m_wallet_file = field_filename;
 
   bool was_deprecated_wallet = m_restore_deterministic_wallet && ((old_language == crypto::ElectrumWords::old_language_name) ||
     crypto::ElectrumWords::get_is_old_style_seed(m_electrum_seed));
@@ -919,13 +969,12 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m
     return false;
   }
 
-  bool testnet = command_line::get_arg(vm, arg_testnet);
   m_wallet.reset(new tools::wallet2(testnet));
   m_wallet->callback(this);
 
   try
   {
-    if (!seed.empty())
+    if (!field_seed.empty())
     {
       m_wallet->generate(m_wallet_file, password, m_recovery_key, recover, false);
     }
@@ -936,17 +985,27 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m
         fail_msg_writer() << tr("failed to verify view key secret key");
         return false;
       }
-      if (!crypto::secret_key_to_public_key(spendkey, address.m_spend_public_key)) {
-        fail_msg_writer() << tr("failed to verify spend key secret key");
-        return false;
-      }
 
-      if (spendkey_string.empty())
+      if (field_spendkey.empty())
       {
+        // if we have an addres but no spend key, we can deduce the spend public key
+        // from the address
+        if (field_address_found)
+        {
+          cryptonote::account_public_address address2;
+          bool has_payment_id;
+          crypto::hash8 new_payment_id;
+          get_account_integrated_address_from_str(address2, has_payment_id, new_payment_id, testnet, field_address);
+          address.m_spend_public_key = address2.m_spend_public_key;
+        }
         m_wallet->generate(m_wallet_file, password, address, viewkey);
       }
       else
       {
+        if (!crypto::secret_key_to_public_key(spendkey, address.m_spend_public_key)) {
+          fail_msg_writer() << tr("failed to verify spend key secret key");
+          return false;
+        }
         m_wallet->generate(m_wallet_file, password, address, spendkey, viewkey);
       }
     }
@@ -957,13 +1016,44 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m
     return false;
   }
 
-  m_wallet->set_refresh_from_block_height(scan_from_height);
+  m_wallet->set_refresh_from_block_height(field_scan_from_height);
 
   wallet_file = m_wallet_file;
 
   return r;
 }
 
+static bool is_local_daemon(const std::string &address)
+{
+  // extract host
+  epee::net_utils::http::url_content u_c;
+  if (!epee::net_utils::parse_url(address, u_c))
+  {
+    LOG_PRINT_L1("Failed to determine whether daemon is local, assuming not");
+    return false;
+  }
+  if (u_c.host.empty())
+  {
+    LOG_PRINT_L1("Failed to determine whether daemon is local, assuming not");
+    return false;
+  }
+
+  // resolve to IP
+  boost::asio::io_service io_service;
+  boost::asio::ip::tcp::resolver resolver(io_service);
+  boost::asio::ip::tcp::resolver::query query(u_c.host, "");
+  boost::asio::ip::tcp::resolver::iterator i = resolver.resolve(query);
+  while (i != boost::asio::ip::tcp::resolver::iterator())
+  {
+    const boost::asio::ip::tcp::endpoint &ep = *i;
+    if (ep.address().is_loopback())
+      return true;
+    ++i;
+  }
+
+  return false;
+}
+
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::init(const boost::program_options::variables_map& vm)
 {
@@ -999,6 +1089,17 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
   if (m_daemon_address.empty())
     m_daemon_address = std::string("http://") + m_daemon_host + ":" + std::to_string(m_daemon_port);
 
+  // set --trusted-daemon if local
+  try
+  {
+    if (is_local_daemon(m_daemon_address))
+    {
+      LOG_PRINT_L1(tr("Daemon is local, assuming trusted"));
+      m_trusted_daemon = true;
+    }
+  }
+  catch (const std::exception &e) { }
+
   tools::password_container pwd_container;
   if (!get_password(vm, true, pwd_container))
     return false;
@@ -1020,6 +1121,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
       if (m_electrum_seed.empty())
       {
         m_electrum_seed = command_line::input_line("Specify Electrum seed: ");
+        if (std::cin.eof())
+          return false;
         if (m_electrum_seed.empty())
         {
           fail_msg_writer() << tr("specify a recovery parameter with the --electrum-seed=\"words list here\"");
@@ -1037,6 +1140,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
     {
       // parse address
       std::string address_string = command_line::input_line("Standard address: ");
+      if (std::cin.eof())
+        return false;
       if (address_string.empty()) {
         fail_msg_writer() << tr("No data supplied, cancelled");
         return false;
@@ -1052,6 +1157,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
 
       // parse view secret key
       std::string viewkey_string = command_line::input_line("View key: ");
+      if (std::cin.eof())
+        return false;
       if (viewkey_string.empty()) {
         fail_msg_writer() << tr("No data supplied, cancelled");
         return false;
@@ -1084,6 +1191,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
     {
       // parse address
       std::string address_string = command_line::input_line("Standard address: ");
+      if (std::cin.eof())
+        return false;
       if (address_string.empty()) {
         fail_msg_writer() << tr("No data supplied, cancelled");
         return false;
@@ -1099,6 +1208,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
 
       // parse spend secret key
       std::string spendkey_string = command_line::input_line("Spend key: ");
+      if (std::cin.eof())
+        return false;
       if (spendkey_string.empty()) {
         fail_msg_writer() << tr("No data supplied, cancelled");
         return false;
@@ -1113,6 +1224,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
 
       // parse view secret key
       std::string viewkey_string = command_line::input_line("View key: ");
+      if (std::cin.eof())
+        return false;
       if (viewkey_string.empty()) {
         fail_msg_writer() << tr("No data supplied, cancelled");
         return false;
@@ -1232,6 +1345,8 @@ std::string simple_wallet::get_mnemonic_language()
   while (language_number < 0)
   {
     language_choice = command_line::input_line(tr("Enter the number corresponding to the language of your choice: "));
+    if (std::cin.eof())
+      return std::string();
     try
     {
       language_number = std::stoi(language_choice);
@@ -1270,6 +1385,8 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
         "a deprecated version of the wallet. Please use the new seed that we provide.\n");
     }
     mnemonic_language = get_mnemonic_language();
+    if (mnemonic_language.empty())
+      return false;
   }
 
   m_wallet_file = wallet_file;
@@ -1397,6 +1514,8 @@ bool simple_wallet::open_wallet(const string &wallet_file, const std::string& pa
         message_writer(epee::log_space::console_color_green, false) << "\n" << tr("You had been using "
           "a deprecated version of the wallet. Please proceed to upgrade your wallet.\n");
         std::string mnemonic_language = get_mnemonic_language();
+        if (mnemonic_language.empty())
+          return false;
         m_wallet->set_seed_language(mnemonic_language);
         m_wallet->rewrite(m_wallet_file, password);
 
@@ -2013,6 +2132,10 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str
 
           // prompt the user for confirmation given the dns query and dnssec status
           std::string confirm_dns_ok = command_line::input_line(prompt.str());
+          if (std::cin.eof())
+          {
+            return true;
+          }
           if (confirm_dns_ok != "Y" && confirm_dns_ok != "y" && confirm_dns_ok != "Yes" && confirm_dns_ok != "yes"
             && confirm_dns_ok != tr("yes") && confirm_dns_ok != tr("no"))
           {
@@ -2071,9 +2194,9 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str
     // figure out what tx will be necessary
     std::vector<tools::wallet2::pending_tx> ptx_vector;
     if (new_algorithm)
-      ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, 0 /* unused fee arg*/, extra);
+      ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, 0 /* unused fee arg*/, extra, m_trusted_daemon);
     else
-      ptx_vector = m_wallet->create_transactions(dsts, fake_outs_count, 0 /* unlock_time */, 0 /* unused fee arg*/, extra);
+      ptx_vector = m_wallet->create_transactions(dsts, fake_outs_count, 0 /* unlock_time */, 0 /* unused fee arg*/, extra, m_trusted_daemon);
 
     // if more than one tx necessary, prompt user to confirm
     if (m_wallet->always_confirm_transfers() || ptx_vector.size() > 1)
@@ -2097,6 +2220,8 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str
             print_money(total_fee)).str();
         }
         std::string accepted = command_line::input_line(prompt_str);
+        if (std::cin.eof())
+          return true;
         if (accepted != "Y" && accepted != "y" && accepted != "Yes" && accepted != "yes")
         {
           fail_msg_writer() << tr("transaction cancelled.");
@@ -2159,6 +2284,9 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str
   catch (const tools::error::tx_rejected& e)
   {
     fail_msg_writer() << (boost::format(tr("transaction %s was rejected by daemon with status: ")) % get_transaction_hash(e.tx())) << e.status();
+    std::string reason = e.reason();
+    if (!reason.empty())
+      fail_msg_writer() << tr("Reason: ") << reason;
   }
   catch (const tools::error::tx_sum_overflow& e)
   {
@@ -2254,6 +2382,8 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_)
         print_money(total_fee)).str();
     }
     std::string accepted = command_line::input_line(prompt_str);
+    if (std::cin.eof())
+      return true;
     if (accepted != "Y" && accepted != "y" && accepted != "Yes" && accepted != "yes")
     {
       fail_msg_writer() << tr("transaction cancelled.");
@@ -2316,6 +2446,9 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_)
   catch (const tools::error::tx_rejected& e)
   {
     fail_msg_writer() << (boost::format(tr("transaction %s was rejected by daemon with status: ")) % get_transaction_hash(e.tx())) << e.status();
+    std::string reason = e.reason();
+    if (!reason.empty())
+      fail_msg_writer() << tr("Reason: ") << reason;
   }
   catch (const tools::error::tx_sum_overflow& e)
   {
@@ -2425,13 +2558,18 @@ bool simple_wallet::check_tx_key(const std::vector<std::string> &args_)
   COMMAND_RPC_GET_TRANSACTIONS::response res;
   req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
   if (!net_utils::invoke_http_json_remote_command2(m_daemon_address + "/gettransactions", req, res, m_http_client) ||
-      res.txs_as_hex.empty())
+      (res.txs.empty() && res.txs_as_hex.empty()))
   {
     fail_msg_writer() << tr("failed to get transaction from daemon");
     return true;
   }
   cryptonote::blobdata tx_data;
-  if (!string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data))
+  bool ok;
+  if (!res.txs.empty())
+    ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data);
+  else
+    ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data);
+  if (!ok)
   {
     fail_msg_writer() << tr("failed to parse transaction from daemon");
     return true;
@@ -2735,6 +2873,7 @@ int main(int argc, char* argv[])
 
   std::string lang = i18n_get_language();
   tools::sanitize_locale();
+  tools::set_strict_default_file_permissions(true);
 
   string_tools::set_module_name_and_folder(argv[0]);
 
@@ -2891,12 +3030,25 @@ int main(int argc, char* argv[])
     }
 
     tools::wallet2 wal(testnet,restricted);
+    bool quit = false;
+    tools::signal_handler::install([&wal, &quit](int) {
+      quit = true;
+      wal.stop();
+    });
     try
     {
       LOG_PRINT_L0(sw::tr("Loading wallet..."));
       wal.load(wallet_file, password);
       wal.init(daemon_address);
       wal.refresh();
+      // if we ^C during potentially length load/refresh, there's no server loop yet
+      if (quit)
+      {
+        LOG_PRINT_L0(sw::tr("Storing wallet..."));
+        wal.store();
+        LOG_PRINT_GREEN(sw::tr("Stored ok"), LOG_LEVEL_0);
+        return 1;
+      }
       LOG_PRINT_GREEN(sw::tr("Loaded ok"), LOG_LEVEL_0);
     }
     catch (const std::exception& e)
@@ -2907,10 +3059,8 @@ int main(int argc, char* argv[])
     tools::wallet_rpc_server wrpc(wal);
     bool r = wrpc.init(vm);
     CHECK_AND_ASSERT_MES(r, 1, sw::tr("Failed to initialize wallet rpc server"));
-
     tools::signal_handler::install([&wrpc, &wal](int) {
       wrpc.send_stop_signal();
-      wal.store();
     });
     LOG_PRINT_L0(sw::tr("Starting wallet rpc server"));
     wrpc.run();

src/version.h.in

@@ -1,4 +1,4 @@
 #define MONERO_VERSION_TAG "@VERSIONTAG@"
-#define MONERO_VERSION "0.9.3.0"
+#define MONERO_VERSION "0.9.4.0"
 #define MONERO_RELEASE_NAME "Hydrogen Helix"
 #define MONERO_VERSION_FULL MONERO_VERSION "-" MONERO_VERSION_TAG

src/wallet/wallet2.cpp

@@ -52,6 +52,7 @@ using namespace epee;
 #include "rapidjson/document.h"
 #include "rapidjson/writer.h"
 #include "rapidjson/stringbuffer.h"
+#include "common/json_util.h"
 
 extern "C"
 {
@@ -72,6 +73,7 @@ using namespace cryptonote;
 #define KILL_IOSERVICE()  \
     do { \
       work.reset(); \
+      while (!ioservice.stopped()) ioservice.poll(); \
       threadpool.join_all(); \
       ioservice.stop(); \
     } while(0)
@@ -1004,7 +1006,7 @@ namespace
  * \param keys_file_name Name of wallet file
  * \param password       Password of wallet file
  */
-void wallet2::load_keys(const std::string& keys_file_name, const std::string& password)
+bool wallet2::load_keys(const std::string& keys_file_name, const std::string& password)
 {
   wallet2::keys_file_data keys_file_data;
   std::string buf;
@@ -1033,34 +1035,44 @@ void wallet2::load_keys(const std::string& keys_file_name, const std::string& pa
   }
   else
   {
-    account_data = std::string(json["key_data"].GetString(), json["key_data"].GetString() +
+    if (!json.HasMember("key_data"))
-      json["key_data"].GetStringLength());
-    if (json.HasMember("seed_language"))
     {
-      set_seed_language(std::string(json["seed_language"].GetString(), json["seed_language"].GetString() +
+      LOG_ERROR("Field key_data not found in JSON");
-        json["seed_language"].GetStringLength()));
+      return false;
     }
-    if (json.HasMember("watch_only"))
+    if (!json["key_data"].IsString())
     {
-      m_watch_only = json["watch_only"].GetInt() != 0;
+      LOG_ERROR("Field key_data found in JSON, but not String");
+      return false;
     }
-    else
+    const char *field_key_data = json["key_data"].GetString();
+    account_data = std::string(field_key_data, field_key_data + json["key_data"].GetStringLength());
+
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, seed_language, std::string, String, false);
+    if (field_seed_language_found)
     {
-      m_watch_only = false;
+      set_seed_language(field_seed_language);
     }
-    m_always_confirm_transfers = json.HasMember("always_confirm_transfers") && (json["always_confirm_transfers"].GetInt() != 0);
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, watch_only, int, Int, false);
-    m_store_tx_info = (json.HasMember("store_tx_keys") && (json["store_tx_keys"].GetInt() != 0))
+    m_watch_only = field_watch_only_found && field_watch_only;
-                   || (json.HasMember("store_tx_info") && (json["store_tx_info"].GetInt() != 0));
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, always_confirm_transfers, int, Int, false);
-    m_default_mixin = json.HasMember("default_mixin") ? json["default_mixin"].GetUint() : 0;
+    m_always_confirm_transfers = field_always_confirm_transfers_found && field_always_confirm_transfers;
-    m_auto_refresh = !json.HasMember("auto_refresh") || (json["auto_refresh"].GetInt() != 0);
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, store_tx_keys, int, Int, false);
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, store_tx_info, int, Int, false);
+    m_store_tx_info = (field_store_tx_keys_found && (field_store_tx_keys != 0))
+                   || (field_store_tx_info_found && (field_store_tx_info != 0));
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, default_mixin, unsigned int, Uint, false);
+    m_default_mixin = field_default_mixin_found ? field_default_mixin : 0;
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, auto_refresh, int, Int, false);
+    m_auto_refresh = !field_auto_refresh_found || (field_auto_refresh != 0);
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, refresh_type, int, Int, false);
     m_refresh_type = RefreshType::RefreshDefault;
-    if (json.HasMember("refresh_type"))
+    if (field_refresh_type_found)
     {
-      int type = json["refresh_type"].GetInt();
+      if (field_refresh_type == RefreshFull || field_refresh_type == RefreshOptimizeCoinbase || field_refresh_type == RefreshNoCoinbase)
-      if (type == RefreshFull || type == RefreshOptimizeCoinbase || type == RefreshNoCoinbase)
+        m_refresh_type = (RefreshType)field_refresh_type;
-        m_refresh_type = (RefreshType)type;
       else
-        LOG_PRINT_L0("Unknown refresh-type value (" << type << "), using default");
+        LOG_PRINT_L0("Unknown refresh-type value (" << field_refresh_type << "), using default");
     }
   }
 
@@ -1070,6 +1082,7 @@ void wallet2::load_keys(const std::string& keys_file_name, const std::string& pa
   if(!m_watch_only)
     r = r && verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key);
   THROW_WALLET_EXCEPTION_IF(!r, error::invalid_password);
+  return true;
 }
 
 /*!
@@ -1358,7 +1371,10 @@ void wallet2::load(const std::string& wallet_, const std::string& password)
   bool exists = boost::filesystem::exists(m_keys_file, e);
   THROW_WALLET_EXCEPTION_IF(e || !exists, error::file_not_found, m_keys_file);
 
-  load_keys(m_keys_file, password);
+  if (!load_keys(m_keys_file, password))
+  {
+    THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, m_keys_file);
+  }
   LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_testnet));
 
   //keys loaded ok!
@@ -1718,47 +1734,12 @@ namespace
 // returns:
 //    direct return: amount of money found
 //    modified reference: selected_transfers, a list of iterators/indices of input sources
-uint64_t wallet2::select_transfers(uint64_t needed_money, bool add_dust, uint64_t dust, bool hf2_rules, std::list<transfer_container::iterator>& selected_transfers)
+uint64_t wallet2::select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::list<transfer_container::iterator>& selected_transfers, bool trusted_daemon)
 {
-  std::vector<size_t> unused_transfers_indices;
-  std::vector<size_t> unused_dust_indices;
-
-  // aggregate sources available for transfers
-  // if dust needed, take dust from only one source (so require source has at least dust amount)
-  for (size_t i = 0; i < m_transfers.size(); ++i)
-  {
-    const transfer_details& td = m_transfers[i];
-    if (!td.m_spent && is_transfer_unlocked(td))
-    {
-      if (dust < td.amount() && is_valid_decomposed_amount(td.amount()))
-        unused_transfers_indices.push_back(i);
-      else
-      {
-        // for hf2 rules, we disregard dust, which will be spendable only
-        // via sweep_dust. If we're asked to add dust, though, we still
-        // consider them, as this will be a mixin 0 tx (and thus we may
-        // end up with a tx with one mixable output and N dusty ones).
-        // This should be made better at some point...
-        if (!hf2_rules || add_dust)
-          unused_dust_indices.push_back(i);
-      }
-    }
-  }
-
-  bool select_one_dust = add_dust && !unused_dust_indices.empty();
   uint64_t found_money = 0;
-  while (found_money < needed_money && (!unused_transfers_indices.empty() || !unused_dust_indices.empty()))
+  while (found_money < needed_money && !unused_transfers_indices.empty())
   {
-    size_t idx;
+    size_t idx = pop_random_value(unused_transfers_indices);
-    if (select_one_dust)
-    {
-      idx = pop_random_value(unused_dust_indices);
-      select_one_dust = false;
-    }
-    else
-    {
-      idx = !unused_transfers_indices.empty() ? pop_random_value(unused_transfers_indices) : pop_random_value(unused_dust_indices);
-    }
 
     transfer_container::iterator it = m_transfers.begin() + idx;
     selected_transfers.push_back(it);
@@ -1780,18 +1761,18 @@ void wallet2::add_unconfirmed_tx(const cryptonote::transaction& tx, const std::v
 }
 
 //----------------------------------------------------------------------------------------------------
-void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count,
+void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices,
-                       uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx)
+                       uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon)
 {
-  transfer(dsts, fake_outputs_count, unlock_time, fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), tx, ptx);
+  transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), tx, ptx, trusted_daemon);
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count,
+void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices,
-                       uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra)
+                       uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon)
 {
   cryptonote::transaction tx;
   pending_tx ptx;
-  transfer(dsts, fake_outputs_count, unlock_time, fee, extra, tx, ptx);
+  transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, tx, ptx, trusted_daemon);
 }
 
 namespace {
@@ -1943,13 +1924,14 @@ void wallet2::commit_tx(pending_tx& ptx)
 
   COMMAND_RPC_SEND_RAW_TX::request req;
   req.tx_as_hex = epee::string_tools::buff_to_hex_nodelimer(tx_to_blob(ptx.tx));
+  req.do_not_relay = false;
   COMMAND_RPC_SEND_RAW_TX::response daemon_send_resp;
   m_daemon_rpc_mutex.lock();
   bool r = epee::net_utils::invoke_http_json_remote_command2(m_daemon_address + "/sendrawtransaction", req, daemon_send_resp, m_http_client, 200000);
   m_daemon_rpc_mutex.unlock();
   THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "sendrawtransaction");
   THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "sendrawtransaction");
-  THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, daemon_send_resp.status);
+  THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, daemon_send_resp.status, daemon_send_resp.reason);
 
   txid = get_transaction_hash(ptx.tx);
   crypto::hash payment_id = cryptonote::null_hash;
@@ -1988,8 +1970,9 @@ void wallet2::commit_tx(std::vector<pending_tx>& ptx_vector)
 //
 // this function will make multiple calls to wallet2::transfer if multiple
 // transactions will be required
-std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra)
+std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra, bool trusted_daemon)
 {
+  const std::vector<size_t> unused_transfers_indices = select_available_outputs_from_histogram(fake_outs_count + 1, true, trusted_daemon);
 
   // failsafe split attempt counter
   size_t attempt_count = 0;
@@ -2019,7 +2002,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<crypto
 	uint64_t needed_fee = 0;
 	do
 	{
-	  transfer(dst_vector, fake_outs_count, unlock_time, needed_fee, extra, tx, ptx);
+	  transfer(dst_vector, fake_outs_count, unused_transfers_indices, unlock_time, needed_fee, extra, tx, ptx, trusted_daemon);
 	  auto txBlob = t_serializable_object_to_blob(ptx.tx);
           needed_fee = calculate_fee(txBlob);
 	} while (ptx.fee < needed_fee);
@@ -2252,7 +2235,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent
 // This system allows for sending (almost) the entire balance, since it does
 // not generate spurious change in all txes, thus decreasing the instantaneous
 // usable balance.
-std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra)
+std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra, bool trusted_daemon)
 {
   std::vector<size_t> unused_transfers_indices;
   std::vector<size_t> unused_dust_indices;
@@ -2672,9 +2655,8 @@ std::vector<uint64_t> wallet2::get_unspent_amounts_vector()
   return vector;
 }
 //----------------------------------------------------------------------------------------------------
-std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_daemon)
+std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t count, bool atleast, bool trusted_daemon)
 {
-  // request all outputs with at least 3 instances, so we can use mixin 2 with
   epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request> req_t = AUTO_VAL_INIT(req_t);
   epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response, std::string> resp_t = AUTO_VAL_INIT(resp_t);
   m_daemon_rpc_mutex.lock();
@@ -2683,7 +2665,7 @@ std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_dae
   req_t.method = "get_output_histogram";
   if (trusted_daemon)
     req_t.params.amounts = get_unspent_amounts_vector();
-  req_t.params.min_count = 3;
+  req_t.params.min_count = count;
   req_t.params.max_count = 0;
   bool r = net_utils::invoke_http_json_remote_command2(m_daemon_address + "/json_rpc", req_t, resp_t, m_http_client);
   m_daemon_rpc_mutex.unlock();
@@ -2697,14 +2679,32 @@ std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_dae
     mixable.insert(i.amount);
   }
 
-  return select_available_outputs([mixable](const transfer_details &td) {
+  return select_available_outputs([mixable, atleast](const transfer_details &td) {
     const uint64_t amount = td.amount();
-    if (mixable.find(amount) == mixable.end())
+    if (atleast) {
-      return true;
+      if (mixable.find(amount) != mixable.end())
+        return true;
+    }
+    else {
+      if (mixable.find(amount) == mixable.end())
+        return true;
+    }
     return false;
   });
 }
 //----------------------------------------------------------------------------------------------------
+std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_daemon)
+{
+  // request all outputs with less than 3 instances
+  return select_available_outputs_from_histogram(3, false, trusted_daemon);
+}
+//----------------------------------------------------------------------------------------------------
+std::vector<size_t> wallet2::select_available_mixable_outputs(bool trusted_daemon)
+{
+  // request all outputs with at least 3 instances, so we can use mixin 2 with
+  return select_available_outputs_from_histogram(3, true, trusted_daemon);
+}
+//----------------------------------------------------------------------------------------------------
 std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bool trusted_daemon)
 {
   // From hard fork 1, we don't consider small amounts to be dust anymore

src/wallet/wallet2.h

@@ -274,11 +274,11 @@ namespace tools
     uint64_t unlocked_balance() const;
     uint64_t unlocked_dust_balance(const tx_dust_policy &dust_policy) const;
     template<typename T>
-    void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy);
+    void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon);
     template<typename T>
-    void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx& ptx);
+    void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon);
-    void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra);
+    void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon);
-    void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx);
+    void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon);
     template<typename T>
     void transfer_from(const std::vector<size_t> &outs, size_t num_outputs, uint64_t unlock_time, uint64_t needed_fee, T destination_split_strategy, const tx_dust_policy& dust_policy, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx);
     template<typename T>
@@ -287,8 +287,8 @@ namespace tools
 
     void commit_tx(pending_tx& ptx_vector);
     void commit_tx(std::vector<pending_tx>& ptx_vector);
-    std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee, const std::vector<uint8_t> extra);
+    std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee, const std::vector<uint8_t> extra, bool trusted_daemon);
-    std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra);
+    std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra, bool trusted_daemon);
     std::vector<pending_tx> create_unmixable_sweep_transactions(bool trusted_daemon);
     bool check_connection();
     void get_transfers(wallet2::transfer_container& incoming_transfers) const;
@@ -379,7 +379,7 @@ namespace tools
      * \param keys_file_name Name of wallet file
      * \param password       Password of wallet file
      */
-    void load_keys(const std::string& keys_file_name, const std::string& password);
+    bool load_keys(const std::string& keys_file_name, const std::string& password);
     void process_new_transaction(const cryptonote::transaction& tx, uint64_t height, bool miner_tx);
     void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height);
     void detach_blockchain(uint64_t height);
@@ -390,7 +390,7 @@ namespace tools
     void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<cryptonote::block_complete_entry> &blocks);
     void pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::list<cryptonote::block_complete_entry> &prev_blocks, std::list<cryptonote::block_complete_entry> &blocks, bool &error);
     void process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, uint64_t& blocks_added);
-    uint64_t select_transfers(uint64_t needed_money, bool add_dust, uint64_t dust, bool hf2_rules, std::list<transfer_container::iterator>& selected_transfers);
+    uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::list<transfer_container::iterator>& selected_transfers, bool trusted_daemon);
     bool prepare_file_names(const std::string& file_path);
     void process_unconfirmed(const cryptonote::transaction& tx, uint64_t height);
     void process_outgoing(const cryptonote::transaction& tx, uint64_t height, uint64_t spent, uint64_t received);
@@ -404,8 +404,10 @@ namespace tools
     uint64_t get_upper_tranaction_size_limit();
     void check_pending_txes();
     std::vector<uint64_t> get_unspent_amounts_vector();
+    std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool trusted_daemon);
     std::vector<size_t> select_available_outputs(const std::function<bool(const transfer_details &td)> &f);
     std::vector<size_t> select_available_unmixable_outputs(bool trusted_daemon);
+    std::vector<size_t> select_available_mixable_outputs(bool trusted_daemon);
 
     cryptonote::account_base m_account;
     std::string m_daemon_address;
@@ -563,17 +565,17 @@ namespace tools
   }
   //----------------------------------------------------------------------------------------------------
   template<typename T>
-  void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count,
+  void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices,
-    uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy)
+    uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon)
   {
     pending_tx ptx;
     cryptonote::transaction tx;
-    transfer(dsts, fake_outputs_count, unlock_time, fee, extra, destination_split_strategy, dust_policy, tx, ptx);
+    transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, destination_split_strategy, dust_policy, tx, ptx, trusted_daemon);
   }
 
   template<typename T>
-  void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count,
+  void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices,
-    uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx)
+    uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx, bool trusted_daemon)
   {
     using namespace cryptonote;
     // throw if attempting a transaction with no destinations
@@ -594,9 +596,7 @@ namespace tools
     // randomly select inputs for transaction
     // throw if requested send amount is greater than amount available to send
     std::list<transfer_container::iterator> selected_transfers;
-    bool hf2_rules = use_fork_rules(2); // first fork has version 2
+    uint64_t found_money = select_transfers(needed_money, unused_transfers_indices, selected_transfers, trusted_daemon);
-    const bool add_dust = (0 == fake_outputs_count) && hf2_rules;
-    uint64_t found_money = select_transfers(needed_money, add_dust, dust_policy.dust_threshold, hf2_rules, selected_transfers);
     THROW_WALLET_EXCEPTION_IF(found_money < needed_money, error::not_enough_money, found_money, needed_money - fee, fee);
 
     typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry;

src/wallet/wallet_errors.h

@@ -458,15 +458,17 @@ namespace tools
     //----------------------------------------------------------------------------------------------------
     struct tx_rejected : public transfer_error
     {
-      explicit tx_rejected(std::string&& loc, const cryptonote::transaction& tx, const std::string& status)
+      explicit tx_rejected(std::string&& loc, const cryptonote::transaction& tx, const std::string& status, const std::string& reason)
         : transfer_error(std::move(loc), "transaction was rejected by daemon")
         , m_tx(tx)
         , m_status(status)
+        , m_reason(reason)
       {
       }
 
       const cryptonote::transaction& tx() const { return m_tx; }
       const std::string& status() const { return m_status; }
+      const std::string& reason() const { return m_reason; }
 
       std::string to_string() const
       {
@@ -474,12 +476,17 @@ namespace tools
         ss << transfer_error::to_string() << ", status = " << m_status << ", tx:\n";
         cryptonote::transaction tx = m_tx;
         ss << cryptonote::obj_to_json_str(tx);
+        if (!m_reason.empty())
+        {
+          ss << " (" << m_reason << ")";
+        }
         return ss.str();
       }
 
     private:
       cryptonote::transaction m_tx;
       std::string m_status;
+      std::string m_reason;
     };
     //----------------------------------------------------------------------------------------------------
     struct tx_sum_overflow : public transfer_error

src/wallet/wallet_rpc_server.cpp

@@ -232,7 +232,7 @@ namespace tools
         LOG_PRINT_L1("Requested mixin " << req.mixin << " too low for hard fork 2, using 2");
         mixin = 2;
       }
-      std::vector<wallet2::pending_tx> ptx_vector = m_wallet.create_transactions(dsts, mixin, req.unlock_time, req.fee, extra);
+      std::vector<wallet2::pending_tx> ptx_vector = m_wallet.create_transactions(dsts, mixin, req.unlock_time, req.fee, extra, req.trusted_daemon);
 
       // reject proposed transactions if there are more than one.  see on_transfer_split below.
       if (ptx_vector.size() != 1)
@@ -299,9 +299,9 @@ namespace tools
       }
       std::vector<wallet2::pending_tx> ptx_vector;
       if (req.new_algorithm)
-        ptx_vector = m_wallet.create_transactions_2(dsts, mixin, req.unlock_time, req.fee, extra);
+        ptx_vector = m_wallet.create_transactions_2(dsts, mixin, req.unlock_time, req.fee, extra, req.trusted_daemon);
       else
-        ptx_vector = m_wallet.create_transactions(dsts, mixin, req.unlock_time, req.fee, extra);
+        ptx_vector = m_wallet.create_transactions(dsts, mixin, req.unlock_time, req.fee, extra, req.trusted_daemon);
 
       m_wallet.commit_tx(ptx_vector);
 

src/wallet/wallet_rpc_server_commands_defs.h

@@ -115,6 +115,7 @@ namespace wallet_rpc
       uint64_t unlock_time;
       std::string payment_id;
       bool get_tx_key;
+      bool trusted_daemon;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(destinations)
@@ -123,6 +124,7 @@ namespace wallet_rpc
         KV_SERIALIZE(unlock_time)
         KV_SERIALIZE(payment_id)
         KV_SERIALIZE(get_tx_key)
+        KV_SERIALIZE(trusted_daemon)
       END_KV_SERIALIZE_MAP()
     };
 
@@ -149,6 +151,7 @@ namespace wallet_rpc
       std::string payment_id;
       bool new_algorithm;
       bool get_tx_keys;
+      bool trusted_daemon;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(destinations)
@@ -158,6 +161,7 @@ namespace wallet_rpc
         KV_SERIALIZE(payment_id)
         KV_SERIALIZE(new_algorithm)
         KV_SERIALIZE(get_tx_keys)
+        KV_SERIALIZE(trusted_daemon)
       END_KV_SERIALIZE_MAP()
     };
 

tests/core_tests/chaingen.h

@@ -470,13 +470,15 @@ inline bool replay_events_through_core(cryptonote::core& cr, const std::vector<t
   CATCH_ENTRY_L0("replay_events_through_core", false);
 }
 //--------------------------------------------------------------------------
-template<class t_test_class>
+template<typename t_test_class>
 struct get_test_options {
-  const std::pair<uint8_t, uint64_t> hard_forks[1] = {std::make_pair((uint8_t)1, (uint64_t)0)};
+  const std::pair<uint8_t, uint64_t> hard_forks[1];
   const cryptonote::test_options test_options = {
     hard_forks
   };
+  get_test_options():hard_forks{std::make_pair((uint8_t)1, (uint64_t)0)}{}
 };
+
 //--------------------------------------------------------------------------
 template<class t_test_class>
 inline bool do_replay_events(std::vector<test_event_entry>& events)

tests/functional_tests/transactions_flow_test.cpp

@@ -85,7 +85,7 @@ bool do_send_money(tools::wallet2& w1, tools::wallet2& w2, size_t mix_in_factor,
   try
   {
     tools::wallet2::pending_tx ptx;
-    w1.transfer(dsts, mix_in_factor, 0, TEST_FEE, std::vector<uint8_t>(), tools::detail::null_split_strategy, tools::tx_dust_policy(TEST_DUST_THRESHOLD), tx, ptx);
+    w1.transfer(dsts, mix_in_factor, 0, TEST_FEE, std::vector<uint8_t>(), tools::detail::null_split_strategy, tools::tx_dust_policy(TEST_DUST_THRESHOLD), tx, ptx, true);
     w1.commit_tx(ptx);
     return true;
   }

tests/unit_tests/CMakeLists.txt

@@ -49,7 +49,8 @@ set(unit_tests_sources
   test_format_utils.cpp
   test_peerlist.cpp
   test_protocol_pack.cpp
-  hardfork.cpp)
+  hardfork.cpp
+  unbound.cpp)
 
 set(unit_tests_headers
   unit_tests_utils.h)
@@ -69,6 +70,7 @@ target_link_libraries(unit_tests
     ${Boost_REGEX_LIBRARY}
     ${Boost_SYSTEM_LIBRARY}
     ${Boost_THREAD_LIBRARY}
+    ${UNBOUND_LIBRARIES}
     ${EXTRA_LIBRARIES})
 set_property(TARGET unit_tests
   PROPERTY

tests/unit_tests/unbound.cpp

@@ -0,0 +1,53 @@
+// Copyright (c) 2016, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#include "gtest/gtest.h"
+
+#ifdef STATICLIB
+
+extern "C" int dnskey_algo_id_is_supported(int);
+
+TEST(unbound, supported_algorithms)
+{
+  // Monero causes these to be tried, but we don't have access
+  // to this internal unbound header here, so we use raw numbers
+  // LDNS_RSASHA1            = 5,
+  // LDNS_RSASHA1_NSEC3      = 7,
+  // LDNS_RSASHA256          = 8,   /* RFC 5702 */
+  // LDNS_ECDSAP256SHA256    = 13,  /* RFC 6605 */
+
+  ASSERT_TRUE(dnskey_algo_id_is_supported(5));
+  ASSERT_TRUE(dnskey_algo_id_is_supported(7));
+  ASSERT_TRUE(dnskey_algo_id_is_supported(8));
+  ASSERT_TRUE(dnskey_algo_id_is_supported(13));
+}
+
+#endif
+