trezor: adapt to new passphrase mechanism

- choice where to enter passphrase is now made on the host
- use wipeable string in the comm stack
- wipe passphrase memory
- protocol optimizations, prepare for new firmware version
- minor fixes and improvements
- tests fixes, HF12 support

contrib/epee/include/net/http_base.h

@@ -33,6 +33,7 @@
 #include <string>
 #include <utility>
 
+#include "memwipe.h"
 #include "string_tools.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -200,6 +201,11 @@ namespace net_utils
 				this->~http_response_info();
 				new(this) http_response_info();
 			}
+
+			void wipe()
+			{
+				memwipe(&m_body[0], m_body.size());
+			}
 		};
 	}
 }

contrib/epee/include/net/http_client.h

@@ -466,6 +466,11 @@ namespace net_utils
 				return m_net_client.get_bytes_received();
 			}
 			//---------------------------------------------------------------------------
+			void wipe_response()
+			{
+				m_response_info.wipe();
+			}
+			//---------------------------------------------------------------------------
 		private: 
 			//---------------------------------------------------------------------------
 			inline bool handle_reciev(std::chrono::milliseconds timeout)

external/trezor-common

@@ -1 +1 @@
-Subproject commit 31a0073c62738827b48d725facd3766879429124
+Subproject commit bff7fdfe436c727982cc553bdfb29a9021b423b0

src/device/device.hpp

@@ -78,7 +78,7 @@ namespace hw {
         virtual void on_button_request(uint64_t code=0) {}
         virtual void on_button_pressed() {}
         virtual boost::optional<epee::wipeable_string> on_pin_request() { return boost::none; }
-        virtual boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) { return boost::none; }
+        virtual boost::optional<epee::wipeable_string> on_passphrase_request(bool & on_device) { on_device = true; return boost::none; }
         virtual void on_progress(const device_progress& event) {}
         virtual ~i_device_callback() = default;
     };

src/device_trezor/device_trezor.cpp

@@ -101,7 +101,7 @@ namespace trezor {
       return device_trezor_base::disconnect();
     }
 
-    void device_trezor::device_state_reset_unsafe()
+    void device_trezor::device_state_initialize_unsafe()
     {
       require_connected();
       if (m_live_refresh_in_progress)
@@ -117,7 +117,7 @@ namespace trezor {
       }
 
       m_live_refresh_in_progress = false;
-      device_trezor_base::device_state_reset_unsafe();
+      device_trezor_base::device_state_initialize_unsafe();
     }
 
     void device_trezor::live_refresh_thread_main()
@@ -221,7 +221,7 @@ namespace trezor {
       CHECK_AND_ASSERT_THROW_MES(!payment_id || !subaddress || subaddress->is_zero(), "Subaddress cannot be integrated");
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
 
       auto req = std::make_shared<messages::monero::MoneroGetAddress>();
@@ -245,7 +245,7 @@ namespace trezor {
         const boost::optional<cryptonote::network_type> & network_type){
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
 
       auto req = std::make_shared<messages::monero::MoneroGetWatchKey>();
@@ -274,7 +274,7 @@ namespace trezor {
     {
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
 
       auto req = protocol::tx::get_tx_key(tx_aux_data);
@@ -294,15 +294,15 @@ namespace trezor {
 
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
 
       std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> req;
 
       std::vector<protocol::ki::MoneroTransferDetails> mtds;
       std::vector<protocol::ki::MoneroExportedKeyImage> kis;
-      protocol::ki::key_image_data(wallet, transfers, mtds);
+      protocol::ki::key_image_data(wallet, transfers, mtds, client_version() <= 1);
-      protocol::ki::generate_commitment(mtds, transfers, req);
+      protocol::ki::generate_commitment(mtds, transfers, req, client_version() <= 1);
 
       EVENT_PROGRESS(0.);
       this->set_msg_addr<messages::monero::MoneroKeyImageExportInitRequest>(req.get());
@@ -386,7 +386,7 @@ namespace trezor {
 
     void device_trezor::live_refresh_start_unsafe()
     {
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
 
       auto req = std::make_shared<messages::monero::MoneroLiveRefreshStartRequest>();
@@ -492,7 +492,7 @@ namespace trezor {
 
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
       transaction_versions_check(unsigned_tx, aux_data);
 
@@ -514,7 +514,7 @@ namespace trezor {
         auto & cpend = signed_tx.ptx.back();
         cpend.tx = cdata.tx;
         cpend.dust = 0;
-        cpend.fee = 0;
+        cpend.fee = cpend.tx.rct_signatures.txnFee;
         cpend.dust_added_to_fee = false;
         cpend.change_dts = cdata.tx_data.change_dts;
         cpend.selected_transfers = cdata.tx_data.selected_transfers;
@@ -524,6 +524,7 @@ namespace trezor {
 
         // Transaction check
         try {
+          MDEBUG("signed transaction: " << cryptonote::get_transaction_hash(cpend.tx) << ENDL << cryptonote::obj_to_json_str(cpend.tx) << ENDL);
           transaction_check(cdata, aux_data);
         } catch(const std::exception &e){
           throw exc::ProtocolException(std::string("Transaction verification failed: ") + e.what());
@@ -582,7 +583,7 @@ namespace trezor {
 
       require_connected();
       if (idx > 0)
-        device_state_reset_unsafe();
+        device_state_initialize_unsafe();
 
       require_initialized();
       EVENT_PROGRESS(0, 1, 1);
@@ -670,28 +671,42 @@ namespace trezor {
 #undef EVENT_PROGRESS
     }
 
-    void device_trezor::transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data)
+    unsigned device_trezor::client_version()
     {
       auto trezor_version = get_version();
-      unsigned client_version = 1;  // default client version for tx
-
       if (trezor_version <= pack_version(2, 0, 10)){
-        client_version = 0;
+        throw exc::TrezorException("Trezor firmware 2.0.10 and lower are not supported. Please update.");
       }
 
+      // default client version, higher versions check will be added
+      unsigned client_version = 1;
+
+#ifdef WITH_TREZOR_DEBUGGING
+      // Override client version for tests
+      const char *env_trezor_client_version = nullptr;
+      if ((env_trezor_client_version = getenv("TREZOR_CLIENT_VERSION")) != nullptr){
+        auto succ = epee::string_tools::get_xtype_from_string(client_version, env_trezor_client_version);
+        if (succ){
+          MINFO("Trezor client version overriden by TREZOR_CLIENT_VERSION to: " << client_version);
+        }
+      }
+#endif
+      return client_version;
+    }
+
+    void device_trezor::transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data)
+    {
+      unsigned cversion = client_version();
+
       if (aux_data.client_version){
         auto wanted_client_version = aux_data.client_version.get();
-        if (wanted_client_version > client_version){
+        if (wanted_client_version > cversion){
-          throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol. Please update.");
+          throw exc::TrezorException("Trezor has too old firmware version. Please update.");
         } else {
-          client_version = wanted_client_version;
+          cversion = wanted_client_version;
         }
       }
-      aux_data.client_version = client_version;
+      aux_data.client_version = cversion;
-
-      if (client_version == 0 && aux_data.bp_version && aux_data.bp_version.get() != 1){
-        throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol (BPv2+). Please update.");
-      }
     }
 
     void device_trezor::transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg)

src/device_trezor/device_trezor.hpp

@@ -67,10 +67,11 @@ namespace trezor {
       bool m_live_refresh_enabled;
       size_t m_num_transations_to_sign;
 
+      unsigned client_version();
       void transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data);
       void transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg);
       void transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data);
-      void device_state_reset_unsafe() override;
+      void device_state_initialize_unsafe() override;
       void live_refresh_start_unsafe();
       void live_refresh_finish_unsafe();
       void live_refresh_thread_main();

src/device_trezor/device_trezor_base.cpp

@@ -28,6 +28,7 @@
 //
 
 #include "device_trezor_base.hpp"
+#include "memwipe.h"
 #include <boost/algorithm/string/classification.hpp>
 #include <boost/algorithm/string/split.hpp>
 #include <boost/regex.hpp>
@@ -151,7 +152,7 @@ namespace trezor {
 
     bool device_trezor_base::disconnect() {
       TREZOR_AUTO_LOCK_DEVICE();
-      m_device_state.clear();
+      m_device_session_id.clear();
       m_features.reset();
 
       if (m_transport){
@@ -292,8 +293,8 @@ namespace trezor {
         case messages::MessageType_PassphraseRequest:
           on_passphrase_request(input, dynamic_cast<const messages::common::PassphraseRequest*>(input.m_msg.get()));
           return true;
-        case messages::MessageType_PassphraseStateRequest:
+        case messages::MessageType_Deprecated_PassphraseStateRequest:
-          on_passphrase_state_request(input, dynamic_cast<const messages::common::PassphraseStateRequest*>(input.m_msg.get()));
+          on_passphrase_state_request(input, dynamic_cast<const messages::common::Deprecated_PassphraseStateRequest*>(input.m_msg.get()));
           return true;
         case messages::MessageType_PinMatrixRequest:
           on_pin_request(input, dynamic_cast<const messages::common::PinMatrixRequest*>(input.m_msg.get()));
@@ -361,23 +362,34 @@ namespace trezor {
       return false;
     }
 
-    void device_trezor_base::device_state_reset_unsafe()
+    void device_trezor_base::device_state_initialize_unsafe()
     {
       require_connected();
+      std::string tmp_session_id;
       auto initMsg = std::make_shared<messages::management::Initialize>();
+      const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() {
+        memwipe(&tmp_session_id[0], tmp_session_id.size());
+      });
 
-      if(!m_device_state.empty()) {
+      if(!m_device_session_id.empty()) {
-        initMsg->set_allocated_state(&m_device_state);
+        tmp_session_id.assign(m_device_session_id.data(), m_device_session_id.size());
+        initMsg->set_allocated_session_id(&tmp_session_id);
       }
 
       m_features = this->client_exchange<messages::management::Features>(initMsg);
-      initMsg->release_state();
+      if (m_features->has_session_id()){
+        m_device_session_id = m_features->session_id();
+      } else {
+        m_device_session_id.clear();
+      }
+
+      initMsg->release_session_id();
     }
 
     void device_trezor_base::device_state_reset()
     {
       TREZOR_AUTO_LOCK_CMD();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
     }
 
 #ifdef WITH_TREZOR_DEBUGGING
@@ -441,48 +453,89 @@ namespace trezor {
         pin = m_pin;
       }
 
-      // TODO: remove PIN from memory
+      std::string pin_field;
       messages::common::PinMatrixAck m;
       if (pin) {
-        m.set_pin(pin.get().data(), pin.get().size());
+        pin_field.assign(pin->data(), pin->size());
+        m.set_allocated_pin(&pin_field);
       }
+
+      const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() {
+        m.release_pin();
+        if (!pin_field.empty()){
+          memwipe(&pin_field[0], pin_field.size());
+        }
+      });
+
       resp = call_raw(&m);
     }
 
     void device_trezor_base::on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg)
     {
       CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
-      MDEBUG("on_passhprase_request, on device: " << msg->on_device());
+      MDEBUG("on_passhprase_request");
+
+      // Backward compatibility, migration clause.
+      if (msg->has__on_device() && msg->_on_device()){
+        messages::common::PassphraseAck m;
+        resp = call_raw(&m);
+        return;
+      }
+
+      bool on_device = true;
+      if (msg->has__on_device() && !msg->_on_device()){
+        on_device = false;  // do not enter on device, old devices.
+      }
+
+      if (on_device && m_features && m_features->capabilities_size() > 0){
+        on_device = false;
+        for (auto it = m_features->capabilities().begin(); it != m_features->capabilities().end(); it++) {
+          if (*it == messages::management::Features::Capability_PassphraseEntry){
+            on_device = true;
+          }
+        }
+      }
+
       boost::optional<epee::wipeable_string> passphrase;
-      TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, msg->on_device());
+      TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, on_device);
-
-      if (!passphrase && m_passphrase){
-        passphrase = m_passphrase;
-      }
-
-      m_passphrase = boost::none;
 
+      std::string passphrase_field;
       messages::common::PassphraseAck m;
-      if (!msg->on_device() && passphrase){
+      m.set_on_device(on_device);
-        // TODO: remove passphrase from memory
+      if (!on_device) {
-        m.set_passphrase(passphrase.get().data(), passphrase.get().size());
+        if (!passphrase && m_passphrase) {
+          passphrase = m_passphrase;
+        }
+
+        if (m_passphrase) {
+          m_passphrase = boost::none;
+        }
+
+        if (passphrase) {
+          passphrase_field.assign(passphrase->data(), passphrase->size());
+          m.set_allocated_passphrase(&passphrase_field);
+        }
       }
 
-      if (!m_device_state.empty()){
+      const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() {
-        m.set_allocated_state(&m_device_state);
+        m.release_passphrase();
-      }
+        if (!passphrase_field.empty()){
+          memwipe(&passphrase_field[0], passphrase_field.size());
+        }
+      });
 
       resp = call_raw(&m);
-      m.release_state();
     }
 
-    void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg)
+    void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::Deprecated_PassphraseStateRequest * msg)
     {
       MDEBUG("on_passhprase_state_request");
       CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
 
-      m_device_state = msg->state();
+      if (msg->has_state()) {
-      messages::common::PassphraseStateAck m;
+        m_device_session_id = msg->state();
+      }
+      messages::common::Deprecated_PassphraseStateAck m;
       resp = call_raw(&m);
     }
 
@@ -510,7 +563,7 @@ namespace trezor {
       }
 
       auto msg = std::make_shared<messages::management::LoadDevice>();
-      msg->set_mnemonic(mnemonic);
+      msg->add_mnemonics(mnemonic);
       msg->set_pin(pin);
       msg->set_passphrase_protection(passphrase_protection);
       msg->set_label(label);
@@ -535,7 +588,8 @@ namespace trezor {
       return boost::none;
     }
 
-    boost::optional<epee::wipeable_string> trezor_debug_callback::on_passphrase_request(bool on_device) {
+    boost::optional<epee::wipeable_string> trezor_debug_callback::on_passphrase_request(bool & on_device) {
+      on_device = true;
       return boost::none;
     }
 

src/device_trezor/device_trezor_base.hpp

@@ -70,7 +70,7 @@ namespace trezor {
 
       void on_button_request(uint64_t code=0) override;
       boost::optional<epee::wipeable_string> on_pin_request() override;
-      boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override;
+      boost::optional<epee::wipeable_string> on_passphrase_request(bool & on_device) override;
       void on_passphrase_state_request(const std::string &state);
       void on_disconnect();
     protected:
@@ -94,7 +94,7 @@ namespace trezor {
 
       std::string m_full_name;
       std::vector<unsigned int> m_wallet_deriv_path;
-      std::string m_device_state;  // returned after passphrase entry, session
+      epee::wipeable_string m_device_session_id;  // returned after passphrase entry, session
       std::shared_ptr<messages::management::Features> m_features;  // features from the last device reset
       boost::optional<epee::wipeable_string> m_pin;
       boost::optional<epee::wipeable_string> m_passphrase;
@@ -117,7 +117,7 @@ namespace trezor {
       void require_initialized() const;
       void call_ping_unsafe();
       void test_ping();
-      virtual void device_state_reset_unsafe();
+      virtual void device_state_initialize_unsafe();
       void ensure_derivation_path() noexcept;
 
       // Communication methods
@@ -315,7 +315,7 @@ namespace trezor {
     void on_button_pressed();
     void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg);
     void on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg);
-    void on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg);
+    void on_passphrase_state_request(GenericMessage & resp, const messages::common::Deprecated_PassphraseStateRequest * msg);
 
 #ifdef WITH_TREZOR_DEBUGGING
     void set_debug(bool debug){

src/device_trezor/trezor/debug_link.cpp

@@ -71,9 +71,9 @@ namespace trezor{
     call(decision, boost::none, true);
   }
 
-  void DebugLink::input_swipe(bool swipe){
+  void DebugLink::input_swipe(messages::debug::DebugLinkDecision_DebugSwipeDirection direction){
     messages::debug::DebugLinkDecision decision;
-    decision.set_up_down(swipe);
+    decision.set_swipe(direction);
     call(decision, boost::none, true);
   }
 

src/device_trezor/trezor/debug_link.hpp

@@ -49,7 +49,7 @@ namespace trezor {
     std::shared_ptr<messages::debug::DebugLinkState> state();
     void input_word(const std::string & word);
     void input_button(bool button);
-    void input_swipe(bool swipe);
+    void input_swipe(messages::debug::DebugLinkDecision_DebugSwipeDirection direction);
     void press_yes() { input_button(true); }
     void press_no() { input_button(false); }
     void stop();

src/device_trezor/trezor/protocol.cpp

@@ -145,7 +145,8 @@ namespace ki {
 
   bool key_image_data(wallet_shim * wallet,
                       const std::vector<tools::wallet2::transfer_details> & transfers,
-                      std::vector<MoneroTransferDetails> & res)
+                      std::vector<MoneroTransferDetails> & res,
+                      bool need_all_additionals)
   {
     for(auto & td : transfers){
       ::crypto::public_key tx_pub_key = wallet->get_tx_pub_key_from_received_outs(td);
@@ -157,8 +158,14 @@ namespace ki {
       cres.set_out_key(key_to_string(boost::get<cryptonote::txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key));
       cres.set_tx_pub_key(key_to_string(tx_pub_key));
       cres.set_internal_output_index(td.m_internal_output_index);
-      for(auto & aux : additional_tx_pub_keys){
+      cres.set_sub_addr_major(td.m_subaddr_index.major);
-        cres.add_additional_tx_pub_keys(key_to_string(aux));
+      cres.set_sub_addr_minor(td.m_subaddr_index.minor);
+      if (need_all_additionals) {
+        for (auto &aux : additional_tx_pub_keys) {
+          cres.add_additional_tx_pub_keys(key_to_string(aux));
+        }
+      } else if (!additional_tx_pub_keys.empty() && additional_tx_pub_keys.size() > td.m_internal_output_index) {
+        cres.add_additional_tx_pub_keys(key_to_string(additional_tx_pub_keys[td.m_internal_output_index]));
       }
     }
 
@@ -188,7 +195,8 @@ namespace ki {
 
   void generate_commitment(std::vector<MoneroTransferDetails> & mtds,
                            const std::vector<tools::wallet2::transfer_details> & transfers,
-                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req)
+                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req,
+                           bool need_subaddr_indices)
   {
     req = std::make_shared<messages::monero::MoneroKeyImageExportInitRequest>();
 
@@ -213,11 +221,13 @@ namespace ki {
       st.insert(cur.m_subaddr_index.minor);
     }
 
-    for (auto& x: sub_indices){
+    if (need_subaddr_indices) {
-      auto subs = req->add_subs();
+      for (auto &x: sub_indices) {
-      subs->set_account(x.first);
+        auto subs = req->add_subs();
-      for(auto minor : x.second){
+        subs->set_account(x.first);
-        subs->add_minor_indices(minor);
+        for (auto minor : x.second) {
+          subs->add_minor_indices(minor);
+        }
       }
     }
   }
@@ -283,26 +293,6 @@ namespace tx {
     translate_address(dst->mutable_addr(), &(src->addr));
   }
 
-  void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src){
-    for(auto & cur : src->outputs){
-      auto out = dst->add_outputs();
-      out->set_idx(cur.first);
-      translate_rct_key(out->mutable_key(), &(cur.second));
-    }
-
-    dst->set_real_output(src->real_output);
-    dst->set_real_out_tx_key(key_to_string(src->real_out_tx_key));
-    for(auto & cur : src->real_out_additional_tx_keys){
-      dst->add_real_out_additional_tx_keys(key_to_string(cur));
-    }
-
-    dst->set_real_output_in_tx_index(src->real_output_in_tx_index);
-    dst->set_amount(src->amount);
-    dst->set_rct(src->rct);
-    dst->set_mask(key_to_string(src->mask));
-    translate_klrki(dst->mutable_multisig_klrki(), &(src->multisig_kLRki));
-  }
-
   void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src){
     dst->set_k(key_to_string(src->k));
     dst->set_l(key_to_string(src->L));
@@ -369,6 +359,31 @@ namespace tx {
     return res;
   }
 
+  std::string compute_sealing_key(const std::string & master_key, size_t idx, bool is_iv)
+  {
+    // master-key-32B || domain-sep-12B || index-4B
+    uint8_t hash[32] = {0};
+    KECCAK_CTX ctx;
+    std::string sep = is_iv ? "sig-iv" : "sig-key";
+    std::string idx_data = tools::get_varint_data(idx);
+    if (idx_data.size() > 4){
+      throw std::invalid_argument("index is too big");
+    }
+
+    keccak_init(&ctx);
+    keccak_update(&ctx, (const uint8_t *) master_key.data(), master_key.size());
+    keccak_update(&ctx, (const uint8_t *) sep.data(), sep.size());
+    keccak_update(&ctx, hash, 12 - sep.size());
+    keccak_update(&ctx, (const uint8_t *) idx_data.data(), idx_data.size());
+    if (idx_data.size() < 4) {
+      keccak_update(&ctx, hash, 4 - idx_data.size());
+    }
+
+    keccak_finish(&ctx, hash);
+    keccak(hash, sizeof(hash), hash, sizeof(hash));
+    return std::string((const char*) hash, 32);
+  }
+
   TData::TData() {
     rsig_type = 0;
     bp_version = 0;
@@ -383,7 +398,7 @@ namespace tx {
     m_unsigned_tx = unsigned_tx;
     m_aux_data = aux_data;
     m_tx_idx = tx_idx;
-    m_ct.tx_data = cur_tx();
+    m_ct.tx_data = cur_src_tx();
     m_multisig = false;
     m_client_version = 1;
   }
@@ -451,6 +466,41 @@ namespace tx {
     }
   }
 
+  void Signer::set_tx_input(MoneroTransactionSourceEntry * dst, size_t idx, bool need_ring_keys, bool need_ring_indices){
+    const cryptonote::tx_source_entry & src = cur_tx().sources[idx];
+    const tools::wallet2::transfer_details & transfer = get_source_transfer(idx);
+
+    dst->set_real_output(src.real_output);
+    for(size_t i = 0; i < src.outputs.size(); ++i){
+      auto & cur = src.outputs[i];
+      auto out = dst->add_outputs();
+
+      if (i == src.real_output || need_ring_indices || client_version() <= 1) {
+        out->set_idx(cur.first);
+      }
+      if (i == src.real_output || need_ring_keys || client_version() <= 1) {
+        translate_rct_key(out->mutable_key(), &(cur.second));
+      }
+    }
+
+    dst->set_real_out_tx_key(key_to_string(src.real_out_tx_key));
+    dst->set_real_output_in_tx_index(src.real_output_in_tx_index);
+
+    if (client_version() <= 1) {
+      for (auto &cur : src.real_out_additional_tx_keys) {
+        dst->add_real_out_additional_tx_keys(key_to_string(cur));
+      }
+    } else if (!src.real_out_additional_tx_keys.empty()) {
+      dst->add_real_out_additional_tx_keys(key_to_string(src.real_out_additional_tx_keys.at(src.real_output_in_tx_index)));
+    }
+
+    dst->set_amount(src.amount);
+    dst->set_rct(src.rct);
+    dst->set_mask(key_to_string(src.mask));
+    translate_klrki(dst->mutable_multisig_klrki(), &(src.multisig_kLRki));
+    dst->set_subaddr_minor(transfer.m_subaddr_index.minor);
+  }
+
   void Signer::compute_integrated_indices(TsxData * tsx_data){
     if (m_aux_data == nullptr || m_aux_data->tx_recipients.empty()){
       return;
@@ -492,6 +542,7 @@ namespace tx {
     // extract payment ID from construction data
     auto & tsx_data = m_ct.tsx_data;
     auto & tx = cur_tx();
+    const size_t input_size = tx.sources.size();
 
     m_ct.tx.version = 2;
     m_ct.tx.unlock_time = tx.unlock_time;
@@ -500,12 +551,20 @@ namespace tx {
     tsx_data.set_version(1);
     tsx_data.set_client_version(client_version());
     tsx_data.set_unlock_time(tx.unlock_time);
-    tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(tx.sources.size()));
+    tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(input_size));
     tsx_data.set_mixin(static_cast<google::protobuf::uint32>(tx.sources[0].outputs.size() - 1));
     tsx_data.set_account(tx.subaddr_account);
     tsx_data.set_monero_version(std::string(MONERO_VERSION) + "|" + MONERO_VERSION_TAG);
     tsx_data.set_hard_fork(m_aux_data->hard_fork ? m_aux_data->hard_fork.get() : 0);
-    assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end());
+
+    if (client_version() <= 1){
+      assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end());
+    }
+
+    // TODO: use HF_VERSION_CLSAG after CLSAG is merged
+    if (tsx_data.hard_fork() >= 13){
+      throw exc::ProtocolException("CLSAG is not yet implemented");
+    }
 
     // Rsig decision
     auto rsig_data = tsx_data.mutable_rsig_data();
@@ -525,6 +584,11 @@ namespace tx {
       translate_dst_entry(dst, &cur);
     }
 
+    m_ct.source_permutation.clear();
+    for (size_t n = 0; n < input_size; ++n){
+      m_ct.source_permutation.push_back(n);
+    }
+
     compute_integrated_indices(&tsx_data);
 
     int64_t fee = 0;
@@ -559,7 +623,7 @@ namespace tx {
     CHECK_AND_ASSERT_THROW_MES(idx < cur_tx().sources.size(), "Invalid source index");
     m_ct.cur_input_idx = idx;
     auto res = std::make_shared<messages::monero::MoneroTransactionSetInputRequest>();
-    translate_src_entry(res->mutable_src_entr(), &(cur_tx().sources[idx]));
+    set_tx_input(res->mutable_src_entr(), idx, false, true);
     return res;
   }
 
@@ -582,11 +646,6 @@ namespace tx {
   void Signer::sort_ki(){
     const size_t input_size = cur_tx().sources.size();
 
-    m_ct.source_permutation.clear();
-    for (size_t n = 0; n < input_size; ++n){
-      m_ct.source_permutation.push_back(n);
-    }
-
     CHECK_AND_ASSERT_THROW_MES(m_ct.tx.vin.size() == input_size, "Invalid vector size");
     std::sort(m_ct.source_permutation.begin(), m_ct.source_permutation.end(), [&](const size_t i0, const size_t i1) {
       const cryptonote::txin_to_key &tk0 = boost::get<cryptonote::txin_to_key>(m_ct.tx.vin[i0]);
@@ -614,6 +673,9 @@ namespace tx {
 
   std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> Signer::step_permutation(){
     sort_ki();
+    if (client_version() >= 2){
+      return nullptr;
+    }
 
     auto res = std::make_shared<messages::monero::MoneroTransactionInputsPermutationRequest>();
     assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end());
@@ -634,17 +696,10 @@ namespace tx {
     auto tx = m_ct.tx_data;
     auto res = std::make_shared<messages::monero::MoneroTransactionInputViniRequest>();
     auto & vini = m_ct.tx.vin[idx];
-    translate_src_entry(res->mutable_src_entr(), &(tx.sources[idx]));
+    set_tx_input(res->mutable_src_entr(), idx, false, false);
     res->set_vini(cryptonote::t_serializable_object_to_blob(vini));
     res->set_vini_hmac(m_ct.tx_in_hmacs[idx]);
-
+    res->set_orig_idx(m_ct.source_permutation[idx]);
-    if (client_version() == 0) {
-      CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index");
-      CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index");
-      res->set_pseudo_out(m_ct.pseudo_outs[idx]);
-      res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]);
-    }
-
     return res;
   }
 
@@ -657,31 +712,6 @@ namespace tx {
   }
 
   void Signer::step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack){
-    if (client_version() > 0 || !is_offloading()){
-      return;
-    }
-
-    // If offloading, expect rsig configuration.
-    if (!ack->has_rsig_data()){
-      throw exc::ProtocolException("Rsig offloading requires rsig param");
-    }
-
-    auto & rsig_data = ack->rsig_data();
-    if (!rsig_data.has_mask()){
-      throw exc::ProtocolException("Gamma masks not present in offloaded version");
-    }
-
-    auto & mask = rsig_data.mask();
-    if (mask.size() != 32 * num_outputs()){
-      throw exc::ProtocolException("Invalid number of gamma masks");
-    }
-
-    m_ct.rsig_gamma.reserve(num_outputs());
-    for(size_t c=0; c < num_outputs(); ++c){
-      rct::key cmask{};
-      memcpy(cmask.bytes, mask.data() + c * 32, 32);
-      m_ct.rsig_gamma.emplace_back(cmask);
-    }
   }
 
   std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_set_output(size_t idx){
@@ -696,15 +726,6 @@ namespace tx {
     auto & cur_dst = m_ct.tx_data.splitted_dsts[idx];
     translate_dst_entry(res->mutable_dst_entr(), &cur_dst);
     res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]);
-
-    // Range sig offloading to the host
-    // ClientV0 sends offloaded BP with the last message in the batch.
-    // ClientV1 needs additional message after the last message in the batch as BP uses deterministic masks.
-    if (client_version() == 0 && is_offloading() && should_compute_bp_now()) {
-      auto rsig_data = res->mutable_rsig_data();
-      compute_bproof(*rsig_data);
-    }
-
     return res;
   }
 
@@ -814,7 +835,7 @@ namespace tx {
   }
 
   std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_rsig(size_t idx){
-    if (client_version() == 0 || !is_offloading() || !should_compute_bp_now()){
+    if (!is_offloading() || !should_compute_bp_now()){
       return nullptr;
     }
 
@@ -917,11 +938,12 @@ namespace tx {
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.spend_encs.size(), "Invalid transaction index");
 
     auto res = std::make_shared<messages::monero::MoneroTransactionSignInputRequest>();
-    translate_src_entry(res->mutable_src_entr(), &(m_ct.tx_data.sources[idx]));
+    set_tx_input(res->mutable_src_entr(), idx, true, true);
     res->set_vini(cryptonote::t_serializable_object_to_blob(m_ct.tx.vin[idx]));
     res->set_vini_hmac(m_ct.tx_in_hmacs[idx]);
     res->set_pseudo_out_alpha(m_ct.alphas[idx]);
     res->set_spend_key(m_ct.spend_encs[idx]);
+    res->set_orig_idx(m_ct.source_permutation[idx]);
 
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index");
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index");
@@ -931,10 +953,7 @@ namespace tx {
   }
 
   void Signer::step_sign_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSignInputAck> ack){
-    rct::mgSig mg;
+    m_ct.signatures.push_back(ack->signature());
-    if (!cn_deserialize(ack->signature(), mg)){
-      throw exc::ProtocolException("Cannot deserialize mg[i]");
-    }
 
     // Sync updated pseudo_outputs, client_version>=1, HF10+
     if (client_version() >= 1 && ack->has_pseudo_out()){
@@ -948,12 +967,9 @@ namespace tx {
         string_to_key(m_ct.rv->pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out());
       }
     }
-
-    m_ct.rv->p.MGs.push_back(mg);
   }
 
   std::shared_ptr<messages::monero::MoneroTransactionFinalRequest> Signer::step_final(){
-    m_ct.tx.rct_signatures = *(m_ct.rv);
     return std::make_shared<messages::monero::MoneroTransactionFinalRequest>();
   }
 
@@ -976,6 +992,42 @@ namespace tx {
     m_ct.enc_salt1 = ack->salt();
     m_ct.enc_salt2 = ack->rand_mult();
     m_ct.enc_keys = ack->tx_enc_keys();
+
+    // Opening the sealed signatures
+    if (client_version() >= 3){
+      if(!ack->has_opening_key()){
+        throw exc::ProtocolException("Client version 3+ requires sealed signatures");
+      }
+
+      for(size_t i = 0; i < m_ct.signatures.size(); ++i){
+        CHECK_AND_ASSERT_THROW_MES(m_ct.signatures[i].size() > crypto::chacha::TAG_SIZE, "Invalid signature size");
+        std::string nonce = compute_sealing_key(ack->opening_key(), i, true);
+        std::string key = compute_sealing_key(ack->opening_key(), i, false);
+        size_t plen = m_ct.signatures[i].size() - crypto::chacha::TAG_SIZE;
+        std::unique_ptr<uint8_t[]> plaintext(new uint8_t[plen]);
+        uint8_t * buff = plaintext.get();
+
+        protocol::crypto::chacha::decrypt(
+            m_ct.signatures[i].data(),
+            m_ct.signatures[i].size(),
+            reinterpret_cast<const uint8_t *>(key.data()),
+            reinterpret_cast<const uint8_t *>(nonce.data()),
+            reinterpret_cast<char *>(buff), &plen);
+        m_ct.signatures[i].assign(reinterpret_cast<const char *>(buff), plen);
+      }
+    }
+
+    // CLSAG support comes here once it is merged to the Monero
+    m_ct.rv->p.MGs.reserve(m_ct.signatures.size());
+    for(size_t i = 0; i < m_ct.signatures.size(); ++i) {
+      rct::mgSig mg;
+      if (!cn_deserialize(m_ct.signatures[i], mg)) {
+        throw exc::ProtocolException("Cannot deserialize mg[i]");
+      }
+      m_ct.rv->p.MGs.push_back(mg);
+    }
+
+    m_ct.tx.rct_signatures = *(m_ct.rv);
   }
 
   std::string Signer::store_tx_aux_info(){

src/device_trezor/trezor/protocol.hpp

@@ -118,7 +118,8 @@ namespace ki {
    */
   bool key_image_data(wallet_shim * wallet,
                       const std::vector<tools::wallet2::transfer_details> & transfers,
-                      std::vector<MoneroTransferDetails> & res);
+                      std::vector<MoneroTransferDetails> & res,
+                      bool need_all_additionals=false);
 
   /**
    * Computes a hash over MoneroTransferDetails. Commitment used in the KI sync.
@@ -130,7 +131,8 @@ namespace ki {
    */
   void generate_commitment(std::vector<MoneroTransferDetails> & mtds,
                            const std::vector<tools::wallet2::transfer_details> & transfers,
-                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req);
+                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req,
+                           bool need_subaddr_indices=false);
 
   /**
    * Processes Live refresh step response, parses KI, checks the signature
@@ -158,13 +160,13 @@ namespace tx {
 
   void translate_address(MoneroAccountPublicAddress * dst, const cryptonote::account_public_address * src);
   void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src);
-  void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src);
   void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src);
   void translate_rct_key(MoneroRctKey * dst, const rct::ctkey * src);
   std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none);
   std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none);
   std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none);
   ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt);
+  std::string compute_sealing_key(const std::string & master_key, size_t idx, bool is_iv=false);
 
   typedef boost::variant<rct::rangeSig, rct::Bulletproof> rsig_v;
 
@@ -198,6 +200,7 @@ namespace tx {
     std::vector<std::string> pseudo_outs_hmac;
     std::vector<std::string> couts;
     std::vector<std::string> couts_dec;
+    std::vector<std::string> signatures;
     std::vector<rct::key> rsig_gamma;
     std::string tx_prefix_hash;
     std::string enc_salt1;
@@ -221,16 +224,33 @@ namespace tx {
     unsigned m_client_version;
     bool m_multisig;
 
-    const tx_construction_data & cur_tx(){
+    const tx_construction_data & cur_src_tx() const {
       CHECK_AND_ASSERT_THROW_MES(m_tx_idx < m_unsigned_tx->txes.size(), "Invalid transaction index");
       return m_unsigned_tx->txes[m_tx_idx];
     }
 
+    const tx_construction_data & cur_tx() const {
+      return m_ct.tx_data;
+    }
+
+    const tools::wallet2::transfer_details & get_transfer(size_t idx) const {
+      CHECK_AND_ASSERT_THROW_MES(idx < m_unsigned_tx->transfers.second.size() + m_unsigned_tx->transfers.first && idx >= m_unsigned_tx->transfers.first, "Invalid transfer index");
+      return m_unsigned_tx->transfers.second[idx - m_unsigned_tx->transfers.first];
+    }
+
+    const tools::wallet2::transfer_details & get_source_transfer(size_t idx) const {
+      const auto & sel_transfers = cur_tx().selected_transfers;
+      CHECK_AND_ASSERT_THROW_MES(idx < m_ct.source_permutation.size(), "Invalid source index - permutation");
+      CHECK_AND_ASSERT_THROW_MES(m_ct.source_permutation[idx] < sel_transfers.size(), "Invalid source index");
+      return get_transfer(sel_transfers.at(m_ct.source_permutation[idx]));
+    }
+
     void extract_payment_id();
     void compute_integrated_indices(TsxData * tsx_data);
     bool should_compute_bp_now() const;
     void compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data);
     void process_bproof(rct::Bulletproof & bproof);
+    void set_tx_input(MoneroTransactionSourceEntry * dst, size_t idx, bool need_ring_keys=false, bool need_ring_indices=false);
 
   public:
     Signer(wallet_shim * wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx = 0, hw::tx_aux_data * aux_data = nullptr);

src/device_trezor/trezor/transport.cpp

@@ -56,6 +56,11 @@ namespace trezor{
     return true;
   }
 
+  bool t_serialize(const epee::wipeable_string & in, std::string & out){
+    out.assign(in.data(), in.size());
+    return true;
+  }
+
   bool t_serialize(const json_val & in, std::string & out){
     rapidjson::StringBuffer sb;
     rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
@@ -75,6 +80,11 @@ namespace trezor{
     return true;
   }
 
+  bool t_deserialize(std::string & in, epee::wipeable_string & out){
+    out = epee::wipeable_string(in);
+    return true;
+  }
+
   bool t_deserialize(const std::string & in, json & out){
     if (out.Parse(in.c_str()).HasParseError()) {
       throw exc::CommunicationException("JSON parse error");
@@ -192,61 +202,69 @@ namespace trezor{
     const auto msg_size = message_size(req);
     const auto buff_size = serialize_message_buffer_size(msg_size) + 2;
 
-    std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]);
+    epee::wipeable_string req_buff;
-    uint8_t * req_buff_raw = req_buff.get();
+    epee::wipeable_string chunk_buff;
+
+    req_buff.resize(buff_size);
+    chunk_buff.resize(REPLEN);
+
+    uint8_t * req_buff_raw = reinterpret_cast<uint8_t *>(req_buff.data());
+    uint8_t * chunk_buff_raw = reinterpret_cast<uint8_t *>(chunk_buff.data());
+
     req_buff_raw[0] = '#';
     req_buff_raw[1] = '#';
 
     serialize_message(req, msg_size, req_buff_raw + 2, buff_size - 2);
 
     size_t offset = 0;
-    uint8_t chunk_buff[REPLEN];
 
     // Chunk by chunk upload
     while(offset < buff_size){
       auto to_copy = std::min((size_t)(buff_size - offset), (size_t)(REPLEN - 1));
 
-      chunk_buff[0] = '?';
+      chunk_buff_raw[0] = '?';
-      memcpy(chunk_buff + 1, req_buff_raw + offset, to_copy);
+      memcpy(chunk_buff_raw + 1, req_buff_raw + offset, to_copy);
 
       // Pad with zeros
       if (to_copy < REPLEN - 1){
-        memset(chunk_buff + 1 + to_copy, 0, REPLEN - 1 - to_copy);
+        memset(chunk_buff_raw + 1 + to_copy, 0, REPLEN - 1 - to_copy);
       }
 
-      transport.write_chunk(chunk_buff, REPLEN);
+      transport.write_chunk(chunk_buff_raw, REPLEN);
       offset += REPLEN - 1;
     }
   }
 
   void ProtocolV1::read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type){
-    char chunk[REPLEN];
+    epee::wipeable_string chunk_buff;
+    chunk_buff.resize(REPLEN);
+    char * chunk_buff_raw = chunk_buff.data();
 
     // Initial chunk read
-    size_t nread = transport.read_chunk(chunk, REPLEN);
+    size_t nread = transport.read_chunk(chunk_buff_raw, REPLEN);
     if (nread != REPLEN){
       throw exc::CommunicationException("Read chunk has invalid size");
     }
 
-    if (strncmp(chunk, "?##", 3) != 0){
+    if (memcmp(chunk_buff_raw, "?##", 3) != 0){
       throw exc::CommunicationException("Malformed chunk");
     }
 
     uint16_t tag;
     uint32_t len;
     nread -= 3 + 6;
-    deserialize_message_header(chunk + 3, tag, len);
+    deserialize_message_header(chunk_buff_raw + 3, tag, len);
 
-    std::string data_acc(chunk + 3 + 6, nread);
+    epee::wipeable_string data_acc(chunk_buff_raw + 3 + 6, nread);
     data_acc.reserve(len);
 
     while(nread < len){
-      const size_t cur = transport.read_chunk(chunk, REPLEN);
+      const size_t cur = transport.read_chunk(chunk_buff_raw, REPLEN);
-      if (chunk[0] != '?'){
+      if (chunk_buff_raw[0] != '?'){
         throw exc::CommunicationException("Chunk malformed");
       }
 
-      data_acc.append(chunk + 1, cur - 1);
+      data_acc.append(chunk_buff_raw + 1, cur - 1);
       nread += cur - 1;
     }
 
@@ -259,7 +277,7 @@ namespace trezor{
     }
 
     std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(tag));
-    if (!msg_wrap->ParseFromArray(data_acc.c_str(), len)){
+    if (!msg_wrap->ParseFromArray(data_acc.data(), len)){
       throw exc::CommunicationException("Message could not be parsed");
     }
 
@@ -426,15 +444,16 @@ namespace trezor{
 
     const auto msg_size = message_size(req);
     const auto buff_size = serialize_message_buffer_size(msg_size);
+    epee::wipeable_string req_buff;
+    req_buff.resize(buff_size);
 
-    std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]);
+    uint8_t * req_buff_raw = reinterpret_cast<uint8_t *>(req_buff.data());
-    uint8_t * req_buff_raw = req_buff.get();
 
     serialize_message(req, msg_size, req_buff_raw, buff_size);
 
     std::string uri = "/call/" + m_session.get();
-    std::string req_hex = epee::to_hex::string(epee::span<const std::uint8_t>(req_buff_raw, buff_size));
+    epee::wipeable_string res_hex;
-    std::string res_hex;
+    epee::wipeable_string req_hex = epee::to_hex::wipeable_string(epee::span<const std::uint8_t>(req_buff_raw, buff_size));
 
     bool req_status = invoke_bridge_http(uri, req_hex, res_hex, m_http_client);
     if (!req_status){
@@ -449,15 +468,15 @@ namespace trezor{
       throw exc::CommunicationException("Could not read, no response stored");
     }
 
-    std::string bin_data;
+    boost::optional<epee::wipeable_string> bin_data = m_response->parse_hexstr();
-    if (!epee::string_tools::parse_hexstr_to_binbuff(m_response.get(), bin_data)){
+    if (!bin_data){
       throw exc::CommunicationException("Response is not well hexcoded");
     }
 
     uint16_t msg_tag;
     uint32_t msg_len;
-    deserialize_message_header(bin_data.c_str(), msg_tag, msg_len);
+    deserialize_message_header(bin_data->data(), msg_tag, msg_len);
-    if (bin_data.size() != msg_len + 6){
+    if (bin_data->size() != msg_len + 6){
       throw exc::CommunicationException("Response is not well hexcoded");
     }
 
@@ -466,7 +485,7 @@ namespace trezor{
     }
 
     std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(msg_tag));
-    if (!msg_wrap->ParseFromArray(bin_data.c_str() + 6, msg_len)){
+    if (!msg_wrap->ParseFromArray(bin_data->data() + 6, msg_len)){
       throw exc::EncodingException("Response is not well hexcoded");
     }
     msg = msg_wrap;

src/device_trezor/trezor/transport.hpp

@@ -66,10 +66,12 @@ namespace trezor {
 
   // Base HTTP comm serialization.
   bool t_serialize(const std::string & in, std::string & out);
+  bool t_serialize(const epee::wipeable_string & in, std::string & out);
   bool t_serialize(const json_val & in, std::string & out);
   std::string t_serialize(const json_val & in);
 
   bool t_deserialize(const std::string & in, std::string & out);
+  bool t_deserialize(std::string & in, epee::wipeable_string & out);
   bool t_deserialize(const std::string & in, json & out);
 
   // Flexible json serialization. HTTP client tailored for bridge API
@@ -84,6 +86,13 @@ namespace trezor {
     additional_params.push_back(std::make_pair("Content-Type","application/json; charset=utf-8"));
 
     const http::http_response_info* pri = nullptr;
+    const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() {
+      if (!req_param.empty()) {
+        memwipe(&req_param[0], req_param.size());
+      }
+      transport.wipe_response();
+    });
+
     if(!transport.invoke(uri, method, req_param, timeout, &pri, std::move(additional_params)))
     {
       MERROR("Failed to invoke http request to  " << uri);
@@ -103,7 +112,7 @@ namespace trezor {
       return false;
     }
 
-    return t_deserialize(pri->m_body, result_struct);
+    return t_deserialize(const_cast<http::http_response_info*>(pri)->m_body, result_struct);
   }
 
   // Forward decl
@@ -186,7 +195,7 @@ namespace trezor {
     std::string m_bridge_host;
     boost::optional<std::string> m_device_path;
     boost::optional<std::string> m_session;
-    boost::optional<std::string> m_response;
+    boost::optional<epee::wipeable_string> m_response;
     boost::optional<json> m_device_info;
   };
 

src/simplewallet/simplewallet.cpp

@@ -5576,14 +5576,19 @@ boost::optional<epee::wipeable_string> simple_wallet::on_device_pin_request()
   return pwd_container->password();
 }
 //----------------------------------------------------------------------------------------------------
-boost::optional<epee::wipeable_string> simple_wallet::on_device_passphrase_request(bool on_device)
+boost::optional<epee::wipeable_string> simple_wallet::on_device_passphrase_request(bool & on_device)
 {
-  if (on_device){
+  if (on_device) {
-    message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device");
+    std::string accepted = input_line(tr(
-    return boost::none;
+        "Device asks for passphrase. Do you want to enter the passphrase on device (Y) (or on the host (N))?"));
+    if (std::cin.eof() || command_line::is_yes(accepted)) {
+      message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device");
+      return boost::none;
+    }
   }
 
   PAUSE_READLINE();
+  on_device = false;
   std::string msg = tr("Enter device passphrase");
   auto pwd_container = tools::password_container::prompt(false, msg.c_str());
   THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device passphrase"));

src/simplewallet/simplewallet.h

@@ -348,7 +348,7 @@ namespace cryptonote
     virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason);
     virtual void on_device_button_request(uint64_t code);
     virtual boost::optional<epee::wipeable_string> on_device_pin_request();
-    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device);
+    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device);
     //----------------------------------------------------------
 
     friend class refresh_progress_reporter_t;

src/wallet/api/wallet.cpp

@@ -267,13 +267,15 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback
       return boost::none;
     }
 
-    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device)
+    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device)
     {
       if (m_listener) {
         auto passphrase = m_listener->onDevicePassphraseRequest(on_device);
-        if (!on_device && passphrase) {
+        if (passphrase) {
           return boost::make_optional(epee::wipeable_string((*passphrase).data(), (*passphrase).size()));
         }
+      } else {
+        on_device = true;
       }
       return boost::none;
     }

src/wallet/api/wallet2_api.h

@@ -400,8 +400,8 @@ struct WalletListener
     /**
      * @brief called by device when passphrase entry is needed
      */
-    virtual optional<std::string> onDevicePassphraseRequest(bool on_device) {
+    virtual optional<std::string> onDevicePassphraseRequest(bool & on_device) {
-        if (!on_device) throw std::runtime_error("Not supported");
+        on_device = true;
         return optional<std::string>();
     }
 

src/wallet/wallet2.cpp

@@ -1109,10 +1109,12 @@ boost::optional<epee::wipeable_string> wallet_device_callback::on_pin_request()
   return boost::none;
 }
 
-boost::optional<epee::wipeable_string> wallet_device_callback::on_passphrase_request(bool on_device)
+boost::optional<epee::wipeable_string> wallet_device_callback::on_passphrase_request(bool & on_device)
 {
   if (wallet)
     return wallet->on_device_passphrase_request(on_device);
+  else
+    on_device = true;
   return boost::none;
 }
 
@@ -13547,10 +13549,12 @@ boost::optional<epee::wipeable_string> wallet2::on_device_pin_request()
   return boost::none;
 }
 //----------------------------------------------------------------------------------------------------
-boost::optional<epee::wipeable_string> wallet2::on_device_passphrase_request(bool on_device)
+boost::optional<epee::wipeable_string> wallet2::on_device_passphrase_request(bool & on_device)
 {
   if (nullptr != m_callback)
     return m_callback->on_device_passphrase_request(on_device);
+  else
+    on_device = true;
   return boost::none;
 }
 //----------------------------------------------------------------------------------------------------

src/wallet/wallet2.h

@@ -145,7 +145,7 @@ private:
     virtual void on_device_button_request(uint64_t code) {}
     virtual void on_device_button_pressed() {}
     virtual boost::optional<epee::wipeable_string> on_device_pin_request() { return boost::none; }
-    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device) { return boost::none; }
+    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device) { on_device = true; return boost::none; }
     virtual void on_device_progress(const hw::device_progress& event) {};
     // Common callbacks
     virtual void on_pool_tx_removed(const crypto::hash &txid) {}
@@ -159,7 +159,7 @@ private:
     void on_button_request(uint64_t code=0) override;
     void on_button_pressed() override;
     boost::optional<epee::wipeable_string> on_pin_request() override;
-    boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override;
+    boost::optional<epee::wipeable_string> on_passphrase_request(bool & on_device) override;
     void on_progress(const hw::device_progress& event) override;
   private:
     wallet2 * wallet;
@@ -1487,7 +1487,7 @@ private:
     void on_device_button_request(uint64_t code);
     void on_device_button_pressed();
     boost::optional<epee::wipeable_string> on_device_pin_request();
-    boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device);
+    boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device);
     void on_device_progress(const hw::device_progress& event);
 
     std::string get_rpc_status(const std::string &s) const;

tests/trezor/daemon.cpp

@@ -129,7 +129,7 @@ void mock_daemon::init()
   m_rpc_server.nettype(m_network_type);
 
   CHECK_AND_ASSERT_THROW_MES(m_protocol.init(m_vm), "Failed to initialize cryptonote protocol.");
-  CHECK_AND_ASSERT_THROW_MES(m_rpc_server.init(m_vm, false, main_rpc_port), "Failed to initialize RPC server.");
+  CHECK_AND_ASSERT_THROW_MES(m_rpc_server.init(m_vm, false, main_rpc_port, false), "Failed to initialize RPC server.");
 
   if (m_start_p2p)
     CHECK_AND_ASSERT_THROW_MES(m_server.init(m_vm), "Failed to initialize p2p server.");
@@ -313,7 +313,7 @@ void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_addres
 {
   bool blocks_mined = false;
   const uint64_t start_height = get_height();
-  const auto mining_timeout = std::chrono::seconds(30);
+  const auto mining_timeout = std::chrono::seconds(120);
   MDEBUG("Current height before mining: " << start_height);
 
   start_mining(miner_address);

tests/trezor/daemon.h

@@ -76,7 +76,7 @@ public:
   typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw;
   typedef nodetool::node_server<t_protocol_raw> t_node_server;
 
-  static constexpr const std::chrono::seconds rpc_timeout = std::chrono::seconds(60);
+  static constexpr const std::chrono::seconds rpc_timeout = std::chrono::seconds(120);
 
   cryptonote::core * m_core;
   t_protocol_raw m_protocol;

tests/trezor/trezor_tests.cpp

@@ -38,6 +38,7 @@
 using namespace cryptonote;
 
 #include <boost/regex.hpp>
+#include <common/apply_permutation.h>
 #include "common/util.h"
 #include "common/command_line.h"
 #include "trezor_tests.h"
@@ -72,9 +73,10 @@ namespace
 #define TREZOR_SETUP_CHAIN(NAME) do {                                                                           \
   ++tests_count;                                                                                                \
   try {                                                                                                         \
-    setup_chain(core, trezor_base, chain_path, fix_chain, vm_core);                                            \
+    setup_chain(core, trezor_base, chain_path, fix_chain, vm_core);                                             \
   } catch (const std::exception& ex) {                                                                          \
-    failed_tests.emplace_back("gen_trezor_base " #NAME);                                                        \
+    MERROR("Chain setup failed for " << NAME);                                                                  \
+    throw;                                                                                                      \
   }                                                                                                             \
 } while(0)
 
@@ -136,10 +138,11 @@ int main(int argc, char* argv[])
     hw::register_device(HW_TREZOR_NAME, ensure_trezor_test_device());  // shim device for call tracking
 
     // Bootstrapping common chain & accounts
-    const uint8_t initial_hf =  (uint8_t)get_env_long("TEST_MIN_HF", 11);
+    const uint8_t initial_hf =  (uint8_t)get_env_long("TEST_MIN_HF", 12);
-    const uint8_t max_hf = (uint8_t)get_env_long("TEST_MAX_HF", 11);
+    const uint8_t max_hf = (uint8_t)get_env_long("TEST_MAX_HF", 12);
+    auto sync_test = get_env_long("TEST_KI_SYNC", 1);
     MINFO("Test versions " << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")");
-    MINFO("Testing hardforks [" << (int)initial_hf << ", " << (int)max_hf << "]");
+    MINFO("Testing hardforks [" << (int)initial_hf << ", " << (int)max_hf << "], sync-test: " << sync_test);
 
     cryptonote::core core_obj(nullptr);
     cryptonote::core * const core = &core_obj;
@@ -181,7 +184,7 @@ int main(int argc, char* argv[])
       trezor_base.daemon(daemon);
 
       // Hard-fork independent tests
-      if (hf == initial_hf)
+      if (hf == initial_hf && sync_test > 0)
       {
         TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync_without_refresh, core, trezor_base);
         TREZOR_COMMON_TEST_CASE(gen_trezor_live_refresh, core, trezor_base);
@@ -191,7 +194,6 @@ int main(int argc, char* argv[])
       TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo, core, trezor_base);
       TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short, core, trezor_base);
       TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short_integrated, core, trezor_base);
-      TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_long, core, trezor_base);
       TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo, core, trezor_base);
       TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_acc1, core, trezor_base);
       TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_sub, core, trezor_base);
@@ -338,10 +340,7 @@ static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base,
     if (!unserialize_chain_from_file(events, trezor_base, chain_path))
     {
       MERROR("Failed to deserialize data from file: " << chain_path);
-      if (!fix_chain)
+      CHECK_AND_ASSERT_THROW_MES(fix_chain, "Chain load error");
-      {
-        throw std::runtime_error("Chain load error");
-      }
     } else
     {
       trezor_base.load(events);
@@ -648,6 +647,8 @@ void gen_trezor_base::fork(gen_trezor_base & other)
   other.m_alice_account = m_alice_account;
   other.m_eve_account = m_eve_account;
   other.m_trezor = m_trezor;
+  other.m_generator.set_events(&other.m_events);
+  other.m_generator.set_network_type(m_network_type);
 }
 
 void gen_trezor_base::clear()
@@ -700,6 +701,8 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events)
 
   // Events, custom genesis so it matches wallet genesis
   auto & generator = m_generator;  // macro shortcut
+  generator.set_events(&events);
+  generator.set_network_type(m_network_type);
 
   cryptonote::block blk_gen;
   std::vector<size_t> block_weights;
@@ -852,6 +855,8 @@ void gen_trezor_base::load(std::vector<test_event_entry>& events)
 {
   init_fields();
   m_events = events;
+  m_generator.set_events(&m_events);
+  m_generator.set_network_type(m_network_type);
 
   unsigned acc_idx = 0;
   cryptonote::account_base * accounts[] = {TREZOR_ACCOUNT_ORDERING};
@@ -919,29 +924,19 @@ void gen_trezor_base::rewind_blocks(std::vector<test_event_entry>& events, size_
 void gen_trezor_base::fix_hf(std::vector<test_event_entry>& events)
 {
   // If current test requires higher hard-fork, move it up
-  const auto current_hf = m_hard_forks.back().first;
+  auto current_hf = m_hard_forks.back().first;
+  CHECK_AND_ASSERT_THROW_MES(current_hf <= m_top_hard_fork, "Generated chain hardfork is higher than desired maximum");
+  CHECK_AND_ASSERT_THROW_MES(m_rct_config.bp_version != 2 || m_top_hard_fork >= 10, "Desired maximum is too low for BPv2");
 
-  if (current_hf > m_top_hard_fork)
+  for(;current_hf < m_top_hard_fork; current_hf+=1)
-  {
-    throw std::runtime_error("Generated chain hardfork is higher than desired maximum");
-  }
-
-  if (m_rct_config.bp_version == 2 && m_top_hard_fork < 10)
-  {
-    throw std::runtime_error("Desired maximum is too low for BPv2");
-  }
-
-  if (current_hf < m_top_hard_fork)
   {
+    auto const hf_to_add = current_hf + 1;
     auto hardfork_height = num_blocks(events);
-    ADD_HARDFORK(m_hard_forks, m_top_hard_fork, hardfork_height);
-    add_top_hfork(events, m_hard_forks);
-    MDEBUG("Hardfork added at height: " << hardfork_height << ", from " << (int)current_hf << " to " << (int)m_top_hard_fork);
 
-    if (current_hf < 10)
+    ADD_HARDFORK(m_hard_forks, hf_to_add, hardfork_height);
-    { // buffer blocks, add 10 to apply v10 rules
+    add_top_hfork(events, m_hard_forks);
-      rewind_blocks(events, 10, m_top_hard_fork);
+    MDEBUG("Hardfork added at height: " << hardfork_height << ", from " << (int)current_hf << " to " << (int)hf_to_add);
-    }
+    rewind_blocks(events, 10, hf_to_add);
   }
 }
 
@@ -1271,7 +1266,6 @@ void gen_trezor_base::set_hard_fork(uint8_t hf)
 
 #define TREZOR_SKIP_IF_VERSION_LEQ(x) if (m_trezor->get_version() <= x) { MDEBUG("Test skipped"); return true; }
 #define TREZOR_TEST_PAYMENT_ID "\xde\xad\xc0\xde\xde\xad\xc0\xde"
-#define TREZOR_TEST_PAYMENT_ID_LONG "\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde"
 
 tsx_builder * tsx_builder::sources(std::vector<cryptonote::tx_source_entry> & sources, std::vector<size_t> & selected_transfers)
 {
@@ -1424,13 +1418,26 @@ tsx_builder * tsx_builder::construct_pending_tx(tools::wallet2::pending_tx &ptx,
   std::vector<crypto::secret_key> additional_tx_keys;
   std::vector<tx_destination_entry> destinations_copy = m_destinations;
 
+  auto sources_copy = m_sources;
   auto change_addr = m_from->get_account().get_keys().m_account_address;
   bool r = construct_tx_and_get_tx_key(m_from->get_account().get_keys(), subaddresses, m_sources, destinations_copy,
                                        change_addr, extra ? extra.get() : std::vector<uint8_t>(), tx, 0, tx_key,
                                        additional_tx_keys, true, m_rct_config, nullptr);
-
   CHECK_AND_ASSERT_THROW_MES(r, "Transaction construction failed");
 
+  // Selected transfers permutation
+  std::vector<size_t> ins_order;
+  for (size_t n = 0; n < m_sources.size(); ++n)
+  {
+    for (size_t idx = 0; idx < sources_copy.size(); ++idx)
+    {
+      CHECK_AND_ASSERT_THROW_MES((size_t)sources_copy[idx].real_output < sources_copy[idx].outputs.size(), "Invalid real_output");
+      if (sources_copy[idx].outputs[sources_copy[idx].real_output].second.dest == m_sources[n].outputs[m_sources[n].real_output].second.dest)
+        ins_order.push_back(idx);
+    }
+  }
+  CHECK_AND_ASSERT_THROW_MES(ins_order.size() == m_sources.size(), "Failed to work out sources permutation");
+
   ptx.key_images = "";
   ptx.fee = TESTS_DEFAULT_FEE;
   ptx.dust = 0;
@@ -1438,6 +1445,7 @@ tsx_builder * tsx_builder::construct_pending_tx(tools::wallet2::pending_tx &ptx,
   ptx.tx = tx;
   ptx.change_dts = m_destinations.back();
   ptx.selected_transfers = m_selected_transfers;
+  tools::apply_permutation(ins_order, ptx.selected_transfers);
   ptx.tx_key = tx_key;
   ptx.additional_tx_keys = additional_tx_keys;
   ptx.dests = m_destinations;
@@ -1671,22 +1679,6 @@ bool gen_trezor_1utxo_paymentid_short_integrated::generate(std::vector<test_even
   TREZOR_TEST_SUFFIX();
 }
 
-bool gen_trezor_1utxo_paymentid_long::generate(std::vector<test_event_entry>& events)
-{
-  TREZOR_TEST_PREFIX();
-  t_builder->cur_height(num_blocks(events) - 1)
-      ->mixin(TREZOR_TEST_MIXIN)
-      ->fee(TREZOR_TEST_FEE)
-      ->from(m_wl_alice.get(), 0)
-      ->compute_sources(boost::none, MK_COINS(1), -1, -1)
-      ->add_destination(m_eve_account, false, 1000)
-      ->payment_id(TREZOR_TEST_PAYMENT_ID_LONG)
-      ->rct_config(m_rct_config)
-      ->build_tx();
-
-  TREZOR_TEST_SUFFIX();
-}
-
 bool gen_trezor_4utxo::generate(std::vector<test_event_entry>& events)
 {
   TREZOR_TEST_PREFIX();

tests/trezor/trezor_tests.h

@@ -264,12 +264,6 @@ public:
   bool generate(std::vector<test_event_entry>& events) override;
 };
 
-class gen_trezor_1utxo_paymentid_long : public gen_trezor_base
-{
-public:
-  bool generate(std::vector<test_event_entry>& events) override;
-};
-
 class gen_trezor_4utxo : public gen_trezor_base
 {
 public: