From 0d90123cac012a824120dbd2ac3e06209f039ab3 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 26 Feb 2017 11:59:42 +0000 Subject: [PATCH 1/8] http_client: allow derived class to get headers at start --- contrib/epee/include/net/http_client.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index d73eda39..1e497074 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -336,6 +336,11 @@ using namespace std; piece_of_transfer.clear(); return true; } + //--------------------------------------------------------------------------- + virtual bool on_header(const http_response_info &headers) + { + return true; + } //--------------------------------------------------------------------------- inline bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) @@ -505,6 +510,12 @@ using namespace std; m_header_cache.erase(m_header_cache.begin()+pos+4, m_header_cache.end()); analize_cached_header_and_invoke_state(); + if (!on_header(m_response_info)) + { + MDEBUG("Connection cancelled by on_header"); + m_state = reciev_machine_state_done; + return false; + } m_header_cache.clear(); if(!recv_buff.size() && (m_state != reciev_machine_state_error && m_state != reciev_machine_state_done)) need_more_data = true; From 9bf017edf200cdb39e576b28d0263223a5afcaf9 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 26 Feb 2017 20:58:07 +0000 Subject: [PATCH 2/8] http_client: allow cancelling a download --- contrib/epee/include/net/http_client.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 1e497074..67e63f7b 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -538,7 +538,11 @@ using namespace std; } CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_len_in_remain >= recv_buff.size()"); m_len_in_remain -= recv_buff.size(); - m_pcontent_encoding_handler->update_in(recv_buff); + if (!m_pcontent_encoding_handler->update_in(recv_buff)) + { + m_state = reciev_machine_state_done; + return false; + } if(m_len_in_remain == 0) m_state = reciev_machine_state_done; @@ -711,7 +715,11 @@ using namespace std; m_len_in_remain = 0; } - m_pcontent_encoding_handler->update_in(chunk_body); + if (!m_pcontent_encoding_handler->update_in(chunk_body)) + { + m_state = reciev_machine_state_error; + return false; + } if(!m_len_in_remain) m_chunked_state = http_chunked_state_chunk_head; From 63f0e074ebcce0dff6f78072ed751aaa1d9b601f Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 26 Feb 2017 12:00:45 +0000 Subject: [PATCH 3/8] download: async API --- src/common/download.cpp | 169 ++++++++++++++++++++++++++++++++++------ src/common/download.h | 10 ++- 2 files changed, 153 insertions(+), 26 deletions(-) diff --git a/src/common/download.cpp b/src/common/download.cpp index c5ee797d..9c2330b1 100644 --- a/src/common/download.cpp +++ b/src/common/download.cpp @@ -40,27 +40,70 @@ namespace tools { - static bool download_thread(const std::string &path, const std::string &url) + struct download_thread_control { + const std::string path; + const std::string uri; + std::function result_cb; + std::function progress_cb; + bool stop; + bool stopped; + bool success; + boost::thread thread; + boost::mutex mutex; + + download_thread_control(const std::string &path, const std::string &uri, std::function result_cb, std::function progress_cb): + path(path), uri(uri), result_cb(result_cb), progress_cb(progress_cb), stop(false), stopped(false), success(false) {} + ~download_thread_control() { if (thread.joinable()) thread.detach(); } + }; + + static void download_thread(download_async_handle control) + { + struct stopped_setter + { + stopped_setter(const download_async_handle &control): control(control) {} + ~stopped_setter() { control->stopped = true; } + download_async_handle control; + } stopped_setter(control); + try { - MINFO("Downloading " << url << " to " << path); + boost::unique_lock lock(control->mutex); + MINFO("Downloading " << control->uri << " to " << control->path); std::ofstream f; - f.open(path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); + f.open(control->path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); if (!f.good()) { - MERROR("Failed to open file " << path); - return false; + MERROR("Failed to open file " << control->path); + control->result_cb(control->path, control->uri, control->success); + return; } class download_client: public epee::net_utils::http::http_simple_client { public: - download_client(std::ofstream &f): f(f) {} + download_client(download_async_handle control, std::ofstream &f): + control(control), f(f), content_length(-1), total(0) {} virtual ~download_client() { f.close(); } + virtual bool on_header(const epee::net_utils::http::http_response_info &headers) + { + ssize_t length; + if (epee::string_tools::get_xtype_from_string(length, headers.m_header_info.m_content_length) && length >= 0) + { + MINFO("Content-Length: " << length); + content_length = length; + } + return true; + } virtual bool handle_target_data(std::string &piece_of_transfer) { try { + boost::lock_guard lock(control->mutex); + if (control->stop) + return false; f << piece_of_transfer; + total += piece_of_transfer.size(); + if (control->progress_cb && !control->progress_cb(control->path, control->uri, total, content_length)) + return false; return f.good(); } catch (const std::exception &e) @@ -70,69 +113,145 @@ namespace tools } } private: + download_async_handle control; std::ofstream &f; - } client(f); + ssize_t content_length; + size_t total; + } client(control, f); epee::net_utils::http::url_content u_c; - if (!epee::net_utils::parse_url(url, u_c)) + if (!epee::net_utils::parse_url(control->uri, u_c)) { - MWARNING("Failed to parse URL " << url); - return false; + MERROR("Failed to parse URL " << control->uri); + control->result_cb(control->path, control->uri, control->success); + return; } if (u_c.host.empty()) { - MWARNING("Failed to determine address from URL " << url); - return false; + MERROR("Failed to determine address from URL " << control->uri); + control->result_cb(control->path, control->uri, control->success); + return; } + + lock.unlock(); + uint16_t port = u_c.port ? u_c.port : 80; MDEBUG("Connecting to " << u_c.host << ":" << port); client.set_server(u_c.host, std::to_string(port), boost::none); if (!client.connect(std::chrono::seconds(30))) { - MERROR("Failed to connect to " << url); - return false; + boost::lock_guard lock(control->mutex); + MERROR("Failed to connect to " << control->uri); + control->result_cb(control->path, control->uri, control->success); + return; } MDEBUG("GETting " << u_c.uri); const epee::net_utils::http::http_response_info *info = NULL; if (!client.invoke_get(u_c.uri, std::chrono::seconds(30), "", &info)) { - MERROR("Failed to connect to " << url); + boost::lock_guard lock(control->mutex); + MERROR("Failed to connect to " << control->uri); client.disconnect(); - return false; + control->result_cb(control->path, control->uri, control->success); + return; + } + if (control->stop) + { + boost::lock_guard lock(control->mutex); + MDEBUG("Download cancelled"); + client.disconnect(); + control->result_cb(control->path, control->uri, control->success); + return; } if (!info) { - MERROR("Failed invoking GET command to " << url << ", no status info returned"); + boost::lock_guard lock(control->mutex); + MERROR("Failed invoking GET command to " << control->uri << ", no status info returned"); client.disconnect(); - return false; + control->result_cb(control->path, control->uri, control->success); + return; } MDEBUG("response code: " << info->m_response_code); + MDEBUG("response length: " << info->m_header_info.m_content_length); MDEBUG("response comment: " << info->m_response_comment); MDEBUG("response body: " << info->m_body); for (const auto &f: info->m_additional_fields) MDEBUG("additional field: " << f.first << ": " << f.second); if (info->m_response_code != 200) { + boost::lock_guard lock(control->mutex); MERROR("Status code " << info->m_response_code); client.disconnect(); - return false; + control->result_cb(control->path, control->uri, control->success); + return; } client.disconnect(); f.close(); MDEBUG("Download complete"); - return true; + lock.lock(); + control->success = true; + control->result_cb(control->path, control->uri, control->success); + return; } catch (const std::exception &e) { MERROR("Exception in download thread: " << e.what()); - return false; + // fall through and call result_cb not from the catch block to avoid another exception } + boost::lock_guard lock(control->mutex); + control->result_cb(control->path, control->uri, control->success); } - bool download(const std::string &path, const std::string &url) + bool download(const std::string &path, const std::string &url, std::function cb) { - bool success; - std::unique_ptr thread(new boost::thread([&](){ success = download_thread(path, url); })); - thread->join(); + bool success = false; + download_async_handle handle = download_async(path, url, [&success](const std::string&, const std::string&, bool result) {success = result;}, cb); + download_wait(handle); return success; } + + download_async_handle download_async(const std::string &path, const std::string &url, std::function result, std::function progress) + { + download_async_handle control = std::make_shared(path, url, result, progress); + control->thread = boost::thread([control](){ download_thread(control); }); + return control; + } + + bool download_finished(const download_async_handle &control) + { + CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle"); + boost::lock_guard lock(control->mutex); + return control->stopped; + } + + bool download_error(const download_async_handle &control) + { + CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle"); + boost::lock_guard lock(control->mutex); + return !control->success; + } + + bool download_wait(const download_async_handle &control) + { + CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle"); + { + boost::lock_guard lock(control->mutex); + if (control->stopped) + return true; + } + control->thread.join(); + return true; + } + + bool download_cancel(const download_async_handle &control) + { + CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle"); + { + boost::lock_guard lock(control->mutex); + if (control->stopped) + return true; + control->stop = true; + } + control->thread.join(); + return true; + } } diff --git a/src/common/download.h b/src/common/download.h index ab764468..917cb227 100644 --- a/src/common/download.h +++ b/src/common/download.h @@ -32,5 +32,13 @@ namespace tools { - bool download(const std::string &path, const std::string &url); + struct download_thread_control; + typedef std::shared_ptr download_async_handle; + + bool download(const std::string &path, const std::string &url, std::function progress = NULL); + download_async_handle download_async(const std::string &path, const std::string &url, std::function result, std::function progress = NULL); + bool download_error(const download_async_handle &h); + bool download_finished(const download_async_handle &h); + bool download_wait(const download_async_handle &h); + bool download_cancel(const download_async_handle &h); } From f6211322e57a70f381856f8604b8fb9b19ec1f69 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 26 Feb 2017 21:00:38 +0000 Subject: [PATCH 4/8] core: make update download cancellable --- src/cryptonote_core/cryptonote_core.cpp | 67 ++++++++++++++++++------- src/cryptonote_core/cryptonote_core.h | 5 ++ 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index cfe3b544..e0484d06 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -75,7 +75,8 @@ namespace cryptonote m_target_blockchain_height(0), m_checkpoints_path(""), m_last_dns_checkpoints_update(0), - m_last_json_checkpoints_update(0) + m_last_json_checkpoints_update(0), + m_update_download(0) { set_cryptonote_protocol(pprotocol); } @@ -134,6 +135,15 @@ namespace cryptonote void core::stop() { m_blockchain_storage.cancel(); + + tools::download_async_handle handle; + { + boost::lock_guard lock(m_update_mutex); + handle = m_update_download; + m_update_download = 0; + } + if (handle) + tools::download_cancel(handle); } //----------------------------------------------------------------------------------- void core::init_options(boost::program_options::options_description& desc) @@ -1133,32 +1143,55 @@ namespace cryptonote boost::filesystem::path path(epee::string_tools::get_current_module_folder()); path /= filename; + boost::unique_lock lock(m_update_mutex); + + if (m_update_download != 0) + { + MCDEBUG("updates", "Already downloading update"); + return true; + } + crypto::hash file_hash; if (!tools::sha256sum(path.string(), file_hash) || (hash != epee::string_tools::pod_to_hex(file_hash))) { MCDEBUG("updates", "We don't have that file already, downloading"); - if (!tools::download(path.string(), url)) - { - MCERROR("updates", "Failed to download " << url); - return false; - } - if (!tools::sha256sum(path.string(), file_hash)) - { - MCERROR("updates", "Failed to hash " << path); - return false; - } - if (hash != epee::string_tools::pod_to_hex(file_hash)) - { - MCERROR("updates", "Download from " << url << " does not match the expected hash"); - return false; - } - MGINFO("New version downloaded to " << path); + m_last_update_length = 0; + m_update_download = tools::download_async(path.string(), url, [this, hash](const std::string &path, const std::string &uri, bool success) { + if (success) + { + crypto::hash file_hash; + if (!tools::sha256sum(path, file_hash)) + { + MCERROR("updates", "Failed to hash " << path); + } + if (hash != epee::string_tools::pod_to_hex(file_hash)) + { + MCERROR("updates", "Download from " << uri << " does not match the expected hash"); + } + MGINFO("New version downloaded to " << path); + } + else + { + MCERROR("updates", "Failed to download " << uri); + } + boost::unique_lock lock(m_update_mutex); + m_update_download = 0; + }, [this](const std::string &path, const std::string &uri, size_t length, ssize_t content_length) { + if (length >= m_last_update_length + 1024 * 1024 * 10) + { + m_last_update_length = length; + MCDEBUG("updates", "Downloaded " << length << "/" << (content_length ? std::to_string(content_length) : "unknown")); + } + return true; + }); } else { MCDEBUG("updates", "We already have " << path << " with expected hash"); } + lock.unlock(); + if (check_updates_level == UPDATES_DOWNLOAD) return true; diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 7323bef2..02d58691 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -39,6 +39,7 @@ #include "p2p/net_node_common.h" #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" +#include "common/download.h" #include "tx_pool.h" #include "blockchain.h" #include "cryptonote_basic/miner.h" @@ -844,6 +845,10 @@ namespace cryptonote UPDATES_DOWNLOAD, UPDATES_UPDATE, } check_updates_level; + + tools::download_async_handle m_update_download; + size_t m_last_update_length; + boost::mutex m_update_mutex; }; } From f36c5f1e084e2591da5bdd92d4dd4713f4312415 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 28 Feb 2017 22:14:19 +0000 Subject: [PATCH 5/8] download: give download threads distinct names --- src/common/download.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/common/download.cpp b/src/common/download.cpp index 9c2330b1..b2f95b21 100644 --- a/src/common/download.cpp +++ b/src/common/download.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include +#include #include #include #include @@ -59,6 +60,10 @@ namespace tools static void download_thread(download_async_handle control) { + static std::atomic thread_id(0); + + MLOG_SET_THREAD_NAME("DL" + std::to_string(thread_id++)); + struct stopped_setter { stopped_setter(const download_async_handle &control): control(control) {} From 749ebacebd98ad17c9d7c636f9d90534a691a2bc Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 28 Feb 2017 22:15:54 +0000 Subject: [PATCH 6/8] download: check available disk space before downloading We don't check *while* the download happens, so it might still be that we don't have enough space later --- src/common/download.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/common/download.cpp b/src/common/download.cpp index b2f95b21..28aac5a5 100644 --- a/src/common/download.cpp +++ b/src/common/download.cpp @@ -95,6 +95,14 @@ namespace tools { MINFO("Content-Length: " << length); content_length = length; + boost::filesystem::path path(control->path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if (si.available < (size_t)content_length) + { + const uint64_t avail = (si.available + 1023) / 1024, needed = (content_length + 1023) / 1024; + MERROR("Not enough space to download " << needed << " kB to " << path << " (" << avail << " kB available)"); + return false; + } } return true; } From 02097c87ebca12c34329acab89899a633db1d8ba Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 4 Mar 2017 18:29:18 +0000 Subject: [PATCH 7/8] core: print the "new update found" message in cyan, for visibility --- src/cryptonote_core/cryptonote_core.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index e0484d06..9c5fae4e 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1128,7 +1128,7 @@ namespace cryptonote return true; std::string url = tools::get_update_url(software, subdir, buildtag, version, true); - MGINFO("Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash); + MCLOG_CYAN(el::Level::Info, "global", "Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash); if (check_updates_level == UPDATES_NOTIFY) return true; @@ -1168,7 +1168,7 @@ namespace cryptonote { MCERROR("updates", "Download from " << uri << " does not match the expected hash"); } - MGINFO("New version downloaded to " << path); + MCLOG_CYAN(el::Level::Info, "updates", "New version downloaded to " << path); } else { From b553c282fbe733ac35802845c0dfcfb316e2818d Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 4 Mar 2017 18:45:33 +0000 Subject: [PATCH 8/8] rpc: fix BUILD_TAG mispelling (BUILDTAG) This ensures a manual or RPC update tries the right build tag, rather than source, which is currently not setup --- src/rpc/core_rpc_server.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index b114f77a..9f7e8aa3 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1491,8 +1491,8 @@ namespace cryptonote bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res) { static const char software[] = "monero"; -#ifdef BUILDTAG - static const char buildtag[] = BOOST_PP_STRINGIZE(BUILDTAG); +#ifdef BUILD_TAG + static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG); #else static const char buildtag[] = "source"; #endif