Merge pull request #1841

b553c282 rpc: fix BUILD_TAG mispelling (BUILDTAG) (moneromooo-monero)
02097c87 core: print the "new update found" message in cyan, for visibility (moneromooo-monero)
749ebace download: check available disk space before downloading (moneromooo-monero)
f36c5f1e download: give download threads distinct names (moneromooo-monero)
f6211322 core: make update download cancellable (moneromooo-monero)
63f0e074 download: async API (moneromooo-monero)
9bf017ed http_client: allow cancelling a download (moneromooo-monero)
0d90123c http_client: allow derived class to get headers at start (moneromooo-monero)
This commit is contained in:
Riccardo Spagni 2017-03-06 15:17:08 +00:00
commit ba0767477d
No known key found for this signature in database
GPG key ID: 55432DF31CCD4FCD
6 changed files with 245 additions and 48 deletions

View file

@ -335,6 +335,11 @@ using namespace std;
m_response_info.m_body += piece_of_transfer; m_response_info.m_body += piece_of_transfer;
piece_of_transfer.clear(); piece_of_transfer.clear();
return true; return true;
}
//---------------------------------------------------------------------------
virtual bool on_header(const http_response_info &headers)
{
return true;
} }
//--------------------------------------------------------------------------- //---------------------------------------------------------------------------
inline inline
@ -505,6 +510,12 @@ using namespace std;
m_header_cache.erase(m_header_cache.begin()+pos+4, m_header_cache.end()); m_header_cache.erase(m_header_cache.begin()+pos+4, m_header_cache.end());
analize_cached_header_and_invoke_state(); analize_cached_header_and_invoke_state();
if (!on_header(m_response_info))
{
MDEBUG("Connection cancelled by on_header");
m_state = reciev_machine_state_done;
return false;
}
m_header_cache.clear(); m_header_cache.clear();
if(!recv_buff.size() && (m_state != reciev_machine_state_error && m_state != reciev_machine_state_done)) if(!recv_buff.size() && (m_state != reciev_machine_state_error && m_state != reciev_machine_state_done))
need_more_data = true; need_more_data = true;
@ -527,7 +538,11 @@ using namespace std;
} }
CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_len_in_remain >= recv_buff.size()"); CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_len_in_remain >= recv_buff.size()");
m_len_in_remain -= recv_buff.size(); m_len_in_remain -= recv_buff.size();
m_pcontent_encoding_handler->update_in(recv_buff); if (!m_pcontent_encoding_handler->update_in(recv_buff))
{
m_state = reciev_machine_state_done;
return false;
}
if(m_len_in_remain == 0) if(m_len_in_remain == 0)
m_state = reciev_machine_state_done; m_state = reciev_machine_state_done;
@ -700,7 +715,11 @@ using namespace std;
m_len_in_remain = 0; m_len_in_remain = 0;
} }
m_pcontent_encoding_handler->update_in(chunk_body); if (!m_pcontent_encoding_handler->update_in(chunk_body))
{
m_state = reciev_machine_state_error;
return false;
}
if(!m_len_in_remain) if(!m_len_in_remain)
m_chunked_state = http_chunked_state_chunk_head; m_chunked_state = http_chunked_state_chunk_head;

View file

@ -27,6 +27,7 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string> #include <string>
#include <atomic>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#include <boost/asio.hpp> #include <boost/asio.hpp>
#include <boost/thread/thread.hpp> #include <boost/thread/thread.hpp>
@ -40,27 +41,82 @@
namespace tools namespace tools
{ {
static bool download_thread(const std::string &path, const std::string &url) struct download_thread_control
{ {
const std::string path;
const std::string uri;
std::function<void(const std::string&, const std::string&, bool)> result_cb;
std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress_cb;
bool stop;
bool stopped;
bool success;
boost::thread thread;
boost::mutex mutex;
download_thread_control(const std::string &path, const std::string &uri, std::function<void(const std::string&, const std::string&, bool)> result_cb, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress_cb):
path(path), uri(uri), result_cb(result_cb), progress_cb(progress_cb), stop(false), stopped(false), success(false) {}
~download_thread_control() { if (thread.joinable()) thread.detach(); }
};
static void download_thread(download_async_handle control)
{
static std::atomic<unsigned int> thread_id(0);
MLOG_SET_THREAD_NAME("DL" + std::to_string(thread_id++));
struct stopped_setter
{
stopped_setter(const download_async_handle &control): control(control) {}
~stopped_setter() { control->stopped = true; }
download_async_handle control;
} stopped_setter(control);
try try
{ {
MINFO("Downloading " << url << " to " << path); boost::unique_lock<boost::mutex> lock(control->mutex);
MINFO("Downloading " << control->uri << " to " << control->path);
std::ofstream f; std::ofstream f;
f.open(path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); f.open(control->path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc);
if (!f.good()) { if (!f.good()) {
MERROR("Failed to open file " << path); MERROR("Failed to open file " << control->path);
return false; control->result_cb(control->path, control->uri, control->success);
return;
} }
class download_client: public epee::net_utils::http::http_simple_client class download_client: public epee::net_utils::http::http_simple_client
{ {
public: public:
download_client(std::ofstream &f): f(f) {} download_client(download_async_handle control, std::ofstream &f):
control(control), f(f), content_length(-1), total(0) {}
virtual ~download_client() { f.close(); } virtual ~download_client() { f.close(); }
virtual bool on_header(const epee::net_utils::http::http_response_info &headers)
{
ssize_t length;
if (epee::string_tools::get_xtype_from_string(length, headers.m_header_info.m_content_length) && length >= 0)
{
MINFO("Content-Length: " << length);
content_length = length;
boost::filesystem::path path(control->path);
boost::filesystem::space_info si = boost::filesystem::space(path);
if (si.available < (size_t)content_length)
{
const uint64_t avail = (si.available + 1023) / 1024, needed = (content_length + 1023) / 1024;
MERROR("Not enough space to download " << needed << " kB to " << path << " (" << avail << " kB available)");
return false;
}
}
return true;
}
virtual bool handle_target_data(std::string &piece_of_transfer) virtual bool handle_target_data(std::string &piece_of_transfer)
{ {
try try
{ {
boost::lock_guard<boost::mutex> lock(control->mutex);
if (control->stop)
return false;
f << piece_of_transfer; f << piece_of_transfer;
total += piece_of_transfer.size();
if (control->progress_cb && !control->progress_cb(control->path, control->uri, total, content_length))
return false;
return f.good(); return f.good();
} }
catch (const std::exception &e) catch (const std::exception &e)
@ -70,69 +126,145 @@ namespace tools
} }
} }
private: private:
download_async_handle control;
std::ofstream &f; std::ofstream &f;
} client(f); ssize_t content_length;
size_t total;
} client(control, f);
epee::net_utils::http::url_content u_c; epee::net_utils::http::url_content u_c;
if (!epee::net_utils::parse_url(url, u_c)) if (!epee::net_utils::parse_url(control->uri, u_c))
{ {
MWARNING("Failed to parse URL " << url); MERROR("Failed to parse URL " << control->uri);
return false; control->result_cb(control->path, control->uri, control->success);
return;
} }
if (u_c.host.empty()) if (u_c.host.empty())
{ {
MWARNING("Failed to determine address from URL " << url); MERROR("Failed to determine address from URL " << control->uri);
return false; control->result_cb(control->path, control->uri, control->success);
return;
} }
lock.unlock();
uint16_t port = u_c.port ? u_c.port : 80; uint16_t port = u_c.port ? u_c.port : 80;
MDEBUG("Connecting to " << u_c.host << ":" << port); MDEBUG("Connecting to " << u_c.host << ":" << port);
client.set_server(u_c.host, std::to_string(port), boost::none); client.set_server(u_c.host, std::to_string(port), boost::none);
if (!client.connect(std::chrono::seconds(30))) if (!client.connect(std::chrono::seconds(30)))
{ {
MERROR("Failed to connect to " << url); boost::lock_guard<boost::mutex> lock(control->mutex);
return false; MERROR("Failed to connect to " << control->uri);
control->result_cb(control->path, control->uri, control->success);
return;
} }
MDEBUG("GETting " << u_c.uri); MDEBUG("GETting " << u_c.uri);
const epee::net_utils::http::http_response_info *info = NULL; const epee::net_utils::http::http_response_info *info = NULL;
if (!client.invoke_get(u_c.uri, std::chrono::seconds(30), "", &info)) if (!client.invoke_get(u_c.uri, std::chrono::seconds(30), "", &info))
{ {
MERROR("Failed to connect to " << url); boost::lock_guard<boost::mutex> lock(control->mutex);
MERROR("Failed to connect to " << control->uri);
client.disconnect(); client.disconnect();
return false; control->result_cb(control->path, control->uri, control->success);
return;
}
if (control->stop)
{
boost::lock_guard<boost::mutex> lock(control->mutex);
MDEBUG("Download cancelled");
client.disconnect();
control->result_cb(control->path, control->uri, control->success);
return;
} }
if (!info) if (!info)
{ {
MERROR("Failed invoking GET command to " << url << ", no status info returned"); boost::lock_guard<boost::mutex> lock(control->mutex);
MERROR("Failed invoking GET command to " << control->uri << ", no status info returned");
client.disconnect(); client.disconnect();
return false; control->result_cb(control->path, control->uri, control->success);
return;
} }
MDEBUG("response code: " << info->m_response_code); MDEBUG("response code: " << info->m_response_code);
MDEBUG("response length: " << info->m_header_info.m_content_length);
MDEBUG("response comment: " << info->m_response_comment); MDEBUG("response comment: " << info->m_response_comment);
MDEBUG("response body: " << info->m_body); MDEBUG("response body: " << info->m_body);
for (const auto &f: info->m_additional_fields) for (const auto &f: info->m_additional_fields)
MDEBUG("additional field: " << f.first << ": " << f.second); MDEBUG("additional field: " << f.first << ": " << f.second);
if (info->m_response_code != 200) if (info->m_response_code != 200)
{ {
boost::lock_guard<boost::mutex> lock(control->mutex);
MERROR("Status code " << info->m_response_code); MERROR("Status code " << info->m_response_code);
client.disconnect(); client.disconnect();
return false; control->result_cb(control->path, control->uri, control->success);
return;
} }
client.disconnect(); client.disconnect();
f.close(); f.close();
MDEBUG("Download complete"); MDEBUG("Download complete");
return true; lock.lock();
control->success = true;
control->result_cb(control->path, control->uri, control->success);
return;
} }
catch (const std::exception &e) catch (const std::exception &e)
{ {
MERROR("Exception in download thread: " << e.what()); MERROR("Exception in download thread: " << e.what());
return false; // fall through and call result_cb not from the catch block to avoid another exception
} }
boost::lock_guard<boost::mutex> lock(control->mutex);
control->result_cb(control->path, control->uri, control->success);
} }
bool download(const std::string &path, const std::string &url) bool download(const std::string &path, const std::string &url, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> cb)
{ {
bool success; bool success = false;
std::unique_ptr<boost::thread> thread(new boost::thread([&](){ success = download_thread(path, url); })); download_async_handle handle = download_async(path, url, [&success](const std::string&, const std::string&, bool result) {success = result;}, cb);
thread->join(); download_wait(handle);
return success; return success;
} }
download_async_handle download_async(const std::string &path, const std::string &url, std::function<void(const std::string&, const std::string&, bool)> result, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress)
{
download_async_handle control = std::make_shared<download_thread_control>(path, url, result, progress);
control->thread = boost::thread([control](){ download_thread(control); });
return control;
}
bool download_finished(const download_async_handle &control)
{
CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
boost::lock_guard<boost::mutex> lock(control->mutex);
return control->stopped;
}
bool download_error(const download_async_handle &control)
{
CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
boost::lock_guard<boost::mutex> lock(control->mutex);
return !control->success;
}
bool download_wait(const download_async_handle &control)
{
CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
{
boost::lock_guard<boost::mutex> lock(control->mutex);
if (control->stopped)
return true;
}
control->thread.join();
return true;
}
bool download_cancel(const download_async_handle &control)
{
CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
{
boost::lock_guard<boost::mutex> lock(control->mutex);
if (control->stopped)
return true;
control->stop = true;
}
control->thread.join();
return true;
}
} }

View file

@ -32,5 +32,13 @@
namespace tools namespace tools
{ {
bool download(const std::string &path, const std::string &url); struct download_thread_control;
typedef std::shared_ptr<download_thread_control> download_async_handle;
bool download(const std::string &path, const std::string &url, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress = NULL);
download_async_handle download_async(const std::string &path, const std::string &url, std::function<void(const std::string&, const std::string&, bool)> result, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress = NULL);
bool download_error(const download_async_handle &h);
bool download_finished(const download_async_handle &h);
bool download_wait(const download_async_handle &h);
bool download_cancel(const download_async_handle &h);
} }

View file

@ -75,7 +75,8 @@ namespace cryptonote
m_target_blockchain_height(0), m_target_blockchain_height(0),
m_checkpoints_path(""), m_checkpoints_path(""),
m_last_dns_checkpoints_update(0), m_last_dns_checkpoints_update(0),
m_last_json_checkpoints_update(0) m_last_json_checkpoints_update(0),
m_update_download(0)
{ {
set_cryptonote_protocol(pprotocol); set_cryptonote_protocol(pprotocol);
} }
@ -134,6 +135,15 @@ namespace cryptonote
void core::stop() void core::stop()
{ {
m_blockchain_storage.cancel(); m_blockchain_storage.cancel();
tools::download_async_handle handle;
{
boost::lock_guard<boost::mutex> lock(m_update_mutex);
handle = m_update_download;
m_update_download = 0;
}
if (handle)
tools::download_cancel(handle);
} }
//----------------------------------------------------------------------------------- //-----------------------------------------------------------------------------------
void core::init_options(boost::program_options::options_description& desc) void core::init_options(boost::program_options::options_description& desc)
@ -1118,7 +1128,7 @@ namespace cryptonote
return true; return true;
std::string url = tools::get_update_url(software, subdir, buildtag, version, true); std::string url = tools::get_update_url(software, subdir, buildtag, version, true);
MGINFO("Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash); MCLOG_CYAN(el::Level::Info, "global", "Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash);
if (check_updates_level == UPDATES_NOTIFY) if (check_updates_level == UPDATES_NOTIFY)
return true; return true;
@ -1133,32 +1143,55 @@ namespace cryptonote
boost::filesystem::path path(epee::string_tools::get_current_module_folder()); boost::filesystem::path path(epee::string_tools::get_current_module_folder());
path /= filename; path /= filename;
boost::unique_lock<boost::mutex> lock(m_update_mutex);
if (m_update_download != 0)
{
MCDEBUG("updates", "Already downloading update");
return true;
}
crypto::hash file_hash; crypto::hash file_hash;
if (!tools::sha256sum(path.string(), file_hash) || (hash != epee::string_tools::pod_to_hex(file_hash))) if (!tools::sha256sum(path.string(), file_hash) || (hash != epee::string_tools::pod_to_hex(file_hash)))
{ {
MCDEBUG("updates", "We don't have that file already, downloading"); MCDEBUG("updates", "We don't have that file already, downloading");
if (!tools::download(path.string(), url)) m_last_update_length = 0;
m_update_download = tools::download_async(path.string(), url, [this, hash](const std::string &path, const std::string &uri, bool success) {
if (success)
{ {
MCERROR("updates", "Failed to download " << url); crypto::hash file_hash;
return false; if (!tools::sha256sum(path, file_hash))
}
if (!tools::sha256sum(path.string(), file_hash))
{ {
MCERROR("updates", "Failed to hash " << path); MCERROR("updates", "Failed to hash " << path);
return false;
} }
if (hash != epee::string_tools::pod_to_hex(file_hash)) if (hash != epee::string_tools::pod_to_hex(file_hash))
{ {
MCERROR("updates", "Download from " << url << " does not match the expected hash"); MCERROR("updates", "Download from " << uri << " does not match the expected hash");
return false;
} }
MGINFO("New version downloaded to " << path); MCLOG_CYAN(el::Level::Info, "updates", "New version downloaded to " << path);
}
else
{
MCERROR("updates", "Failed to download " << uri);
}
boost::unique_lock<boost::mutex> lock(m_update_mutex);
m_update_download = 0;
}, [this](const std::string &path, const std::string &uri, size_t length, ssize_t content_length) {
if (length >= m_last_update_length + 1024 * 1024 * 10)
{
m_last_update_length = length;
MCDEBUG("updates", "Downloaded " << length << "/" << (content_length ? std::to_string(content_length) : "unknown"));
}
return true;
});
} }
else else
{ {
MCDEBUG("updates", "We already have " << path << " with expected hash"); MCDEBUG("updates", "We already have " << path << " with expected hash");
} }
lock.unlock();
if (check_updates_level == UPDATES_DOWNLOAD) if (check_updates_level == UPDATES_DOWNLOAD)
return true; return true;

View file

@ -39,6 +39,7 @@
#include "p2p/net_node_common.h" #include "p2p/net_node_common.h"
#include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "cryptonote_protocol/cryptonote_protocol_handler_common.h"
#include "storages/portable_storage_template_helper.h" #include "storages/portable_storage_template_helper.h"
#include "common/download.h"
#include "tx_pool.h" #include "tx_pool.h"
#include "blockchain.h" #include "blockchain.h"
#include "cryptonote_basic/miner.h" #include "cryptonote_basic/miner.h"
@ -844,6 +845,10 @@ namespace cryptonote
UPDATES_DOWNLOAD, UPDATES_DOWNLOAD,
UPDATES_UPDATE, UPDATES_UPDATE,
} check_updates_level; } check_updates_level;
tools::download_async_handle m_update_download;
size_t m_last_update_length;
boost::mutex m_update_mutex;
}; };
} }

View file

@ -1491,8 +1491,8 @@ namespace cryptonote
bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res) bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res)
{ {
static const char software[] = "monero"; static const char software[] = "monero";
#ifdef BUILDTAG #ifdef BUILD_TAG
static const char buildtag[] = BOOST_PP_STRINGIZE(BUILDTAG); static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG);
#else #else
static const char buildtag[] = "source"; static const char buildtag[] = "source";
#endif #endif