From eabb519605cab00dbaa5a1868d229f09c74570a6 Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Mon, 5 Jan 2015 20:30:17 +0100 Subject: [PATCH 01/16] 2014 network limit 1.0a +utils +toc -doc -drmonero commands and options for network limiting works very well e.g. for 50 KiB/sec up and down ToS (QoS) flag peer number limit TODO some spikes in ingress/download TODO problems when other up and down limit added "otshell utils" - simple logging (with colors, text files channels) --- .gitignore | 82 +++ CMakeLists.txt | 22 +- contrib/CMakeLists.txt | 3 + .../epee/include/net/abstract_tcp_server2.h | 77 ++- .../epee/include/net/abstract_tcp_server2.inl | 373 ++++++++--- .../net/levin_protocol_handler_async.h | 9 + contrib/otshell_utils/CMakeLists.txt | 14 + contrib/otshell_utils/LICENCE.txt | 21 + contrib/otshell_utils/ccolor.cpp | 116 ++++ contrib/otshell_utils/ccolor.hpp | 73 +++ contrib/otshell_utils/lib_common1.hpp | 51 ++ contrib/otshell_utils/runoptions.cpp | 69 ++ contrib/otshell_utils/runoptions.hpp | 58 ++ contrib/otshell_utils/utils.cpp | 612 ++++++++++++++++++ contrib/otshell_utils/utils.hpp | 446 +++++++++++++ src/CMakeLists.txt | 2 + src/cryptonote_core/CMakeLists.txt | 1 + src/cryptonote_core/blockchain_storage.cpp | 26 + src/cryptonote_core/blockchain_storage.h | 1 + src/cryptonote_protocol/CMakeLists.txt | 46 ++ .../cryptonote_protocol_handler-base.cpp | 266 ++++++++ .../cryptonote_protocol_handler.h | 35 +- .../cryptonote_protocol_handler.inl | 73 ++- src/daemon/CMakeLists.txt | 21 +- src/daemon/daemon_commands_handler.h | 129 ++++ src/p2p/CMakeLists.txt | 46 ++ src/p2p/connection_basic.cpp | 362 +++++++++++ src/p2p/connection_basic.hpp | 139 ++++ src/p2p/net_node.h | 31 +- src/p2p/net_node.inl | 183 ++++-- src/p2p/network_throttle-detail.cpp | 382 +++++++++++ src/p2p/network_throttle-detail.hpp | 133 ++++ src/p2p/network_throttle.cpp | 121 ++++ src/p2p/network_throttle.hpp | 187 ++++++ src/simplewallet/CMakeLists.txt | 1 + tests/core_proxy/CMakeLists.txt | 2 + tests/core_proxy/core_proxy.h | 1 + tests/net_load_tests/CMakeLists.txt | 4 + tests/net_load_tests/clt.cpp | 3 + tests/unit_tests/CMakeLists.txt | 1 + 40 files changed, 4016 insertions(+), 206 deletions(-) create mode 100644 contrib/CMakeLists.txt create mode 100644 contrib/otshell_utils/CMakeLists.txt create mode 100644 contrib/otshell_utils/LICENCE.txt create mode 100644 contrib/otshell_utils/ccolor.cpp create mode 100644 contrib/otshell_utils/ccolor.hpp create mode 100644 contrib/otshell_utils/lib_common1.hpp create mode 100644 contrib/otshell_utils/runoptions.cpp create mode 100644 contrib/otshell_utils/runoptions.hpp create mode 100644 contrib/otshell_utils/utils.cpp create mode 100644 contrib/otshell_utils/utils.hpp create mode 100644 src/cryptonote_protocol/CMakeLists.txt create mode 100644 src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp create mode 100644 src/p2p/CMakeLists.txt create mode 100644 src/p2p/connection_basic.cpp create mode 100644 src/p2p/connection_basic.hpp create mode 100644 src/p2p/network_throttle-detail.cpp create mode 100644 src/p2p/network_throttle-detail.hpp create mode 100644 src/p2p/network_throttle.cpp create mode 100644 src/p2p/network_throttle.hpp diff --git a/.gitignore b/.gitignore index 4f8766a4..3aed0e11 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /doc /build /tags +log/ # vim swap files *.swp @@ -20,4 +21,85 @@ cscope.in.out cscope.po.out +miniupnpcstrings.h +version/ +# Created by https://www.gitignore.io + +### C++ ### +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + + +### CMake ### +CMakeCache.txt +CMakeFiles +Makefile +cmake_install.cmake +install_manifest.txt +*.cmake + +### Linux ### +*~ + +# KDE directory preferences +.directory + + +### Eclipse ### +*.pydevproject +.metadata +.gradle +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# CDT-specific +.cproject + +# PDT-specific +.buildpath + +# sbteclipse plugin +.target + +# TeXlipse plugin +.texlipse diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c1209e0..bdb96746 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,17 +50,24 @@ list(INSERT CMAKE_MODULE_PATH 0 if (NOT DEFINED ENV{DEVELOPER_LOCAL_TOOLS}) message(STATUS "Could not find DEVELOPER_LOCAL_TOOLS in env (not required)") - set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT OFF) -elseif (ENV{DEVELOPER_LOCAL_TOOLS} EQUAL 1) + set(BOOST_IGNORE_SYSTEM_PATHS OFF) +elseif ("$ENV{DEVELOPER_LOCAL_TOOLS}" STREQUAL "1") message(STATUS "Found: env DEVELOPER_LOCAL_TOOLS = 1") - set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT ON) + set(BOOST_IGNORE_SYSTEM_PATHS ON) else() message(STATUS "Found: env DEVELOPER_LOCAL_TOOLS = 0") - set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT OFF) + set(BOOST_IGNORE_SYSTEM_PATHS OFF) endif() -message(STATUS "BOOST_IGNORE_SYSTEM_PATHS defaults to ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}") -option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost installation" ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}) +#message(STATUS "BOOST_IGNORE_SYSTEM_PATHS defaults to ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}") +#option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost installation" ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}) +message(STATUS "BOOST_IGNORE_SYSTEM_PATHS: ${BOOST_IGNORE_SYSTEM_PATHS}") + +# Options (for external/otshell_utils/) +option(WITH_TERMCOLORS "Build with support for unix terminal console colors VT100" ON) +if (WITH_TERMCOLORS) + add_definitions( -DCFG_WITH_TERMCOLORS ) +endif () set_property(GLOBAL PROPERTY USE_FOLDERS ON) enable_testing() @@ -234,7 +241,7 @@ else() endif() endif() -if (BOOST_IGNORE_SYSTEM_PATHS) +if (${BOOST_IGNORE_SYSTEM_PATHS} STREQUAL "ON") set(Boost_NO_SYSTEM_PATHS TRUE) endif() @@ -264,6 +271,7 @@ endif() include(version.cmake) +add_subdirectory(contrib) add_subdirectory(src) if(BUILD_TESTS) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt new file mode 100644 index 00000000..18402a61 --- /dev/null +++ b/contrib/CMakeLists.txt @@ -0,0 +1,3 @@ + +add_subdirectory(otshell_utils) + diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 6c613c5d..1e622321 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -1,3 +1,9 @@ +/** +@file +@author from CrypoNote (see copyright below; Andrey N. Sabelnikov) +@monero rfree +@brief the connection templated-class for one peer connection +*/ // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -26,7 +32,7 @@ -#ifndef _ABSTRACT_TCP_SERVER2_H_ +#ifndef _ABSTRACT_TCP_SERVER2_H_ #define _ABSTRACT_TCP_SERVER2_H_ @@ -36,6 +42,8 @@ #include #include #include +#include +#include #include #include @@ -46,7 +54,8 @@ #include #include "net_utils_base.h" #include "syncobj.h" - +#include "../../../../src/p2p/connection_basic.hpp" +#include "../../../../contrib/otshell_utils/utils.hpp" #define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000 @@ -61,6 +70,12 @@ namespace net_utils protected: virtual ~i_connection_filter(){} }; + + enum t_server_role { // type of the server, e.g. so that we will know how to limit it + NET = 0, // default (not used? used for misc connections maybe?) TODO + RPC = 1, // the rpc commands + P2P = 2 // to other p2p node + }; /************************************************************************/ /* */ @@ -70,13 +85,18 @@ namespace net_utils class connection : public boost::enable_shared_from_this >, private boost::noncopyable, - public i_service_endpoint + public i_service_endpoint, + public connection_basic { public: typedef typename t_protocol_handler::connection_context t_connection_context; /// Construct a connection with the given io_service. - explicit connection(boost::asio::io_service& io_service, - typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter * &pfilter); + + explicit connection( boost::asio::io_service& io_service, + typename t_protocol_handler::config_type& config, + std::atomic &ref_sock_count, // the ++/-- counter + std::atomic &sock_number, // the only increasing ++ number generator + i_connection_filter * &pfilter); virtual ~connection(); /// Get the socket associated with the connection. @@ -90,7 +110,8 @@ namespace net_utils void call_back_starter(); private: //----------------- i_service_endpoint --------------------- - virtual bool do_send(const void* ptr, size_t cb); + virtual bool do_send(const void* ptr, size_t cb); ///< (see do_send from i_service_endpoint) + virtual bool do_send_chunk(const void* ptr, size_t cb); ///< will send (or queue) a part of data virtual bool close(); virtual bool call_run_once_service_io(); virtual bool request_callback(); @@ -107,29 +128,24 @@ namespace net_utils /// Handle completion of a write operation. void handle_write(const boost::system::error_code& e, size_t cb); - /// Strand to ensure the connection's handlers are not called concurrently. - boost::asio::io_service::strand strand_; - - /// Socket for the connection. - boost::asio::ip::tcp::socket socket_; - /// Buffer for incoming data. boost::array buffer_; t_connection_context context; - volatile uint32_t m_want_close_connection; - std::atomic m_was_shutdown; - critical_section m_send_que_lock; - std::list m_send_que; - volatile uint32_t& m_ref_sockets_count; i_connection_filter* &m_pfilter; - volatile bool m_is_multithreaded; + // TODO what do they mean about wait on destructor?? --rfree : //this should be the last one, because it could be wait on destructor, while other activities possible on other threads t_protocol_handler m_protocol_handler; //typename t_protocol_handler::config_type m_dummy_config; std::list > > m_self_refs; // add_ref/release support critical_section m_self_refs_lock; + critical_section m_chunking_lock; // held while we add small chunks of the big do_send() to small do_send_chunk() + + t_server_role m_connection_type; + + public: + void setRPcStation(); }; @@ -146,9 +162,11 @@ namespace net_utils /// Construct the server to listen on the specified TCP address and port, and /// serve up files from the given directory. boosted_tcp_server(); - explicit boosted_tcp_server(boost::asio::io_service& external_io_service); + explicit boosted_tcp_server(boost::asio::io_service& external_io_service, t_server_role s_type); ~boosted_tcp_server(); - + + std::map server_type_map; + void create_server_type_map(); bool init_server(uint32_t port, const std::string address = "0.0.0.0"); bool init_server(const std::string port, const std::string& address = "0.0.0.0"); @@ -254,22 +272,25 @@ namespace net_utils /// Acceptor used to listen for incoming connections. boost::asio::ip::tcp::acceptor acceptor_; - /// The next connection to be accepted. - connection_ptr new_connection_; std::atomic m_stop_signal_sent; uint32_t m_port; - volatile uint32_t m_sockets_count; + std::atomic m_sock_count; + std::atomic m_sock_number; std::string m_address; - std::string m_thread_name_prefix; + std::string m_thread_name_prefix; //TODO: change to enum server_type, now used size_t m_threads_count; i_connection_filter* m_pfilter; std::vector > m_threads; boost::thread::id m_main_thread_id; critical_section m_threads_lock; - volatile uint32_t m_thread_index; - }; -} -} + volatile uint32_t m_thread_index; // TODO change to std::atomic + t_server_role type; + + /// The next connection to be accepted + connection_ptr new_connection_; + }; // class <>boosted_tcp_server +} // namespace +} // namespace #include "abstract_tcp_server2.inl" diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index db3f9e32..8dff192b 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -1,3 +1,9 @@ +/** +@file +@author from CrypoNote (see copyright below; Andrey N. Sabelnikov) +@monero rfree +@brief the connection templated-class for one peer connection +*/ // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -26,7 +32,7 @@ -#include "net_utils_base.h" +//#include "net_utils_base.h" #include #include #include @@ -34,9 +40,20 @@ #include #include #include +#include // TODO +#include // TODO #include "misc_language.h" #include "pragma_comp_defs.h" +#include +#include +#include + +#include "../../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal() + +#include "../../../../contrib/otshell_utils/utils.hpp" +using namespace nOT::nUtils; // TODO + PRAGMA_WARNING_PUSH namespace epee { @@ -48,17 +65,19 @@ namespace net_utils PRAGMA_WARNING_DISABLE_VS(4355) template - connection::connection(boost::asio::io_service& io_service, - typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter* &pfilter) - : strand_(io_service), - socket_(io_service), - m_want_close_connection(0), - m_was_shutdown(0), - m_ref_sockets_count(sock_count), - m_pfilter(pfilter), - m_protocol_handler(this, config, context) + connection::connection( boost::asio::io_service& io_service, + typename t_protocol_handler::config_type& config, + std::atomic &ref_sock_count, // the ++/-- counter + std::atomic &sock_number, // the only increasing ++ number generator + i_connection_filter* &pfilter + ) + : + connection_basic(io_service, ref_sock_count, sock_number), + m_protocol_handler(this, config, context), + m_pfilter( pfilter ), + m_connection_type(NET) { - boost::interprocess::ipcdetail::atomic_inc32(&m_ref_sockets_count); + _info_c("net/sleepRPC", "connection constructor set m_connection_type="< @@ -118,13 +136,13 @@ PRAGMA_WARNING_DISABLE_VS(4355) long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong()); context.set_details(boost::uuids::random_generator()(), ip_, remote_ep.port(), is_income); - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) << + _dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) << " to " << local_ep.address().to_string() << ':' << local_ep.port() << - ", total sockets objects " << m_ref_sockets_count); + ", total sockets objects " << m_ref_sock_count); if(m_pfilter && !m_pfilter->is_remote_ip_allowed(context.m_remote_ip)) { - LOG_PRINT_L2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection"); + _dbg2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection"); close(); return false; } @@ -136,7 +154,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::bind(&connection::handle_read, self, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred))); - + + //set ToS flag + int tos = get_tos_flag(); + boost::asio::detail::socket_option::integer< IPPROTO_IP, IP_TOS > + optionTos( tos ); + socket_.set_option( optionTos ); + //_dbg1("Set ToS flag to " << tos); + return true; CATCH_ENTRY_L0("connection::start()", false); @@ -146,7 +171,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) bool connection::request_callback() { TRY_ENTRY(); - LOG_PRINT_L2("[" << print_connection_context_short(context) << "] request_callback"); + _dbg2("[" << print_connection_context_short(context) << "] request_callback"); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted auto self = safe_shared_from_this(); if(!self) @@ -167,7 +192,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) bool connection::add_ref() { TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] add_ref"); + //_info("[sock " << socket_.native_handle() << "] add_ref"); CRITICAL_REGION_LOCAL(m_self_refs_lock); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted @@ -201,7 +226,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) void connection::call_back_starter() { TRY_ENTRY(); - LOG_PRINT_L2("[" << print_connection_context_short(context) << "] fired_callback"); + _dbg2("[" << print_connection_context_short(context) << "] fired_callback"); m_protocol_handler.handle_qued_callback(); CATCH_ENTRY_L0("connection::call_back_starter()", void()); } @@ -211,17 +236,18 @@ PRAGMA_WARNING_DISABLE_VS(4355) std::size_t bytes_transferred) { TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async read calledback."); + //_info("[sock " << socket_.native_handle() << "] Async read calledback."); if (!e) { - LOG_PRINT("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred, LOG_LEVEL_4); + //_info("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred); + logger_handle_net_read(bytes_transferred); context.m_last_recv = time(NULL); context.m_recv_cnt += bytes_transferred; bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred); if(!recv_res) { - LOG_PRINT("[sock " << socket_.native_handle() << "] protocol_want_close", LOG_LEVEL_4); + //_info("[sock " << socket_.native_handle() << "] protocol_want_close"); //some error in protocol, protocol handler ask to close connection boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); @@ -239,14 +265,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::bind(&connection::handle_read, connection::shared_from_this(), boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred))); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "]Async read requested."); + //_info("[sock " << socket_.native_handle() << "]Async read requested."); } }else { - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value()); + _dbg3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value()); if(e.value() != 2) { - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); + _dbg3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); shutdown(); } } @@ -282,9 +308,88 @@ PRAGMA_WARNING_DISABLE_VS(4355) return true; CATCH_ENTRY_L0("connection::call_run_once_service_io", false); } + //--------------------------------------------------------------------------------- + template + bool connection::do_send(const void* ptr, size_t cb) { + TRY_ENTRY(); + + // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted + auto self = safe_shared_from_this(); + if (!self) return false; + if (m_was_shutdown) return false; + // TODO avoid copy + + const double factor = 32; // TODO config + typedef long long signed int t_safe; // my t_size to avoid any overunderflow in arithmetic + const t_safe chunksize_good = (t_safe)( 1024 * std::max(1.0,factor) ); + const t_safe chunksize_max = chunksize_good * 2 ; + const bool allow_split = (m_connection_type == RPC) ? false : true; // TODO config + + if (allow_split && (cb > chunksize_max)) { + { // LOCK: chunking + epee::critical_region_t send_guard(m_chunking_lock); // *** critical *** + + _mark_c("net/out/size", "do_send() will SPLIT into small chunks, from packet="<0); + + // (in catch block, or uniq pointer) delete buf; + } // each chunk + + _mark_c("net/out/size", "do_send() DONE SPLIT from packet="<::do_send", false); + } // do_send() + //--------------------------------------------------------------------------------- template - bool connection::do_send(const void* ptr, size_t cb) + bool connection::do_send_chunk(const void* ptr, size_t cb) { TRY_ENTRY(); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted @@ -294,49 +399,80 @@ PRAGMA_WARNING_DISABLE_VS(4355) if(m_was_shutdown) return false; - LOG_PRINT("[sock " << socket_.native_handle() << "] SEND " << cb, LOG_LEVEL_4); + //_info("[sock " << socket_.native_handle() << "] SEND " << cb); context.m_last_send = time(NULL); context.m_send_cnt += cb; //some data should be wrote to stream //request complete - epee::critical_region_t send_guard(m_send_que_lock); - if(m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT) + do_send_handler_start( ptr , cb ); // (((H))) + + epee::critical_region_t send_guard(m_send_que_lock); // *** critical *** + long int retry=0; + const long int retry_limit = 5*4; + while (m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT) { - send_guard.unlock(); - LOG_PRINT_L2("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); - close(); - return false; + retry++; + + /* if ( ::cryptonote::core::get_is_stopping() ) { // TODO re-add fast stop + _fact("ABORT queue wait due to stopping"); + return false; // aborted + }*/ + + long int ms = 250 + (rand()%50); + _info_c("net/sleep", "Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="< retry_limit) { + send_guard.unlock(); + _erro("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); + // _mark_c("net/sleep", "send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); + close(); + return false; + } } m_send_que.resize(m_send_que.size()+1); m_send_que.back().assign((const char*)ptr, cb); if(m_send_que.size() > 1) - { - //active operation should be in progress, nothing to do, just wait last operation callback - }else - { - //no active operation - if(m_send_que.size()!=1) - { - LOG_ERROR("Looks like no active operations, but send que size != 1!!"); - return false; - } + { // active operation should be in progress, nothing to do, just wait last operation callback + auto size_now = cb; + _info_c("net/out/size", "do_send() NOW just queues: packet="<::handle_write, self, _1, _2) + //) + ); + //_dbg3("(chunk): " << size_now); + logger_handle_net_write(size_now); + //_info("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size()); + } + + do_send_handler_stop( ptr , cb ); return true; CATCH_ENTRY_L0("connection::do_send", false); - } + } // do_send_chunk //--------------------------------------------------------------------------------- template bool connection::shutdown() @@ -353,7 +489,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) bool connection::close() { TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Que Shutdown called."); + //_info("[sock " << socket_.native_handle() << "] Que Shutdown called."); size_t send_que_size = 0; CRITICAL_REGION_BEGIN(m_send_que_lock); send_que_size = m_send_que.size(); @@ -376,7 +512,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) if (e) { - LOG_PRINT_L1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value()); + _dbg1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value()); shutdown(); return; } @@ -385,7 +521,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) CRITICAL_REGION_BEGIN(m_send_que_lock); if(m_send_que.empty()) { - LOG_ERROR("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!"); + _erro("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!"); return; } @@ -399,10 +535,17 @@ PRAGMA_WARNING_DISABLE_VS(4355) }else { //have more data to send - boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()), - //strand_.wrap( - boost::bind(&connection::handle_write, connection::shared_from_this(), _1, _2)); - //); + auto size_now = m_send_que.front().size(); + _mark_c("net/out/size", "handle_write() NOW SENDS: packet="<::handle_write", void()); } + //--------------------------------------------------------------------------------- + template + void connection::setRPcStation() + { + m_connection_type = RPC; + _fact_c("net/sleepRPC", "set m_connection_type = RPC "); + } /************************************************************************/ /* */ /************************************************************************/ @@ -420,19 +570,27 @@ PRAGMA_WARNING_DISABLE_VS(4355) m_io_service_local_instance(new boost::asio::io_service()), io_service_(*m_io_service_local_instance.get()), acceptor_(io_service_), - new_connection_(new connection(io_service_, m_config, m_sockets_count, m_pfilter)), - m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0) + m_stop_signal_sent(false), m_port(0), + m_sock_count(0), m_sock_number(0), m_threads_count(0), + m_pfilter(NULL), m_thread_index(0), + new_connection_(new connection(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter)) { + create_server_type_map(); m_thread_name_prefix = "NET"; + type = NET; } template - boosted_tcp_server::boosted_tcp_server(boost::asio::io_service& extarnal_io_service): + boosted_tcp_server::boosted_tcp_server(boost::asio::io_service& extarnal_io_service, t_server_role s_type): io_service_(extarnal_io_service), acceptor_(io_service_), - new_connection_(new connection(io_service_, m_config, m_sockets_count, m_pfilter)), - m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0) + m_stop_signal_sent(false), m_port(0), + m_sock_count(0), m_sock_number(0), m_threads_count(0), + m_pfilter(NULL), m_thread_index(0), + type(NET), + new_connection_(new connection(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter)) { + create_server_type_map(); m_thread_name_prefix = "NET"; } //--------------------------------------------------------------------------------- @@ -444,6 +602,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template + void boosted_tcp_server::create_server_type_map() + { + server_type_map["NET"] = t_server_role::NET; + server_type_map["RPC"] = t_server_role::RPC; + server_type_map["P2P"] = t_server_role::P2P; + } + //--------------------------------------------------------------------------------- + template bool boosted_tcp_server::init_server(uint32_t port, const std::string address) { TRY_ENTRY(); @@ -491,6 +657,7 @@ POP_WARNINGS std::string thread_name = std::string("[") + m_thread_name_prefix; thread_name += boost::to_string(local_thr_index) + "]"; log_space::log_singletone::set_thread_log_prefix(thread_name); + // _fact("Thread name: " << m_thread_name_prefix); while(!m_stop_signal_sent) { try @@ -499,14 +666,14 @@ POP_WARNINGS } catch(const std::exception& ex) { - LOG_ERROR("Exception at server worker thread, what=" << ex.what()); + _erro("Exception at server worker thread, what=" << ex.what()); } catch(...) { - LOG_ERROR("Exception at server worker thread, unknown execption"); + _erro("Exception at server worker thread, unknown execption"); } } - LOG_PRINT_L4("Worker thread finished"); + //_info("Worker thread finished"); return true; CATCH_ENTRY_L0("boosted_tcp_server::worker_thread", false); } @@ -515,6 +682,9 @@ POP_WARNINGS void boosted_tcp_server::set_threads_prefix(const std::string& prefix_name) { m_thread_name_prefix = prefix_name; + type = server_type_map[m_thread_name_prefix]; + _note("Set server type to: " << type); + _note("Set server type to: " << m_thread_name_prefix); } //--------------------------------------------------------------------------------- template @@ -539,32 +709,38 @@ POP_WARNINGS { boost::shared_ptr thread(new boost::thread( attrs, boost::bind(&boosted_tcp_server::worker_thread, this))); + _note("Run server thread name: " << m_thread_name_prefix); m_threads.push_back(thread); } CRITICAL_REGION_END(); // Wait for all threads in the pool to exit. - if(wait) + if (wait) // && ! ::cryptonote::core::get_is_stopping()) // TODO fast_exit { - for (std::size_t i = 0; i < m_threads.size(); ++i) - m_threads[i]->join(); + _fact("JOINING all threads"); + for (std::size_t i = 0; i < m_threads.size(); ++i) { + m_threads[i]->join(); + } + _fact("JOINING all threads - almost"); m_threads.clear(); + _fact("JOINING all threads - DONE"); - }else - { + } + else { + _dbg1("Reiniting OK."); return true; } if(wait && !m_stop_signal_sent) { //some problems with the listening socket ?.. - LOG_PRINT_L0("Net service stopped without stop request, restarting..."); + _dbg1("Net service stopped without stop request, restarting..."); if(!this->init_server(m_port, m_address)) { - LOG_PRINT_L0("Reiniting service failed, exit."); + _dbg1("Reiniting service failed, exit."); return false; }else { - LOG_PRINT_L0("Reiniting OK."); + _dbg1("Reiniting OK."); } } } @@ -597,7 +773,7 @@ POP_WARNINGS { if(m_threads[i]->joinable() && !m_threads[i]->try_join_for(ms)) { - LOG_PRINT_L0("Interrupting thread " << m_threads[i]->native_handle()); + _dbg1("Interrupting thread " << m_threads[i]->native_handle()); m_threads[i]->interrupt(); } } @@ -626,19 +802,22 @@ POP_WARNINGS TRY_ENTRY(); if (!e) { - connection_ptr conn(std::move(new_connection_)); - - new_connection_.reset(new connection(io_service_, m_config, m_sockets_count, m_pfilter)); + if (type == RPC) { + new_connection_->setRPcStation(); + _note("New server for RPC connections"); + } + connection_ptr conn(std::move(new_connection_)); + new_connection_.reset(new connection(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter)); acceptor_.async_accept(new_connection_->socket(), boost::bind(&boosted_tcp_server::handle_accept, this, boost::asio::placeholders::error)); bool r = conn->start(true, 1 < m_threads_count); if (!r) - LOG_ERROR("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count); + _erro("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count); }else { - LOG_ERROR("Some problems at accept: " << e.message() << ", connections_count = " << m_sockets_count); + _erro("Some problems at accept: " << e.message() << ", connections_count = " << m_sock_count); } CATCH_ENTRY_L0("boosted_tcp_server::handle_accept", void()); } @@ -648,7 +827,7 @@ POP_WARNINGS { TRY_ENTRY(); - connection_ptr new_connection_l(new connection(io_service_, m_config, m_sockets_count, m_pfilter) ); + connection_ptr new_connection_l(new connection(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter) ); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); ////////////////////////////////////////////////////////////////////////// @@ -658,7 +837,7 @@ POP_WARNINGS boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - LOG_ERROR("Failed to resolve " << adr); + _erro("Failed to resolve " << adr); return false; } ////////////////////////////////////////////////////////////////////////// @@ -704,7 +883,7 @@ POP_WARNINGS { //timeout sock_.close(); - LOG_PRINT_L3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")"); + _dbg3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")"); return false; } } @@ -712,21 +891,21 @@ POP_WARNINGS if (ec || !sock_.is_open()) { - LOG_PRINT("Some problems at connect, message: " << ec.message(), LOG_LEVEL_3); + _dbg3("Some problems at connect, message: " << ec.message()); return false; } - LOG_PRINT_L3("Connected success to " << adr << ':' << port); + _dbg3("Connected success to " << adr << ':' << port); bool r = new_connection_l->start(false, 1 < m_threads_count); if (r) { new_connection_l->get_context(conn_context); - //new_connection_l.reset(new connection(io_service_, m_config, m_sockets_count, m_pfilter)); + //new_connection_l.reset(new connection(io_service_, m_config, m_sock_count, m_pfilter)); } else { - LOG_ERROR("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count); + _erro("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count); } return r; @@ -738,7 +917,7 @@ POP_WARNINGS bool boosted_tcp_server::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, t_callback cb, const std::string& bind_ip) { TRY_ENTRY(); - connection_ptr new_connection_l(new connection(io_service_, m_config, m_sockets_count, m_pfilter) ); + connection_ptr new_connection_l(new connection(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter) ); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); ////////////////////////////////////////////////////////////////////////// @@ -748,7 +927,7 @@ POP_WARNINGS boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - LOG_ERROR("Failed to resolve " << adr); + _erro("Failed to resolve " << adr); return false; } ////////////////////////////////////////////////////////////////////////// @@ -768,7 +947,7 @@ POP_WARNINGS { if(error != boost::asio::error::operation_aborted) { - LOG_PRINT_L3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")"); + _dbg3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")"); new_connection_l->socket().close(); } }); @@ -785,7 +964,7 @@ POP_WARNINGS cb(conn_context, boost::asio::error::operation_aborted);//this mean that deadline timer already queued callback with cancel operation, rare situation }else { - LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port << + _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port << " from " << lep.address().to_string() << ':' << lep.port()); bool r = new_connection_l->start(false, 1 < m_threads_count); if (r) @@ -795,13 +974,13 @@ POP_WARNINGS } else { - LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port); + _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port); cb(conn_context, boost::asio::error::fault); } } }else { - LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port << + _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port << " from " << lep.address().to_string() << ':' << lep.port() << ": " << ec_.message() << ':' << ec_.value()); cb(conn_context, ec_); } @@ -809,6 +988,6 @@ POP_WARNINGS return true; CATCH_ENTRY_L0("boosted_tcp_server::connect_async", false); } -} -} +} // namespace +} // namespace PRAGMA_WARNING_POP diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index e79768b0..5e5e803f 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -81,6 +81,7 @@ public: async_protocol_handler_config():m_pcommands_handler(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE) {} + void del_connections(size_t count); }; @@ -669,6 +670,14 @@ void async_protocol_handler_config::del_connection(async_p } //------------------------------------------------------------------------------------------ template +void async_protocol_handler_config::del_connections(size_t count) // TODO +{ + CRITICAL_REGION_BEGIN(m_connects_lock); + m_connects.clear(); + CRITICAL_REGION_END(); +} +//------------------------------------------------------------------------------------------ +template void async_protocol_handler_config::add_connection(async_protocol_handler* pconn) { CRITICAL_REGION_BEGIN(m_connects_lock); diff --git a/contrib/otshell_utils/CMakeLists.txt b/contrib/otshell_utils/CMakeLists.txt new file mode 100644 index 00000000..7413e0dc --- /dev/null +++ b/contrib/otshell_utils/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required (VERSION 2.6) +project (otshell CXX) + +# Add executable + +file(GLOB otshell_utils_sources # All files in directory: + "*.h" + "*.hpp" + "*.cpp" +) + +add_library (otshell_utils STATIC ${otshell_utils_sources}) +set_target_properties (otshell_utils PROPERTIES OUTPUT_NAME "otshell_utils") +#target_link_libraries (upnpc-static ${LDLIBS}) # to add used libs diff --git a/contrib/otshell_utils/LICENCE.txt b/contrib/otshell_utils/LICENCE.txt new file mode 100644 index 00000000..f351acf1 --- /dev/null +++ b/contrib/otshell_utils/LICENCE.txt @@ -0,0 +1,21 @@ + +This are some files also from OpenTransactions / otshell project, +developed thanks to the awesome OpenTransaction project, organization and developers :) + +Parts of code here was also developed thanks to the excellent Monero project, +thanks to Monero project, organization and developers :) + +[Some] files/code here (in external/otshell_utils) are under licence defined in +src/doc/LICENCE-otshell.txt ; +Others are from monero, with licence in src/doc/LICENCE-monero.txt ; + +For me (rfree) the licence seem compatbile so no problem, personally (as author of many parts of the code, +possibly not all) I do not worry who uses it how; I'am not a lawyer. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Please share :-) This licence can be used e.g. for parts of code that are usable in both open-source FOSS project +Monero and Open Transactions, to share and develop both faster. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + diff --git a/contrib/otshell_utils/ccolor.cpp b/contrib/otshell_utils/ccolor.cpp new file mode 100644 index 00000000..cd93e0de --- /dev/null +++ b/contrib/otshell_utils/ccolor.cpp @@ -0,0 +1,116 @@ +#include "ccolor.hpp" +#include + +// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal +// from http://wiznet.gr/src/ccolor.zip +// edited by rfree - as part of https://github.com/rfree/Open-Transactions/ + +using namespace std; + +#ifdef _MSC_VER + +#define snprintf c99_snprintf + +inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) { + int count = -1; + if (size != 0) + count = _vsnprintf_s(str, size, _TRUNCATE, format, ap); + if (count == -1) + count = _vscprintf(format, ap); + return count; +} + +inline int c99_snprintf(char* str, size_t size, const char* format, ...) { + int count; + va_list ap; + va_start(ap, format); + count = c99_vsnprintf(str, size, format, ap); + va_end(ap); + return count; +} +#endif // _MSC_VER + +#define CC_CONSOLE_COLOR_DEFAULT "\033[0m" +#define CC_FORECOLOR(C) "\033[" #C "m" +#define CC_BACKCOLOR(C) "\033[" #C "m" +#define CC_ATTR(A) "\033[" #A "m" + +namespace zkr +{ + enum Color + { + Black, + Red, + Green, + Yellow, + Blue, + Magenta, + Cyan, + White, + Default = 9 + }; + + enum Attributes + { + Reset, + Bright, + Dim, + Underline, + Blink, + Reverse, + Hidden + }; + + char * cc::color(int attr, int fg, int bg) + { + static const int size = 20; + static char command[size]; + + /* Command is the control command to the terminal */ + snprintf(command, size, "%c[%d;%d;%dm", 0x1B, attr, fg + 30, bg + 40); + return command; + } + + + const char *cc::console = CC_CONSOLE_COLOR_DEFAULT; + const char *cc::underline = CC_ATTR(4); + const char *cc::bold = CC_ATTR(1); + + const char *cc::fore::black = CC_FORECOLOR(30); + const char *cc::fore::blue = CC_FORECOLOR(34); + const char *cc::fore::red = CC_FORECOLOR(31); + const char *cc::fore::magenta = CC_FORECOLOR(35); + const char *cc::fore::green = CC_FORECOLOR(92); + const char *cc::fore::cyan = CC_FORECOLOR(36); + const char *cc::fore::yellow = CC_FORECOLOR(33); + const char *cc::fore::white = CC_FORECOLOR(37); + const char *cc::fore::console = CC_FORECOLOR(39); + + const char *cc::fore::lightblack = CC_FORECOLOR(90); + const char *cc::fore::lightblue = CC_FORECOLOR(94); + const char *cc::fore::lightred = CC_FORECOLOR(91); + const char *cc::fore::lightmagenta = CC_FORECOLOR(95); + const char *cc::fore::lightgreen = CC_FORECOLOR(92); + const char *cc::fore::lightcyan = CC_FORECOLOR(96); + const char *cc::fore::lightyellow = CC_FORECOLOR(93); + const char *cc::fore::lightwhite = CC_FORECOLOR(97); + + const char *cc::back::black = CC_BACKCOLOR(40); + const char *cc::back::blue = CC_BACKCOLOR(44); + const char *cc::back::red = CC_BACKCOLOR(41); + const char *cc::back::magenta = CC_BACKCOLOR(45); + const char *cc::back::green = CC_BACKCOLOR(42); + const char *cc::back::cyan = CC_BACKCOLOR(46); + const char *cc::back::yellow = CC_BACKCOLOR(43); + const char *cc::back::white = CC_BACKCOLOR(47); + const char *cc::back::console = CC_BACKCOLOR(49); + + const char *cc::back::lightblack = CC_BACKCOLOR(100); + const char *cc::back::lightblue = CC_BACKCOLOR(104); + const char *cc::back::lightred = CC_BACKCOLOR(101); + const char *cc::back::lightmagenta = CC_BACKCOLOR(105); + const char *cc::back::lightgreen = CC_BACKCOLOR(102); + const char *cc::back::lightcyan = CC_BACKCOLOR(106); + const char *cc::back::lightyellow = CC_BACKCOLOR(103); + const char *cc::back::lightwhite = CC_BACKCOLOR(107); +} diff --git a/contrib/otshell_utils/ccolor.hpp b/contrib/otshell_utils/ccolor.hpp new file mode 100644 index 00000000..bf5a601a --- /dev/null +++ b/contrib/otshell_utils/ccolor.hpp @@ -0,0 +1,73 @@ +// ccolor.hpp + +// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal +// from http://wiznet.gr/src/ccolor.zip +// edited by rfree - as part of https://github.com/rfree/Open-Transactions/ + +#ifndef INCLUDE_OT_ccolor +#define INCLUDE_OT_ccolor + +#include +#include + +namespace zkr +{ + class cc + { + public: + + class fore + { + public: + static const char *black; + static const char *blue; + static const char *red; + static const char *magenta; + static const char *green; + static const char *cyan; + static const char *yellow; + static const char *white; + static const char *console; + + static const char *lightblack; + static const char *lightblue; + static const char *lightred; + static const char *lightmagenta; + static const char *lightgreen; + static const char *lightcyan; + static const char *lightyellow; + static const char *lightwhite; + }; + + class back + { + public: + static const char *black; + static const char *blue; + static const char *red; + static const char *magenta; + static const char *green; + static const char *cyan; + static const char *yellow; + static const char *white; + static const char *console; + + static const char *lightblack; + static const char *lightblue; + static const char *lightred; + static const char *lightmagenta; + static const char *lightgreen; + static const char *lightcyan; + static const char *lightyellow; + static const char *lightwhite; + }; + + static char *color(int attr, int fg, int bg); + static const char *console; + static const char *underline; + static const char *bold; + }; +} + +#endif + diff --git a/contrib/otshell_utils/lib_common1.hpp b/contrib/otshell_utils/lib_common1.hpp new file mode 100644 index 00000000..108b1847 --- /dev/null +++ b/contrib/otshell_utils/lib_common1.hpp @@ -0,0 +1,51 @@ +/* See other files here for the LICENCE that applies here. */ + + +#ifndef INCLUDE_OT_NEWCLI_COMMON1 +#define INCLUDE_OT_NEWCLI_COMMON1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +// list of thigs from libraries that we pull into namespace nOT::nNewcli +// we might still need to copy/paste it in few places to make IDEs pick it up correctly +#define INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 \ + using std::string; \ + using std::vector; \ + using std::vector; \ + using std::list; \ + using std::set; \ + using std::map; \ + using std::ostream; \ + using std::istream; \ + using std::cin; \ + using std::cerr; \ + using std::cout; \ + using std::cerr; \ + using std::endl; \ + using std::function; \ + using std::unique_ptr; \ + using std::shared_ptr; \ + using std::weak_ptr; \ + using std::enable_shared_from_this; \ + using std::mutex; \ + using std::lock_guard; \ + +#endif + diff --git a/contrib/otshell_utils/runoptions.cpp b/contrib/otshell_utils/runoptions.cpp new file mode 100644 index 00000000..28e7ceb5 --- /dev/null +++ b/contrib/otshell_utils/runoptions.cpp @@ -0,0 +1,69 @@ +/* See other files here for the LICENCE that applies here. */ +/* See header file .hpp for info */ + +#include "runoptions.hpp" + +#include "lib_common1.hpp" + +namespace nOT { + +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces + +// (no debug - this is the default) +// +nodebug (no debug) +// +debug ...... --asdf +// +debug +debugcerr .... --asfs +// +debug +debugfile .... --asfs + +cRunOptions::cRunOptions() + : mRunMode(eRunModeCurrent), mDebug(false), mDebugSendToFile(false), mDebugSendToCerr(false) + ,mDoRunDebugshow(false) +{ } + +vector cRunOptions::ExecuteRunoptionsAndRemoveThem(const vector & args) { + vector arg_clear; // will store only the arguments that are not removed + + for (auto arg : args) { + bool thisIsRunoption=false; + + if (arg.size()>0) { + if (arg.at(0) == '+') thisIsRunoption=true; + } + + if (thisIsRunoption) Exec(arg); // *** + if (! thisIsRunoption) arg_clear.push_back(arg); + } + + Normalize(); + + return arg_clear; +} + +void cRunOptions::Exec(const string & runoption) { // eg: Exec("+debug"); + if (runoption == "+nodebug") { mDebug=false; } + else if (runoption == "+debug") { mDebug=true; } + else if (runoption == "+debugcerr") { mDebug=true; mDebugSendToCerr=true; } + else if (runoption == "+debugfile") { mDebug=true; mDebugSendToFile=true; } + else if (runoption == "+demo") { mRunMode=eRunModeDemo; } + else if (runoption == "+normal") { mRunMode=eRunModeNormal; } + else if (runoption == "+current") { mRunMode=eRunModeCurrent; } + else if (runoption == "+debugshow") { mDebug=true; mDebugSendToCerr=true; mDoRunDebugshow=true; } + else { + cerr << "Unknown runoption in Exec: '" << runoption << "'" << endl; + throw std::runtime_error("Unknown runoption"); + } + // cerr<<"debug="< ExecuteRunoptionsAndRemoveThem(const vector & args); + void Exec(const string & runoption); // eg: Exec("+debug"); + + void Normalize(); +}; + +extern cRunOptions gRunOptions; + + +}; // namespace nOT + + + +#endif + diff --git a/contrib/otshell_utils/utils.cpp b/contrib/otshell_utils/utils.cpp new file mode 100644 index 00000000..489fb307 --- /dev/null +++ b/contrib/otshell_utils/utils.cpp @@ -0,0 +1,612 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug + +/* See other files here for the LICENCE that applies here. */ +/* See header file .hpp for info */ + +#include +#include +#include +#include +#include +#include +#include + +#include "utils.hpp" + +#include "ccolor.hpp" + +#include "lib_common1.hpp" + +#include "runoptions.hpp" + +#if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined (WIN64) + #define OS_TYPE_WINDOWS +#elif defined(__unix__) || defined(__posix) || defined(__linux) || defined(__darwin) || defined(__APPLE__) || defined(__clang__) + #define OS_TYPE_POSIX +#else + #warning "Compiler/OS platform is not recognized" + #warning "Just assuming it will work as POSIX then" + #define OS_TYPE_POSIX +#endif + +#if defined(OS_TYPE_WINDOWS) + #include +#elif defined(OS_TYPE_POSIX) + #include + #include + #include +#else + #error "Compiler/OS platform detection failed - not supported" +#endif + +namespace nOT { +namespace nUtils { + +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces + +myexception::myexception(const char * what) + : std::runtime_error(what) +{ } + +myexception::myexception(const std::string &what) + : std::runtime_error(what) +{ } + +void myexception::Report() const { + _erro("Error: " << what()); +} + +//myexception::~myexception() { } + +// ==================================================================== + +// text trimming +// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring +std::string & ltrim(std::string &s) { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun(std::isspace)))); + return s; +} + +std::string & rtrim(std::string &s) { + s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun(std::isspace))).base(), s.end()); + return s; +} + +std::string & trim(std::string &s) { + return ltrim(rtrim(s)); +} + +std::string get_current_time() +{ + std::stringstream stream; + struct tm * date; + + std::chrono::high_resolution_clock::time_point now = std::chrono::high_resolution_clock::now(); + time_t time_now; + time_now = std::chrono::high_resolution_clock::to_time_t(now); + date = std::localtime(& time_now); + + char date_buff[32]; + std::strftime(date_buff, sizeof(date_buff), "%d-%b-%Y %H:%M:%S.", date); + stream << date_buff; + + std::chrono::high_resolution_clock::duration duration = now.time_since_epoch(); + int64_t micro = std::chrono::duration_cast(duration).count(); + micro %= 1000000; + stream << std::setfill('0') << std::setw(3) << micro; + + return stream.str(); +} + +cNullstream g_nullstream; // extern a stream that does nothing (eats/discards data) + +std::mutex gLoggerGuard; // extern + +// ==================================================================== + +namespace nDetail { + +const char* DbgShortenCodeFileName(const char *s) { + const char *p = s; + const char *a = s; + + bool inc=1; + while (*p) { + ++p; + if (inc && ('\0' != * p)) { a=p; inc=false; } // point to the current character (if valid) becasue previous one was slash + if ((*p)=='/') { a=p; inc=true; } // point at current slash (but set inc to try to point to next character) + } + return a; +} + +} + +// a workaround for MSVC compiler; e.g. see https://bugs.webkit.org/show_bug.cgi?format=multiple&id=125795 +#ifndef _MSC_VER +template +std::unique_ptr make_unique( Args&& ...args ) +{ + return std::unique_ptr( new T( std::forward(args)... ) ); +} +#else + using std::make_unique; +#endif +// ==================================================================== + +char cFilesystemUtils::GetDirSeparator() { + // TODO nicer os detection? + #if defined(OS_TYPE_POSIX) + return '/'; + #elif defined(OS_TYPE_WINDOWS) + return '\\'; + #else + #error "Do not know how to compile this for your platform." + #endif +} + +bool cFilesystemUtils::CreateDirTree(const std::string & dir, bool only_below) { + const bool dbg=false; + //struct stat st; + const char dirch = cFilesystemUtils::GetDirSeparator(); + std::istringstream iss(dir); + string part, sofar=""; + if (dir.size()<1) return false; // illegal name + // dir[0] is valid from here + if (only_below && (dir[0]==dirch)) return false; // no jumping to top (on any os) + while (getline(iss,part,dirch)) { + if (dbg) cout << '['<= mLevel) && (mStream)) { + ostream & output = SelectOutput(level,channel); + output << icon(level) << ' '; + std::thread::id this_id = std::this_thread::get_id(); + output << "{" << Thread2Number(this_id) << "} "; + return output; + } + return g_nullstream; +} + +std::string cLogger::GetLogBaseDir() const { + return "log"; +} + +void cLogger::OpenNewChannel(const std::string & channel) { + size_t last_split = channel.find_last_of(cFilesystemUtils::GetDirSeparator()); + // log/test/aaa + // ^----- last_split + string dir = GetLogBaseDir() + cFilesystemUtils::GetDirSeparator() + channel.substr(0, last_split); + string basefile = channel.substr(last_split+1) + ".log"; + string fname = dir + cFilesystemUtils::GetDirSeparator() + cFilesystemUtils::GetDirSeparator() + basefile; + _dbg1("Starting debug to channel file: " + fname + " in directory ["+dir+"]"); + bool dirok = cFilesystemUtils::CreateDirTree(dir); + if (!dirok) { const string msg = "In logger failed to open directory (" + dir +")."; _erro(msg); throw std::runtime_error(msg); } + std::ofstream * thefile = new std::ofstream( fname.c_str() ); + *thefile << "====== (Log opened: " << fname << ") ======" << endl; + mChannels.insert( std::pair(channel , thefile ) ); +} + +std::ostream & cLogger::SelectOutput(int level, const std::string & channel) { + if (channel=="") return *mStream; + auto obj = mChannels.find(channel); + if (obj == mChannels.end()) { // new channel + OpenNewChannel(channel); + return SelectOutput(level,channel); + } + else { // existing + return * obj->second; + } +} + + +void cLogger::setOutStreamFile(const string &fname) { // switch to using this file + _mark("WILL SWITCH DEBUG NOW to file: " << fname); + mOutfile = make_unique(fname); + mStream = & (*mOutfile); + _mark("Started new debug, to file: " << fname); +} + +void cLogger::setOutStreamFromGlobalOptions() { + if ( gRunOptions.getDebug() ) { + if ( gRunOptions.getDebugSendToFile() ) { + mOutfile = make_unique ("debuglog.txt"); + mStream = & (*mOutfile); + } + else if ( gRunOptions.getDebugSendToCerr() ) { + mStream = & std::cerr; + } + else { + mStream = & g_nullstream; + } + } + else { + mStream = & g_nullstream; + } +} + +void cLogger::setDebugLevel(int level) { + bool note_before = (mLevel > level); // report the level change before or after the change? (on higher level) + if (note_before) _note("Setting debug level to "<= 100) return cc::back::red + ToStr(cc::fore::black) + ToStr("ERROR ") + ToStr(cc::fore::lightyellow) + " " ; + if (level >= 90) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("Warn ") + ToStr(cc::fore::red)+ " " ; + if (level >= 80) return cc::back::lightmagenta + ToStr(cc::fore::black) + ToStr("MARK "); //+ zkr::cc::console + ToStr(cc::fore::lightmagenta)+ " "; + if (level >= 75) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("FACT ") + zkr::cc::console + ToStr(cc::fore::lightyellow)+ " "; + if (level >= 70) return cc::fore::green + ToStr("Note "); + if (level >= 50) return cc::fore::cyan + ToStr("info "); + if (level >= 40) return cc::fore::lightwhite + ToStr("dbg "); + if (level >= 30) return cc::fore::lightblue + ToStr("dbg "); + if (level >= 20) return cc::fore::blue + ToStr("dbg "); + + return " "; +} + +std::string cLogger::endline() const { + return ToStr("") + zkr::cc::console + ToStr("\n"); // TODO replan to avoid needles converting back and forth char*, string etc +} + +int cLogger::Thread2Number(const std::thread::id id) { + auto found = mThread2Number.find( id ); + if (found == mThread2Number.end()) { // new one + mThread2Number_Biggest++; + mThread2Number[id] = mThread2Number_Biggest; + return mThread2Number_Biggest; + // _info("(This is a new thread)"); // recursion! + } else { + return mThread2Number[id]; + } +} + + +// ==================================================================== +// object gCurrentLogger is defined later - in global namespace below + + +// ==================================================================== +// vector debug + +void DisplayStringEndl(std::ostream & out, const std::string text) { + out << text; + out << std::endl; +} + +std::string SpaceFromEscape(const std::string &s) { + std::ostringstream newStr; + for(size_t i = 0; i < s.length();i++) { + if(s[i] == '\\' && s[i+1] ==32) + newStr<<""; + else + newStr<=32 && s[i] <= 126) + newStr<= ending.length()) { + return (0 == all.compare (all.length() - ending.length(), ending.length(), ending)); + } else { + return false; + } +} + + +vector WordsThatMatch(const std::string & sofar, const vector & possib) { + vector ret; + for ( auto rec : possib) { // check of possibilities + if (CheckIfBegins(sofar,rec)) { + rec = EscapeFromSpace(rec); + ret.push_back(rec); // this record matches + } + } + return ret; +} + +char GetLastChar(const std::string & str) { // TODO unicode? + auto s = str.length(); + if (s==0) throw std::runtime_error("Getting last character of empty string (" + ToStr(s) + ")" + OT_CODE_STAMP); + return str.at( s - 1); +} + +std::string GetLastCharIf(const std::string & str) { // TODO unicode? + auto s = str.length(); + if (s==0) return ""; // empty string signalizes ther is nothing to be returned + return std::string( 1 , str.at( s - 1) ); +} + +// ==================================================================== + +// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no. +// Use it like this: ASRT( x>y ); with the semicolon at end, a clever trick forces this syntax :) + +void Assert(bool result, const std::string &stamp, const std::string &condition) { + if (!result) { + _erro("Assert failed at "+stamp+": ASSERT( " << condition << ")"); + throw std::runtime_error("Assert failed at "+stamp+": ASSERT( " + condition + ")"); + } +} + +// ==================================================================== +// advanced string + +const std::string GetMultiline(string endLine) { + std::string result(""); // Taken from OT_CLI_ReadUntilEOF + while (true) { + std::string input_line(""); + if (std::getline(std::cin, input_line, '\n')) + { + input_line += "\n"; + if (input_line[0] == '~') + break; + result += input_line; + } + if (std::cin.eof() ) + { + std::cin.clear(); + break; + } + if (std::cin.fail() ) + { + std::cin.clear(); + break; + } + if (std::cin.bad()) + { + std::cin.clear(); + break; + } + } + return result; +} + +vector SplitString(const string & str){ + std::istringstream iss(str); + vector vec { std::istream_iterator{iss}, std::istream_iterator{} }; + return vec; +} + +bool checkPrefix(const string & str, char prefix) { + if (str.at(0) == prefix) + return true; + return false; +} + +// ==================================================================== +// operation on files + + +#ifdef __unix + +void cEnvUtils::GetTmpTextFile() { + // TODO make this name configurable (depending on project) + char filename[] = "/tmp/otshellutils_text.XXXXXX"; + fd = mkstemp(filename); + if (fd == -1) { + _erro("Can't create the file: " << filename); + return; + } + mFilename = filename; +} + +void cEnvUtils::CloseFile() { + close(fd); + unlink( mFilename.c_str() ); +} + +void cEnvUtils::OpenEditor() { + char* editor = std::getenv("OT_EDITOR"); //TODO Read editor from configuration file + if (editor == NULL) + editor = std::getenv("VISUAL"); + if (editor == NULL) + editor = std::getenv("EDITOR"); + + string command; + if (editor != NULL) + command = ToStr(editor) + " " + mFilename; + else + command = "/usr/bin/editor " + mFilename; + _dbg3("Opening editor with command: " << command); + if ( system( command.c_str() ) == -1 ) + _erro("Cannot execute system command: " << command); +} + +const string cEnvUtils::ReadFromTmpFile() { + std::ifstream ifs(mFilename); + string msg((std::istreambuf_iterator(ifs)), std::istreambuf_iterator()); + return msg; +} + +const string cEnvUtils::Compose() { + GetTmpTextFile(); + OpenEditor(); + string input = ReadFromTmpFile(); + CloseFile(); + return input; +} + +#endif + +const string cEnvUtils::ReadFromFile(const string path) { + std::ifstream ifs(path); + string msg((std::istreambuf_iterator(ifs)), std::istreambuf_iterator()); + return msg; +} + +void hintingToTxt(std::fstream & file, string command, vector &commands) { + if(file.good()) { + file< lightColors; + using namespace zkr; + lightColors.push_back(cc::fore::lightblue); + lightColors.push_back(cc::fore::lightred); + lightColors.push_back(cc::fore::lightmagenta); + lightColors.push_back(cc::fore::lightgreen); + lightColors.push_back(cc::fore::lightcyan); + lightColors.push_back(cc::fore::lightyellow); + lightColors.push_back(cc::fore::lightwhite); + + int sum=0; + + for (auto ch : hash) sum+=ch; + auto color = sum%(lightColors.size()-1); + + return lightColors.at( color ); +} + + +// ==================================================================== +// algorthms + + +}; // namespace nUtil + + +}; // namespace OT + +// global namespace + +const extern int _dbg_ignore = 0; // see description in .hpp + +std::string GetObjectName() { + //static std::string * name=nullptr; + //if (!name) name = new std::string("(global)"); + return ""; +} + +// ==================================================================== + +nOT::nUtils::cLogger gCurrentLogger; + diff --git a/contrib/otshell_utils/utils.hpp b/contrib/otshell_utils/utils.hpp new file mode 100644 index 00000000..6cfd11ee --- /dev/null +++ b/contrib/otshell_utils/utils.hpp @@ -0,0 +1,446 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug + +/* See other files here for the LICENCE that applies here. */ + +#include "ccolor.hpp" +#ifndef INCLUDE_OT_NEWCLI_UTILS +#define INCLUDE_OT_NEWCLI_UTILS + +#include "lib_common1.hpp" +#ifdef __unix + #include +#endif + +#ifndef CFG_WITH_TERMCOLORS + #error "You requested to turn off terminal colors (CFG_WITH_TERMCOLORS), however currently they are hardcoded (this option to turn them off is not yet implemented)." +#endif + +///Macros related to automatic deduction of class name etc; +#define MAKE_CLASS_NAME(NAME) private: static std::string GetObjectName() { return #NAME; } +#define MAKE_STRUCT_NAME(NAME) private: static std::string GetObjectName() { return #NAME; } public: + +namespace nOT { + +namespace nUtils { + +/// @brief general based for my runtime errors +class myexception : public std::runtime_error { + public: + myexception(const char * what); + myexception(const std::string &what); + //virtual ~myexception(); + virtual void Report() const; +}; + +/// @macro Use this macro INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 as a shortcut for various using std::string etc. +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces + +// ====================================================================================== +/// text trimming functions (they do mutate the passes string); they trim based on std::isspace. also return it's reference again +/// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring +std::string & trim(std::string &s); ///< trim text http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring +std::string & ltrim(std::string &s); ///< left trim +std::string & rtrim(std::string &s); ///< right trim + +// ====================================================================================== + +std::string get_current_time(); + +// string conversions +template +std::string ToStr(const T & obj) { + std::ostringstream oss; + oss << obj; + return oss.str(); +} + +struct cNullstream : std::ostream { + cNullstream() : std::ios(0), std::ostream(0) {} +}; +extern cNullstream g_nullstream; // a stream that does nothing (eats/discards data) +// ========== debug ========== +// _dbg_ignore is moved to global namespace (on purpose) + +// TODO make _dbg_ignore thread-safe everywhere + +extern std::mutex gLoggerGuard; + + + +#define _debug_level_c(CHANNEL,LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \ + nOT::nUtils::gLoggerGuard.try_lock(); \ + gCurrentLogger.write_stream(LEVEL,CHANNEL) << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \ + nOT::nUtils::gLoggerGuard.unlock(); \ + } } while(0) + +#define _debug_level(LEVEL,VAR) _debug_level_c("",LEVEL,VAR) + +#define _dbg3(VAR) _debug_level( 20,VAR) +#define _dbg2(VAR) _debug_level( 30,VAR) +#define _dbg1(VAR) _debug_level( 40,VAR) // details +#define _info(VAR) _debug_level( 50,VAR) // more boring info +#define _note(VAR) _debug_level( 70,VAR) // info +#define _fact(VAR) _debug_level( 75,VAR) // interesting event +#define _mark(VAR) _debug_level( 80,VAR) // marked action +#define _warn(VAR) _debug_level( 90,VAR) // some problem +#define _erro(VAR) _debug_level(100,VAR) // error - report + +#define _dbg3_c(C,VAR) _debug_level_c(C, 20,VAR) +#define _dbg2_c(C,VAR) _debug_level_c(C, 30,VAR) +#define _dbg1_c(C,VAR) _debug_level_c(C, 40,VAR) // details +#define _info_c(C,VAR) _debug_level_c(C, 50,VAR) // more boring info +#define _note_c(C,VAR) _debug_level_c(C, 70,VAR) // info +#define _fact_c(C,VAR) _debug_level_c(C, 75,VAR) // interesting event +#define _mark_c(C,VAR) _debug_level_c(C, 80,VAR) // marked action +#define _warn_c(C,VAR) _debug_level_c(C, 90,VAR) // some problem +#define _erro_c(C,VAR) _debug_level_c(C,100,VAR) // error - report + +// lock // because od VAR +#define _scope_debug_level_c(CHANNEL,LEVEL,VAR) \ + std::ostringstream debug_detail_oss; \ + nOT::nUtils::gLoggerGuard.try_lock(); \ + debug_detail_oss << OT_CODE_STAMP << ' ' << VAR ; \ + nOT::nUtils::nDetail::cDebugScopeGuard debugScopeGuard; \ + if (_dbg_ignore mOutfile; + std::ostream * mStream; ///< pointing only! can point to our own mOutfile, or maye to global null stream + + std::map< std::string , std::ofstream * > mChannels; // the ofstream objects are owned by this class + + int mLevel; ///< current debug level + + std::ostream & SelectOutput(int level, const std::string & channel); + void OpenNewChannel(const std::string & channel); + std::string GetLogBaseDir() const; + + std::map< std::thread::id , int > mThread2Number; // change long thread IDs into a short nice number to show + int mThread2Number_Biggest; // current biggest value held there (biggest key) - works as growing-only counter basically + int Thread2Number(const std::thread::id id); // convert the system's thread id into a nice short our id; make one if new thread +}; + + + +// ==================================================================== +// vector debug + +template +std::string vectorToStr(const T & v) { + std::ostringstream oss; + for(auto rec: v) { + oss << rec <<","; + } + return oss.str(); +} + +template +void DisplayVector(std::ostream & out, const std::vector &v, const std::string &delim=" ") { + std::copy( v.begin(), v.end(), std::ostream_iterator(out, delim.c_str()) ); +} + +template +void EndlDisplayVector(std::ostream & out, const std::vector &v, const std::string &delim=" ") { + out << std::endl; + DisplayVector(out,v,delim); +} + +template +void DisplayVectorEndl(std::ostream & out, const std::vector &v, const std::string &delim=" ") { + DisplayVector(out,v,delim); + out << std::endl; +} + +template +void DbgDisplayVector(const std::vector &v, const std::string &delim=" ") { + std::cerr << "["; + std::copy( v.begin(), v.end(), std::ostream_iterator(std::cerr, delim.c_str()) ); + std::cerr << "]"; +} + +string stringToColor(const string &hash); +template +void DisplayMap(std::ostream & out, const std::map &m, const std::string &delim=" ") { + auto *no_color = zkr::cc::fore::console; + for(auto var : m) { + out << stringToColor(var.first) << var.first << delim << var.second << no_color << endl; + } + +} + +template +void EndlDisplayMap(std::ostream & out, const std::map &m, const std::string &delim=" ") { + out << endl; + for(auto var : m) { + out << var.first << delim << var.second << endl; + } +} + +template +void DbgDisplayMap(const std::map &m, const std::string &delim=" ") { + for(auto var : m) { + std::cerr << var.first << delim << var.second << endl; + } +} + + +template +void DbgDisplayVectorEndl(const std::vector &v, const std::string &delim=" ") { + DbgDisplayVector(v,delim); + std::cerr << std::endl; +} + +void DisplayStringEndl(std::ostream & out, const std::string text); + +bool CheckIfBegins(const std::string & beggining, const std::string & all); +bool CheckIfEnds (std::string const & ending, std::string const & all); +std::string SpaceFromEscape(const std::string &s); +std::string EscapeFromSpace(const std::string &s); +vector WordsThatMatch(const std::string & sofar, const vector & possib); +char GetLastChar(const std::string & str); +std::string GetLastCharIf(const std::string & str); // TODO unicode? +std::string EscapeString(const std::string &s); + + +template +std::string DbgVector(const std::vector &v, const std::string &delim="|") { + std::ostringstream oss; + oss << "["; + bool first=true; + for(auto vElement : v) { if (!first) oss<(oss, delim.c_str()) ); + return oss.str(); +} + +template +std::ostream & operator<<(std::ostream & os, const map< T, vector > & obj){ + os << "["; + for(auto const & elem : obj) { + os << " [" << elem.first << "=" << DbgVector(elem.second) << "] "; + } + os << "]"; + return os; +} + +template +std::string DbgMap(const map & map) { + std::ostringstream oss; + oss << map; + return oss.str(); +} + +// ==================================================================== +// assert + +// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no. +// Use it like this: ASRT( x>y ); with the semicolon at end, a clever trick forces this syntax :) +#define ASRT(x) do { if (!(x)) nOT::nUtils::Assert(false, OT_CODE_STAMP, #x); } while(0) + +void Assert(bool result, const std::string &stamp, const std::string &condition); + +// ==================================================================== +// advanced string + +const std::string GetMultiline(string endLine = "~"); +vector SplitString(const string & str); + +bool checkPrefix(const string & str, char prefix = '^'); + +// ==================================================================== +// nUse utils + +enum class eSubjectType {Account, Asset, User, Server, Unknown}; + +string SubjectType2String(const eSubjectType & type); +eSubjectType String2SubjectType(const string & type); + +// ==================================================================== +// operation on files + +/// @brief tools related to filesystem +/// @author rfree (maintainer) +class cFilesystemUtils { // if we do not want to use boost in given project (or we could optionally write boost here later) + public: + static bool CreateDirTree(const std::string & dir, bool only_below=false); + static char GetDirSeparator(); // eg '/' or '\' +}; + + +/// @brief utils to e.g. edit a file from console +/// @author rfree (maintainer) +class cEnvUtils { + int fd; + string mFilename; + + void GetTmpTextFile(); + void CloseFile(); + void OpenEditor(); + const string ReadFromTmpFile(); +public: + const string Compose(); + const string ReadFromFile(const string path); +}; +void hintingToTxt(std::fstream & file, string command, vector &commands); +void generateQuestions (std::fstream & file, string command); +void generateAnswers (std::fstream & file, string command, vector &completions); + +// ==================================================================== + +namespace nOper { // nOT::nUtils::nOper +// cool shortcut operators, like vector + vecotr operator working same as string (appending) +// isolated to namespace because it's unorthodox ide to implement this + +using namespace std; + +// TODO use && and move? +template +vector operator+(const vector &a, const vector &b) { + vector ret = a; + ret.insert( ret.end() , b.begin(), b.end() ); + return ret; +} + +template +vector operator+(const T &a, const vector &b) { + vector ret(1,a); + ret.insert( ret.end() , b.begin(), b.end() ); + return ret; +} + +template +vector operator+(const vector &a, const T &b) { + vector b_vector(1,a); + return a + b_vector; +} + +template +vector& operator+=(vector &a, const vector &b) { + a.insert( a.end() , b.begin(), b.end() ); + return a; +} + +// map +template +map operator+(const map &a, const map &b) { + map ret = a; + for (const auto & elem : b) { + ret.insert(elem); + } + return ret; +} + + +} // nOT::nUtils::nOper + +// ==================================================================== + +// ==================================================================== + +// Algorithms + +// ==================================================================== +// ==================================================================== + + +/** +@brief Special type that on creation will be initialized to have value INIT given as template argument. +Might be usefull e.g. to express in the declaration of class what will be the default value of member variable +See also http://www.boost.org/doc/libs/1_56_0/libs/utility/value_init.htm +Probably not needed when using boost in your project. +*/ +template +class value_init { + private: + T data; + public: + value_init(); + + T& operator=(const T& v) { data=v; return *this; } + operator T const &() const { return data; } + operator T&() { return data; } +}; + +template +value_init::value_init() : data(INIT) { } + +}; // namespace nUtils + +}; // namespace nOT + + +// global namespace +extern nOT::nUtils::cLogger gCurrentLogger; ///< The current main logger. Usually do not use it directly, instead use macros like _dbg1 etc + +std::string GetObjectName(); ///< Method to return name of current object; To use in debug; Can be shadowed in your classes. (Might be not used currently) + +const extern int _dbg_ignore; ///< the global _dbg_ignore, but local code (blocks, classes etc) you could shadow it in your code blocks, +// to override debug compile-time setting for given block/class, e.g. to disable debug in one of your methods or increase it there. +// Or to make it runtime by providing a class normal member and editing it in runtime + +#define OT_CODE_STAMP ( nOT::nUtils::ToStr("[") + nOT::nUtils::nDetail::DbgShortenCodeFileName(__FILE__) + nOT::nUtils::ToStr("+") + nOT::nUtils::ToStr(__LINE__) + nOT::nUtils::ToStr(" ") + (GetObjectName()) + nOT::nUtils::ToStr("::") + nOT::nUtils::ToStr(__FUNCTION__) + nOT::nUtils::ToStr("]")) + + + + +#endif + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a80dfe37..60501cef 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -92,6 +92,8 @@ add_subdirectory(cryptonote_core) add_subdirectory(mnemonics) add_subdirectory(rpc) add_subdirectory(wallet) +add_subdirectory(p2p) +add_subdirectory(cryptonote_protocol) add_subdirectory(connectivity_tool) add_subdirectory(miner) diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index 3abf93f3..9eed1187 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -70,6 +70,7 @@ target_link_libraries(cryptonote_core LINK_PUBLIC common crypto + otshell_utils ${Boost_DATE_TIME_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index e2b6f232..78419121 100644 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -49,6 +49,7 @@ #include "crypto/hash.h" #include "cryptonote_core/checkpoints_create.h" //#include "serialization/json_archive.h" +#include "../../contrib/otshell_utils/utils.hpp" using namespace cryptonote; @@ -1153,6 +1154,31 @@ uint64_t blockchain_storage::block_difficulty(size_t i) return m_blocks[i].cumulative_difficulty - m_blocks[i-1].cumulative_difficulty; } //------------------------------------------------------------------ +double blockchain_storage::get_avg_block_size( size_t count) +{ + if (count > get_current_blockchain_height()) return 500; + + double average = 0; + _dbg1_c("net/blksize", "HEIGHT: " << get_current_blockchain_height()); + _dbg1_c("net/blksize", "BLOCK ID BY HEIGHT: " << get_block_id_by_height(get_current_blockchain_height()) ); + _dbg1_c("net/blksize", "BLOCK TAIL ID: " << get_tail_id() ); + std::vector size_vector; + + get_backward_blocks_sizes(get_current_blockchain_height() - count, size_vector, count); + + std::vector::iterator it; + it = size_vector.begin(); + while (it != size_vector.end()) { + average += *it; + _dbg2_c("net/blksize", "VECTOR ELEMENT: " << (*it) ); + it++; + } + average = average / count; + _dbg1_c("net/blksize", "VECTOR SIZE: " << size_vector.size() << " average=" << average); + + return average; +} +//------------------------------------------------------------------ void blockchain_storage::print_blockchain(uint64_t start_index, uint64_t end_index) { std::stringstream ss; diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index 1bfdf7bd..38bdfbce 100644 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -134,6 +134,7 @@ namespace cryptonote uint64_t get_current_comulative_blocksize_limit(); bool is_storing_blockchain(){return m_is_blockchain_storing;} uint64_t block_difficulty(size_t i); + double get_avg_block_size( size_t count); template bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) diff --git a/src/cryptonote_protocol/CMakeLists.txt b/src/cryptonote_protocol/CMakeLists.txt new file mode 100644 index 00000000..2ea5662a --- /dev/null +++ b/src/cryptonote_protocol/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright (c) 2014, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +cmake_minimum_required (VERSION 2.6) +project (bitmonero CXX) + +file(GLOB CRYPTONOTE_PROTOCOL *) +source_group(cryptonote_protocol FILES ${CRYPTONOTE_PROTOCOL}) + +#add_library(p2p ${P2P}) + +#bitmonero_private_headers(p2p ${CRYPTONOTE_PROTOCOL}) +bitmonero_add_library(cryptonote_protocol ${CRYPTONOTE_PROTOCOL}) +#target_link_libraries(p2p) +# LINK_PRIVATE +# ${Boost_CHRONO_LIBRARY} +# ${Boost_REGEX_LIBRARY} +# ${Boost_SYSTEM_LIBRARY} +# ${Boost_THREAD_LIBRARY} +# ${EXTRA_LIBRARIES}) +add_dependencies(cryptonote_protocol + version) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp new file mode 100644 index 00000000..6b25cb68 --- /dev/null +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -0,0 +1,266 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief This is the place to implement our handlers for protocol network actions, e.g. for ratelimit for download-requests + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include +#include +#include + + +#include +#include + +#include "../../src/cryptonote_protocol/cryptonote_protocol_handler.h" +#include "../../src/p2p/network_throttle.hpp" + +#include "../../contrib/otshell_utils/utils.hpp" +using namespace nOT::nUtils; + +#include "../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal() + +// ################################################################################################ +// ################################################################################################ +// the "header part". Not separeted out for .hpp because point of this modification is +// to rebuild just 1 translation unit while working on this code. +// (But maybe common parts will be separated out later though - if needed) +// ################################################################################################ +// ################################################################################################ + +namespace cryptonote { + +class cryptonote_protocol_handler_base_pimpl { // placeholer if needed + public: + +}; + +} // namespace + +// ################################################################################################ +// ################################################################################################ +// ################################################################################################ +// ################################################################################################ + +namespace cryptonote { + +double cryptonote_protocol_handler_base::estimate_one_block_size() noexcept { // for estimating size of blocks to downloa + const double size_min = 500; // XXX 500 + const int history_len = 20; // how many blocks to average over + + double avg=0; + try { + avg = get_avg_block_size(history_len); + } catch (...) { } + avg = std::max( size_min , avg); + return avg; +} + +cryptonote_protocol_handler_base::cryptonote_protocol_handler_base() { +} + +cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() { +} + +void cryptonote_protocol_handler_base::handler_request_blocks_now(size_t &count_limit) { + using namespace epee::net_utils; + size_t est_req_size=0; // how much data are we now requesting (to be soon send to us) + + const auto count_limit_default = count_limit; + + bool allowed_now = false; // are we now allowed to request or are we limited still + // long int size_limit; + + while (!allowed_now) { + /* if ( ::cryptonote::core::get_is_stopping() ) { // TODO fast exit + _fact("ABORT sleep (before sending requeset) due to stopping"); + break; + }*/ + + //LOG_PRINT_RED("[DBG]" << get_avg_block_size(1), LOG_LEVEL_0); + //{ + long int size_limit1=0, size_limit2=0; + //LOG_PRINT_RED("calculating REQUEST size:", LOG_LEVEL_0); + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + network_throttle_manager::get_global_throttle_in().tick(); + size_limit1 = network_throttle_manager::get_global_throttle_in().get_recommended_size_of_planned_transport(); + } + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); + network_throttle_manager::get_global_throttle_inreq().tick(); + size_limit2 = network_throttle_manager::get_global_throttle_inreq().get_recommended_size_of_planned_transport(); + } + + long int one_block_estimated_size = estimate_one_block_size(); + long int limit_small = std::min( size_limit1 , size_limit2 ); + long int size_limit = limit_small/3 + size_limit1/3 + size_limit2/3; + if (limit_small <= 0) size_limit = 0; + const double estimated_peers = 1.2; // how many peers/threads we want to talk to, in order to not grab entire b/w by 1 thread + const double knob = 1.000; + size_limit /= (estimated_peers / estimated_peers) * knob; + _note_c("net/req-calc" , "calculating REQUEST size:" << size_limit1 << " " << size_limit2 << " small=" << limit_small << " final size_limit="<& ids) { + using namespace epee::net_utils; + LOG_PRINT_L0("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); + LOG_PRINT_RED("RATE LIMIT NOT IMPLEMENTED HERE YET (download at unlimited speed?)" , LOG_LEVEL_0); + _note_c("net/req2", "### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); + // TODO +} + +void cryptonote_protocol_handler_base::handler_response_blocks_now(size_t packet_size) { _scope_mark(""); + using namespace epee::net_utils; + double delay=0; // will be calculated + _dbg1("Packet size: " << packet_size); + do + { // rate limiting + //XXX + /*if (::cryptonote::core::get_is_stopping()) { + _dbg1("We are stopping - so abort sleep"); + return; + }*/ + /*if (m_was_shutdown) { + _dbg2_c("net/netuse/sleep","m_was_shutdown - so abort sleep"); + return; + }*/ + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global + } + + + delay *= 0.50; + //delay = 0; // XXX + if (delay > 0) { + //delay += rand2*0.1; + long int ms = (long int)(delay * 1000); + _info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="< 0); + +// XXX LATER XXX + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().handle_trafic_tcp( packet_size ); // increase counter - global + //epee::critical_region_t guard(m_throttle_global_lock); // *** critical *** + //m_throttle_global.m_out.handle_trafic_tcp( packet_size ); // increase counter - global + } +} + +} // namespace + + diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index a3b79856..b3393928 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -1,3 +1,7 @@ +/// @file +/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) +/// @brief This is the orginal cryptonote protocol network-events handler, modified by us + // Copyright (c) 2014-2015, The Monero Project // // All rights reserved. @@ -41,6 +45,7 @@ #include "cryptonote_core/connection_context.h" #include "cryptonote_core/cryptonote_stat_info.h" #include "cryptonote_core/verification_context.h" +#include PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) @@ -48,8 +53,26 @@ DISABLE_VS_WARNINGS(4355) namespace cryptonote { + class cryptonote_protocol_handler_base_pimpl; + class cryptonote_protocol_handler_base { + private: + std::unique_ptr mI; + + public: + cryptonote_protocol_handler_base(); + virtual ~cryptonote_protocol_handler_base(); + void handler_request_blocks_now(size_t & count_limit); // before asking for blocks, can adjust the limit of download + void handler_request_blocks_history(std::list& ids); // before asking for list of objects, we can change the list still + void handler_response_blocks_now(size_t packet_size); + + virtual double get_avg_block_size( size_t count) const = 0; + virtual double estimate_one_block_size() noexcept; // for estimating size of blocks to download + + virtual std::ofstream& get_logreq() const =0; + }; + template - class t_cryptonote_protocol_handler: public i_cryptonote_protocol + class t_cryptonote_protocol_handler: public i_cryptonote_protocol, cryptonote_protocol_handler_base { public: typedef cryptonote_connection_context connection_context; @@ -107,12 +130,17 @@ namespace cryptonote std::atomic m_syncronized_connections_count; std::atomic m_synchronized; + // static std::ofstream m_logreq; + + double get_avg_block_size(size_t count) const; + template bool post_notify(typename t_parametr::request& arg, cryptonote_connection_context& context) { LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parametr).name() << " -->"); std::string blob; epee::serialization::store_t_to_binary(arg, blob); + //handler_response_blocks_now(blob.size()); // XXX return m_p2p->invoke_notify_to_peer(t_parametr::ID, blob, context); } @@ -124,8 +152,11 @@ namespace cryptonote epee::serialization::store_t_to_binary(arg, arg_buff); return m_p2p->relay_notify_to_all(t_parametr::ID, arg_buff, exlude_context); } + + virtual std::ofstream& get_logreq() const ; }; -} + +} // namespace #include "cryptonote_protocol_handler.inl" diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 2754eb73..aebfcd10 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1,3 +1,7 @@ +/// @file +/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) +/// @brief This is the orginal cryptonote protocol network-events handler, modified by us + // Copyright (c) 2014-2015, The Monero Project // // All rights reserved. @@ -28,14 +32,26 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +// (may contain code and/or modifications by other developers) +// developer rfree: this code is caller of our new network code, and is modded; e.g. for rate limiting + #include #include #include "cryptonote_core/cryptonote_format_utils.h" #include "profile_tools.h" +#include "../../contrib/otshell_utils/utils.hpp" +using namespace nOT::nUtils; + namespace cryptonote { + +// static +// template std::ofstream t_cryptonote_protocol_handler::m_logreq("logreq.txt"); // static + + + //----------------------------------------------------------------------------------------------------------------------- template t_cryptonote_protocol_handler::t_cryptonote_protocol_handler(t_core& rcore, nodetool::i_p2p_endpoint* p_net_layout):m_core(rcore), @@ -100,20 +116,24 @@ namespace cryptonote { std::stringstream ss; - ss << std::setw(25) << std::left << "Remote Host" + ss << std::setw(30) << std::left << "Remote Host" << std::setw(20) << "Peer id" << std::setw(25) << "Recv/Sent (inactive,sec)" << std::setw(25) << "State" << std::setw(20) << "Livetime(seconds)" << ENDL; + uint32_t ip; m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id) { - ss << std::setw(25) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") + - epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port) + ip = ntohl(cntxt.m_remote_ip); + ss << std::setw(30) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") + + epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port) << std::setw(20) << std::hex << peer_id << std::setw(25) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" << std::setw(25) << get_protocol_state_string(cntxt.m_state) - << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) << ENDL; + << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) + << std::setw(10) << (ip > 3232235520 && ip < 3232301055 ? " [LAN]" : "") //TODO: local ip in calss A, B + << ENDL; return true; }); LOG_PRINT_L0("Connections: " << ENDL << ss.str()); @@ -234,11 +254,11 @@ namespace cryptonote block_verification_context bvc = boost::value_initialized(); m_core.pause_mine(); - m_core.handle_incoming_block(arg.b.block, bvc); + m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block m_core.resume_mine(); if(bvc.m_verifivation_failed) { - LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection"); + LOG_PRINT_CCONTEXT_L0("Block verification failed, dropping connection"); m_p2p->drop_connection(context); return 1; } @@ -304,9 +324,19 @@ namespace cryptonote LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() << ", rsp.m_current_blockchain_height=" << rsp.current_blockchain_height << ", missed_ids.size()=" << rsp.missed_ids.size()); post_notify(rsp, context); + //handler_response_blocks_now(sizeof(rsp)); // XXX + //handler_response_blocks_now(200); return 1; } //------------------------------------------------------------------------------------------------------------------------ + + + template + double t_cryptonote_protocol_handler::get_avg_block_size( size_t count) const { + return m_core.get_blockchain_storage().get_avg_block_size(count); + } + + template int t_cryptonote_protocol_handler::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { @@ -378,6 +408,7 @@ namespace cryptonote epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler( boost::bind(&t_core::resume_mine, &m_core)); + LOG_PRINT_CCONTEXT_YELLOW( "Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() , LOG_LEVEL_0); BOOST_FOREACH(const block_complete_entry& block_entry, arg.blocks) { //process transactions @@ -419,7 +450,8 @@ namespace cryptonote LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms"); } } - + size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; + handler_request_blocks_now(count_limit); // XXX request_missing_objects(context, true); return 1; } @@ -455,6 +487,11 @@ namespace cryptonote size_t count = 0; auto it = context.m_needed_objects.begin(); + size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; + //handler_request_blocks_now( count_limit ); // change the limit, sleep(?) XXX + // XXX + count_limit=200; // XXX + _note_c("net/req-calc" , "Setting count_limit: " << count_limit); while(it != context.m_needed_objects.end() && count < BLOCKS_SYNCHRONIZING_DEFAULT_COUNT) { if( !(check_having_blocks && m_core.have_block(*it))) @@ -465,14 +502,16 @@ namespace cryptonote } context.m_needed_objects.erase(it++); } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size()); + LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() + << "requested blocks count=" << count << " / " << count_limit); post_notify(req, context); }else if(context.m_last_response_height < context.m_remote_blockchain_height-1) {//we have to fetch more objects ids, request blockchain entry NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify(r, context); }else { @@ -575,4 +614,18 @@ namespace cryptonote { return relay_post_notify(arg, exclude_context); } -} + + /// @deprecated + template std::ofstream& t_cryptonote_protocol_handler::get_logreq() const { + static std::ofstream * logreq=NULL; + if (!logreq) { + LOG_PRINT_RED("LOG OPENED",LOG_LEVEL_0); + logreq = new std::ofstream("logreq.txt"); // leak mem (singleton) + *logreq << "Opened log" << std::endl; + } + LOG_PRINT_YELLOW("LOG USED",LOG_LEVEL_0); + (*logreq) << "log used" << std::endl; + return *logreq; + } + +} // namespace diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 5f60857d..4a0dcb14 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -32,23 +32,7 @@ set(daemon_sources set(daemon_headers) set(daemon_private_headers - daemon_commands_handler.h - - # cryptonote_protocol - ../cryptonote_protocol/blobdatatype.h - ../cryptonote_protocol/cryptonote_protocol_defs.h - ../cryptonote_protocol/cryptonote_protocol_handler.h - ../cryptonote_protocol/cryptonote_protocol_handler.inl - ../cryptonote_protocol/cryptonote_protocol_handler_common.h - - # p2p - ../p2p/net_node.h - ../p2p/net_node.inl - ../p2p/net_node_common.h - ../p2p/net_peerlist.h - ../p2p/net_peerlist_boost_serialization.h - ../p2p/p2p_protocol_defs.h - ../p2p/stdafx.h) + daemon_commands_handler.h) bitmonero_private_headers(daemon ${daemon_private_headers}) @@ -62,6 +46,9 @@ target_link_libraries(daemon cryptonote_core crypto common + otshell_utils + p2p + cryptonote_protocol ${Boost_CHRONO_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} diff --git a/src/daemon/daemon_commands_handler.h b/src/daemon/daemon_commands_handler.h index 10a07cf8..7cba4ec5 100644 --- a/src/daemon/daemon_commands_handler.h +++ b/src/daemon/daemon_commands_handler.h @@ -40,6 +40,7 @@ #include "common/util.h" #include "crypto/hash.h" #include "version.h" +#include "../../contrib/otshell_utils/utils.hpp" /*! * \brief I don't really know right now @@ -74,6 +75,10 @@ public: m_cmd_binder.set_handler("save", boost::bind(&daemon_cmmands_handler::save, this, _1), "Save blockchain"); m_cmd_binder.set_handler("set_log", boost::bind(&daemon_cmmands_handler::set_log, this, _1), "set_log - Change current log detalization level, is a number 0-4"); m_cmd_binder.set_handler("diff", boost::bind(&daemon_cmmands_handler::diff, this, _1), "Show difficulty"); + m_cmd_binder.set_handler("out_peers", boost::bind(&daemon_cmmands_handler::out_peers_limit, this, _1), "Set max limit of out peers"); + m_cmd_binder.set_handler("limit_up", boost::bind(&daemon_cmmands_handler::limit_up, this, _1), "Set upload limit [kB/s]"); + m_cmd_binder.set_handler("limit_down", boost::bind(&daemon_cmmands_handler::limit_down, this, _1), "Set download limit [kB/s]"); + m_cmd_binder.set_handler("limit", boost::bind(&daemon_cmmands_handler::limit, this, _1), "Set download and upload limit [kB/s]"); } bool start_handling() @@ -398,4 +403,128 @@ private: m_srv.get_payload_object().get_core().get_miner().stop(); return true; } + //-------------------------------------------------------------------------------- + bool out_peers_limit(const std::vector& args) { + if(args.size()!=1) { + std::cout << "Usage: limit_down " << ENDL; + return true; + } + + unsigned int limit; + try { + limit = std::stoi(args[0]); + } + + catch(std::invalid_argument& ex) { + _erro("stoi exception"); + return false; + } + + if (m_srv.m_config.m_net_config.connections_count > limit) + { + int count = m_srv.m_config.m_net_config.connections_count - limit; + m_srv.m_config.m_net_config.connections_count = limit; + m_srv.delete_connections(count); + } + else + m_srv.m_config.m_net_config.connections_count = limit; + + return true; + } + //-------------------------------------------------------------------------------- + bool limit_up(const std::vector& args) + { + if(args.size()!=1) { + std::cout << "Usage: limit_up " << ENDL; + return false; + } + + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + + if (limit==-1) { + limit=128; + //this->islimitup=false; + } + + limit *= 1024; + + + //nodetool::epee::net_utils::connection >::set_rate_up_limit( limit ); + epee::net_utils::connection_basic::set_rate_up_limit( limit ); + std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; + + return true; + } + + //-------------------------------------------------------------------------------- + bool limit_down(const std::vector& args) + { + + if(args.size()!=1) { + std::cout << "Usage: limit_down " << ENDL; + return true; + } + + int limit; + try { + limit = std::stoi(args[0]); + } + + catch(std::invalid_argument& ex) { + return false; + } + + if (limit==-1) { + limit=128; + //this->islimitup=false; + } + + limit *= 1024; + + + //nodetool::epee::net_utils::connection >::set_rate_up_limit( limit ); + epee::net_utils::connection_basic::set_rate_down_limit( limit ); + std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; + + return true; + } + + //-------------------------------------------------------------------------------- + bool limit(const std::vector& args) + { + if(args.size()!=1) { + std::cout << "Usage: limit_down " << ENDL; + return true; + } + + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + + if (limit==-1) { + limit=128; + //this->islimitup=false; + } + + limit *= 1024; + + + //nodetool::epee::net_utils::connection >::set_rate_up_limit( limit ); + epee::net_utils::connection_basic::set_rate_down_limit( limit ); + epee::net_utils::connection_basic::set_rate_up_limit( limit ); + std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; + std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; + + return true; + } }; diff --git a/src/p2p/CMakeLists.txt b/src/p2p/CMakeLists.txt new file mode 100644 index 00000000..541b90fa --- /dev/null +++ b/src/p2p/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright (c) 2014, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +cmake_minimum_required (VERSION 2.6) +project (bitmonero CXX) + +file(GLOB P2P *) +source_group(p2p FILES ${P2P}) + +#add_library(p2p ${P2P}) + +#bitmonero_private_headers(p2p ${P2P}) +bitmonero_add_library(p2p ${P2P}) +#target_link_libraries(p2p) +# LINK_PRIVATE +# ${Boost_CHRONO_LIBRARY} +# ${Boost_REGEX_LIBRARY} +# ${Boost_SYSTEM_LIBRARY} +# ${Boost_THREAD_LIBRARY} +# ${EXTRA_LIBRARIES}) +add_dependencies(p2p + version) diff --git a/src/p2p/connection_basic.cpp b/src/p2p/connection_basic.cpp new file mode 100644 index 00000000..35b0d4c8 --- /dev/null +++ b/src/p2p/connection_basic.cpp @@ -0,0 +1,362 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief base for connection, contains e.g. the ratelimit hooks + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: implementation for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */ + +#include "connection_basic.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include +#include +#include +#include +#include + +#include +#include +#include "../../contrib/epee/include/net/abstract_tcp_server2.h" + +#include "../../contrib/otshell_utils/utils.hpp" +using namespace nOT::nUtils; + +// TODO: +#include "../../src/p2p/network_throttle-detail.hpp" +#include "../../src/cryptonote_core/cryptonote_core.h" + +// ################################################################################################ +// local (TU local) headers +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + + +/* ============================================================================ */ + +class connection_basic_pimpl { + public: + connection_basic_pimpl(const std::string &name); + + static int m_default_tos; + + network_throttle_bw m_throttle; // per-perr + critical_section m_throttle_lock; + + int m_peer_number; // e.g. for debug/stats +}; + + +} // namespace +} // namespace + +// ################################################################################################ +// The implementation part +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// connection_basic_pimpl +// ================================================================================================ + +connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_throttle(name) { } + +// ================================================================================================ +// connection_basic +// ================================================================================================ + +// static variables: +int connection_basic_pimpl::m_default_tos; + +// methods: +connection_basic::connection_basic(boost::asio::io_service& io_service, std::atomic &ref_sock_count, std::atomic &sock_number) + : + mI( new connection_basic_pimpl("peer") ), + strand_(io_service), + socket_(io_service), + m_want_close_connection(false), + m_was_shutdown(false), + m_ref_sock_count(ref_sock_count) +{ + ++ref_sock_count; // increase the global counter + mI->m_peer_number = sock_number.fetch_add(1); // use, and increase the generated number + _note("Spawned connection p2p#"<m_peer_number<<" currently we have sockets count:" << m_ref_sock_count); + boost::filesystem::create_directories("log/dr-monero/net/"); + /*boost::asio::SettableSocketOption option;// = new boost::asio::SettableSocketOption(); + option.level(IPPROTO_IP); + option.name(IP_TOS); + option.value(&tos); + option.size = sizeof(tos); + socket_.set_option(option);*/ + // TODO socket options +} + +connection_basic::~connection_basic() { + _note("Destructing connection p2p#"<m_peer_number); +} + +void connection_basic::set_rate_up_limit(uint64_t limit) { + save_limit_to_file(limit); + { + // TODO remove __SCALING_FACTOR... + const double SCALING_FACTOR = 2.25; // to acheve the best performance + limit *= SCALING_FACTOR; + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().set_target_speed(limit); + } + // connection_basic_pimpl::m_throttle_global.m_out.set_target_speed(limit); +} + +void connection_basic::set_rate_down_limit(uint64_t limit) { + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + network_throttle_manager::get_global_throttle_in().set_target_speed(limit); + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); + network_throttle_manager::get_global_throttle_inreq().set_target_speed(limit); + } + save_limit_to_file(limit); +} + +void connection_basic::set_rate_limit(uint64_t limit) { + // TODO +} +void connection_basic::set_kill_limit (uint64_t limit) { + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + network_throttle_manager::get_global_throttle_in().set_target_kill(limit); + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().set_target_kill(limit); + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); + network_throttle_manager::get_global_throttle_inreq().set_target_kill(limit); + } +} + +void connection_basic::save_limit_to_file(int limit) { + // saving limit to file + std::ofstream file; + file.open("log/dr-monero/limit.info"); + file << limit; +} + +void connection_basic::set_rate_autodetect(uint64_t limit) { + // TODO + LOG_PRINT_L0("inside connection_basic we set autodetect (this is additional notification).."); +} + +void connection_basic::set_tos_flag(int tos) { + connection_basic_pimpl::m_default_tos = tos; +} + +int connection_basic::get_tos_flag() { + return connection_basic_pimpl::m_default_tos; +} + +void connection_basic::sleep_before_packet(size_t packet_size, int phase, int q_len) { + double delay=0; // will be calculated + do + { // rate limiting + //XXX + /*if (::cryptonote::core::get_is_stopping()) { + _dbg1("We are stopping - so abort sleep"); + return; + }*/ + if (m_was_shutdown) { + _dbg2("m_was_shutdown - so abort sleep"); + return; + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global + } + + + delay *= 0.50; + delay = 0; // XXX + if (delay > 0) { + //delay += rand2*0.1; + long int ms = (long int)(delay * 1000); + _info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="< 0); + +// XXX LATER XXX + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().handle_trafic_tcp( packet_size ); // increase counter - global + //epee::critical_region_t guard(m_throttle_global_lock); // *** critical *** + //m_throttle_global.m_out.handle_trafic_tcp( packet_size ); // increase counter - global + } + +} +void connection_basic::set_start_time() { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + m_start_time = network_throttle_manager::get_global_throttle_out().get_time_seconds(); +} + +void connection_basic::do_send_handler_start(const void* ptr , size_t cb ) { + _fact_c("net/out/size", "*** do_sen() called for packet="< max sending time + //if (sending_time > 0.1) network_throttle_manager::get_global_throttle_out().set_overheat(sending_time); // TODO + +} + +void connection_basic::do_send_handler_write_from_queue( const boost::system::error_code& e, size_t cb, int q_len ) { + sleep_before_packet(cb,2,q_len); + _info_c("net/out/size", "handler_write (after write, from queue="<m_throttle_global_lock)> guard(mI->m_throttle_global_lock); // *** critical *** + // mI->m_throttle_global.m_in.handle_trafic_tcp( packet_size ); // increase counter - global + } +} + +void connection_basic::logger_handle_net_peer(size_t size, bool io) { // network data written + // TODO OPTIMIZE! do NOT reopen idiotically :) + std::ostringstream oss; + std::string filename; + if (io) { // write + double time = network_throttle_manager::get_global_throttle_in().get_time_seconds() ; + oss << "log/dr-monero/net/in-peer-" << (mI->m_peer_number) << ".dat" << std::ends; + filename = oss.str(); + network_throttle_manager::get_global_throttle_out().logger_handle_net(filename,time,size); + } + else { // read + double time = network_throttle_manager::get_global_throttle_out().get_time_seconds() ; + oss << "log/dr-monero/net/out-peer-" << (mI->m_peer_number) << ".dat" << std::ends; + filename = oss.str(); + network_throttle_manager::get_global_throttle_in().logger_handle_net(filename,time,size); + } +} + +void connection_basic::logger_handle_net_read(size_t size) { // network data read + std::string filename = "log/dr-monero/net/in-all.data"; + + double time = network_throttle_manager::get_global_throttle_in().get_time_seconds() ; + network_throttle_manager::get_global_throttle_in().logger_handle_net(filename, time, size); + logger_handle_net_peer(size,0); +} + +void connection_basic::logger_handle_net_write(size_t size) { + std::string filename = "log/dr-monero/net/out-all.data"; + double time = network_throttle_manager::get_global_throttle_out().get_time_seconds() ; + network_throttle_manager::get_global_throttle_out().logger_handle_net(filename, time, size); + logger_handle_net_peer(size,1); + +} + +double connection_basic::get_sleep_time(size_t cb) { + auto t = network_throttle_manager::get_global_throttle_out().get_sleep_time(cb); + return t; +} + + +} // namespace +} // namespace + diff --git a/src/p2p/connection_basic.hpp b/src/p2p/connection_basic.hpp new file mode 100644 index 00000000..1b5a2c8a --- /dev/null +++ b/src/p2p/connection_basic.hpp @@ -0,0 +1,139 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief base for connection, contains e.g. the ratelimit hooks + +// ! This file might contain variable names same as in template class connection<> +// ! from files contrib/epee/include/net/abstract_tcp_server2.* +// ! I am not a lawyer; afaik APIs, var names etc are not copyrightable ;) +// ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part) +// ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as: + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +/* rfree: place for hanlers for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */ + +#ifndef INCLUDED_p2p_connection_basic_hpp +#define INCLUDED_p2p_connection_basic_hpp + + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/syncobj.h" + +namespace epee +{ +namespace net_utils +{ + + /************************************************************************/ + /* */ + /************************************************************************/ + /// Represents a single connection from a client. + +class connection_basic_pimpl; // PIMPL for this class + +class connection_basic { // not-templated base class for rapid developmet of some code parts + public: + std::unique_ptr< connection_basic_pimpl > mI; // my Implementation + + // moved here from orginal connecton<> - common member variables that do not depend on template in connection<> + volatile uint32_t m_want_close_connection; + std::atomic m_was_shutdown; + critical_section m_send_que_lock; + std::list m_send_que; + volatile bool m_is_multithreaded; + double m_start_time; + /// Strand to ensure the connection's handlers are not called concurrently. + boost::asio::io_service::strand strand_; + /// Socket for the connection. + boost::asio::ip::tcp::socket socket_; + + std::atomic &m_ref_sock_count; // reference to external counter of existing sockets that we will ++/-- + public: + // first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator + connection_basic(boost::asio::io_service& io_service, std::atomic &ref_sock_count, std::atomic &sock_number); + + virtual ~connection_basic(); + + // various handlers to be called from connection class: + void do_send_handler_start(const void * ptr , size_t cb); + void do_send_handler_delayed(const void * ptr , size_t cb); + void do_send_handler_write(const void * ptr , size_t cb); + void do_send_handler_stop(const void * ptr , size_t cb); + void do_send_handler_after_write( const boost::system::error_code& e, size_t cb ); // from handle_write + void do_send_handler_write_from_queue(const boost::system::error_code& e, size_t cb , int q_len); // from handle_write, sending next part + void do_read_handler_start(const boost::system::error_code& e, std::size_t bytes_transferred); // from read, after read completion + + void logger_handle_net_write(size_t size); // network data written + void logger_handle_net_read(size_t size); // network data read + void logger_handle_net_peer(size_t size, bool io); + + void set_start_time(); + + // config for rate limit + + static void set_rate_up_limit(uint64_t limit); + static void set_rate_down_limit(uint64_t limit); + static void set_rate_limit(uint64_t limit); + static void set_rate_autodetect(uint64_t limit); + static void set_kill_limit (uint64_t limit); + + // config misc + static void set_tos_flag(int tos); // ToS / QoS flag + static int get_tos_flag(); + + // handlers and sleep + void sleep_before_packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) + static void save_limit_to_file(int limit); ///< for dr-monero + static double get_sleep_time(size_t cb); +}; + +} // nameserver +} // nameserver + +#endif + + diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 97fcd56c..48737193 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -80,14 +80,12 @@ namespace nodetool public: typedef t_payload_net_handler payload_net_handler; - node_server( - t_payload_net_handler& payload_handler - , boost::uuids::uuid network_id - ) - : m_payload_handler(payload_handler) - , m_allow_local_ip(false) - , m_hide_my_port(false) - , m_network_id(std::move(network_id)) + node_server(t_payload_net_handler& payload_handler, boost::uuids::uuid network_id) + :m_payload_handler(payload_handler), + m_allow_local_ip(false), + m_no_igd(false), + m_hide_my_port(false), + m_network_id(std::move(network_id)) {} static void init_options(boost::program_options::options_description& desc); @@ -111,6 +109,7 @@ namespace nodetool virtual uint64_t get_connections_count(); size_t get_outgoing_connections_count(); peerlist_manager& get_peerlist_manager(){return m_peerlist;} + void delete_connections(size_t count); private: const std::vector m_seed_nodes_list = { "seeds.moneroseeds.se" @@ -118,6 +117,9 @@ namespace nodetool , "seeds.moneroseeds.ch" , "seeds.moneroseeds.li" }; + + bool islimitup=false; + bool islimitdown=false; typedef COMMAND_REQUEST_STAT_INFO_T COMMAND_REQUEST_STAT_INFO; @@ -197,6 +199,13 @@ namespace nodetool template bool parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor > & arg, Container& container); + bool set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max); + bool set_tos_flag(const boost::program_options::variables_map& vm, int limit); + + bool set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit); + bool set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit); + bool set_rate_limit(const boost::program_options::variables_map& vm, uint64_t limit); + //debug functions std::string print_connections_container(); @@ -214,7 +223,10 @@ namespace nodetool END_KV_SERIALIZE_MAP() }; - config m_config; + public: + config m_config; // TODO was private, add getters? + + private: std::string m_config_folder; bool m_have_address; @@ -224,6 +236,7 @@ namespace nodetool uint32_t m_ip_address; bool m_allow_local_ip; bool m_hide_my_port; + bool m_no_igd; //critical_section m_connections_lock; //connections_indexed_container m_connections; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index ee4a1078..ce70e241 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -84,6 +84,14 @@ namespace nodetool " If this option is given the options add-priority-node and seed-node are ignored"}; const command_line::arg_descriptor > arg_p2p_seed_node = {"seed-node", "Connect to a node to retrieve peer addresses, and disconnect"}; const command_line::arg_descriptor arg_p2p_hide_my_port = {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true}; + + const command_line::arg_descriptor arg_no_igd = {"no-igd", "Disable UPnP port mapping"}; + const command_line::arg_descriptor arg_out_peers = {"out-peers", "set max limit of out peers", -1}; + const command_line::arg_descriptor arg_tos_flag = {"tos-flag", "set TOS flag", -1}; + + const command_line::arg_descriptor arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", -1}; + const command_line::arg_descriptor arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", -1}; + const command_line::arg_descriptor arg_limit_rate = {"limit-rate", "set limit-rate [kB/s]", 128}; } //----------------------------------------------------------------------------------- @@ -99,7 +107,13 @@ namespace nodetool command_line::add_arg(desc, arg_p2p_add_priority_node); command_line::add_arg(desc, arg_p2p_add_exclusive_node); command_line::add_arg(desc, arg_p2p_seed_node); - command_line::add_arg(desc, arg_p2p_hide_my_port); } + command_line::add_arg(desc, arg_p2p_hide_my_port); + command_line::add_arg(desc, arg_no_igd); + command_line::add_arg(desc, arg_out_peers); + command_line::add_arg(desc, arg_tos_flag); + command_line::add_arg(desc, arg_limit_rate_up); + command_line::add_arg(desc, arg_limit_rate_down); + command_line::add_arg(desc, arg_limit_rate); } //----------------------------------------------------------------------------------- template bool node_server::init_config() @@ -120,7 +134,6 @@ namespace nodetool //at this moment we have hardcoded config m_config.m_net_config.handshake_interval = P2P_DEFAULT_HANDSHAKE_INTERVAL; - m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT; m_config.m_net_config.packet_max_size = P2P_DEFAULT_PACKET_MAX_SIZE; //20 MB limit m_config.m_net_config.config_id = 0; // initial config m_config.m_net_config.connection_timeout = P2P_DEFAULT_CONNECTION_TIMEOUT; @@ -165,6 +178,7 @@ namespace nodetool m_port = command_line::get_arg(vm, p2p_bind_arg); m_external_port = command_line::get_arg(vm, arg_p2p_external_port); m_allow_local_ip = command_line::get_arg(vm, arg_p2p_allow_local_ip); + m_no_igd = command_line::get_arg(vm, arg_no_igd); if (command_line::has_arg(vm, arg_p2p_add_peer)) { @@ -184,11 +198,13 @@ namespace nodetool if (!parse_peers_and_add_to_container(vm, arg_p2p_add_exclusive_node, m_exclusive_peers)) return false; } + if (command_line::has_arg(vm, arg_p2p_add_priority_node)) { if (!parse_peers_and_add_to_container(vm, arg_p2p_add_priority_node, m_priority_peers)) return false; } + if (command_line::has_arg(vm, arg_p2p_seed_node)) { if (!parse_peers_and_add_to_container(vm, arg_p2p_seed_node, m_seed_nodes)) @@ -197,6 +213,21 @@ namespace nodetool if(command_line::has_arg(vm, arg_p2p_hide_my_port)) m_hide_my_port = true; + + if ( !set_max_out_peers(vm, command_line::get_arg(vm, arg_out_peers) ) ) + return false; + + if ( !set_tos_flag(vm, command_line::get_arg(vm, arg_tos_flag) ) ) + return false; + + if ( !set_rate_up_limit(vm, command_line::get_arg(vm, arg_limit_rate_up) ) ) + return false; + + if ( !set_rate_down_limit(vm, command_line::get_arg(vm, arg_limit_rate_down) ) ) + return false; + + if ( !set_rate_limit(vm, command_line::get_arg(vm, arg_limit_rate) ) ) + return false; return true; } @@ -375,42 +406,43 @@ namespace nodetool LOG_PRINT_L0("External port defined as " << m_external_port); // Add UPnP port mapping - LOG_PRINT_L0("Attempting to add IGD port mapping."); - int result; - UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, &result); - UPNPUrls urls; - IGDdatas igdData; - char lanAddress[64]; - result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); - freeUPNPDevlist(deviceList); - if (result != 0) { - if (result == 1) { - std::ostringstream portString; - portString << m_listenning_port; - - // Delete the port mapping before we create it, just in case we have dangling port mapping from the daemon not being shut down correctly - UPNP_DeletePortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), "TCP", 0); + if(m_no_igd == false) { + LOG_PRINT_L0("Attempting to add IGD port mapping."); + int result; + UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, &result); + UPNPUrls urls; + IGDdatas igdData; + char lanAddress[64]; + result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); + freeUPNPDevlist(deviceList); + if (result != 0) { + if (result == 1) { + std::ostringstream portString; + portString << m_listenning_port; + + // Delete the port mapping before we create it, just in case we have dangling port mapping from the daemon not being shut down correctly + UPNP_DeletePortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), "TCP", 0); - int portMappingResult; - portMappingResult = UPNP_AddPortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), portString.str().c_str(), lanAddress, CRYPTONOTE_NAME, "TCP", 0, "0"); - if (portMappingResult != 0) { - LOG_ERROR("UPNP_AddPortMapping failed, error: " << strupnperror(portMappingResult)); - } else { - LOG_PRINT_GREEN("Added IGD port mapping.", LOG_LEVEL_0); - } - } else if (result == 2) { - LOG_PRINT_L0("IGD was found but reported as not connected."); - } else if (result == 3) { - LOG_PRINT_L0("UPnP device was found but not recoginzed as IGD."); - } else { - LOG_ERROR("UPNP_GetValidIGD returned an unknown result code."); - } - - FreeUPNPUrls(&urls); - } else { - LOG_PRINT_L0("No IGD was found."); - } + int portMappingResult; + portMappingResult = UPNP_AddPortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), portString.str().c_str(), lanAddress, CRYPTONOTE_NAME, "TCP", 0, "0"); + if (portMappingResult != 0) { + LOG_ERROR("UPNP_AddPortMapping failed, error: " << strupnperror(portMappingResult)); + } else { + LOG_PRINT_GREEN("Added IGD port mapping.", LOG_LEVEL_0); + } + } else if (result == 2) { + LOG_PRINT_L0("IGD was found but reported as not connected."); + } else if (result == 3) { + LOG_PRINT_L0("UPnP device was found but not recoginzed as IGD."); + } else { + LOG_ERROR("UPNP_GetValidIGD returned an unknown result code."); + } + FreeUPNPUrls(&urls); + } else { + LOG_PRINT_L0("No IGD was found."); + } + } return res; } //----------------------------------------------------------------------------------- @@ -1300,4 +1332,83 @@ namespace nodetool return true; } + + template + bool node_server::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max) + { + if(max == -1) { + m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT; + return true; + } + + m_config.m_net_config.connections_count = max; + LOG_PRINT_RED_L0("connections_count: " << m_config.m_net_config.connections_count); + return true; + } + + template + void node_server::delete_connections(size_t count) + { + m_net_server.get_config_object().del_connections(count); + } + + template + bool node_server::set_tos_flag(const boost::program_options::variables_map& vm, int flag) + { + if(flag==-1){ + return true; + } + epee::net_utils::connection >::set_tos_flag(flag); + _dbg1("Set ToS flag " << flag); + return true; + } + + template + bool node_server::set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit) + { + this->islimitup=true; + + if (limit==-1) { + limit=128; + this->islimitup=false; + } + + limit *= 1024; + epee::net_utils::connection >::set_rate_up_limit( limit ); + LOG_PRINT_L0("Set limit-up to " << limit/1024 << " kB/s"); + return true; + } + + template + bool node_server::set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit) + { + this->islimitdown=true; + if(limit==-1) { + limit=128; + this->islimitdown=false; + } + limit *= 1024; + epee::net_utils::connection >::set_rate_down_limit( limit ); + LOG_PRINT_L0("Set limit-down to " << limit/1024 << " kB/s"); + return true; + } + + template + bool node_server::set_rate_limit(const boost::program_options::variables_map& vm, uint64_t limit) + { + limit *= 1024; + if(this->islimitdown==false && this->islimitup==false) { + epee::net_utils::connection >::set_rate_up_limit( limit ); + epee::net_utils::connection >::set_rate_down_limit( limit ); + LOG_PRINT_L0("Set limit to " << limit/1024 << " kB/s"); + } + else if(this->islimitdown==false && this->islimitup==true ) { + epee::net_utils::connection >::set_rate_down_limit( limit ); + } + else if(this->islimitdown==true && this->islimitup==false ) { + epee::net_utils::connection >::set_rate_up_limit( limit ); + } + + return true; + } } diff --git a/src/p2p/network_throttle-detail.cpp b/src/p2p/network_throttle-detail.cpp new file mode 100644 index 00000000..6ea3076a --- /dev/null +++ b/src/p2p/network_throttle-detail.cpp @@ -0,0 +1,382 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief implementaion for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: implementation for throttle details */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include +#include +#include + + + +#include +#include +#include "../../contrib/epee/include/net/abstract_tcp_server2.h" + +// TODO: +#include "../../src/p2p/network_throttle-detail.hpp" + +#include "../../contrib/otshell_utils/utils.hpp" +using namespace nOT::nUtils; + +// ################################################################################################ +// ################################################################################################ +// the "header part". Not separeted out for .hpp because point of this modification is +// to rebuild just 1 translation unit while working on this code. +// (But maybe common parts will be separated out later though - if needed) +// ################################################################################################ +// ################################################################################################ + +using namespace nOT::nUtils; + +namespace epee +{ +namespace net_utils +{ + + +/* ============================================================================ */ + +class connection_basic_pimpl { + public: + connection_basic_pimpl(const std::string &name); + + static int m_default_tos; + + network_throttle_bw m_throttle; // per-perr + critical_section m_throttle_lock; + + void _packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) could be used for different kinds of sleep e.g. direct/queue write +}; + + +} // namespace +} // namespace + + + + + + +// ################################################################################################ +// ################################################################################################ +// The implementation part +// ################################################################################################ +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// network_throttle +// ================================================================================================ + +network_throttle::~network_throttle() { } + +network_throttle::packet_info::packet_info() + : m_size(0) +{ +} + +network_throttle::network_throttle(const std::string &nameshort, const std::string &name, int window_size) + : m_window_size( (window_size==-1) ? 10 : window_size ), + m_history( m_window_size ), m_nameshort(nameshort) +{ + set_name(name); + m_network_add_cost = 128; + m_network_minimal_segment = 256; + m_network_max_segment = 1024*1024; + m_any_packet_yet = false; + m_slot_size = 1.0; // hard coded in few places + m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle + m_target_MB = 0; + +} + +void network_throttle::set_name(const std::string &name) +{ + m_name = name; +} + +void network_throttle::set_target_speed( network_speed_kbps target ) +{ + m_target_speed = target; + _note_c("net/"+m_nameshort, "Setting LIMIT: " << target << " kbps"); +} + +void network_throttle::set_target_kill( network_MB target ) +{ + _note_c("net/"+m_nameshort, "Setting KILL: " << target << " MB hard limit"); + m_target_MB = target; +} + + +void network_throttle::tick() +{ + double time_now = get_time_seconds(); + if (!m_any_packet_yet) m_start_time = time_now; // starting now + + network_time_seconds current_sample_time_slot = time_to_slot( time_now ); // T=13.7 --> 13 (for 1-second smallwindow) + network_time_seconds last_sample_time_slot = time_to_slot( m_last_sample_time ); + + // moving to next position, and filling gaps + // !! during this loop the m_last_sample_time and last_sample_time_slot mean the variable moved in +1 + // TODO optimize when moving few slots at once + while ( (!m_any_packet_yet) || (last_sample_time_slot < current_sample_time_slot)) + { + LOG_PRINT_L4("Moving counter buffer by 1 second " << last_sample_time_slot << " < " << current_sample_time_slot << " (last time " << m_last_sample_time<<")"); + // rotate buffer + for (size_t i=m_history.size()-1; i>=1; --i) m_history[i] = m_history[i-1]; + m_history[0] = packet_info(); + if (! m_any_packet_yet) + { + m_last_sample_time = time_now; + } + m_last_sample_time += 1; last_sample_time_slot = time_to_slot( m_last_sample_time ); // increase and recalculate time, time slot + m_any_packet_yet=true; + } + m_last_sample_time = time_now; // the real exact last time +} + +void network_throttle::handle_trafic_exact(size_t packet_size) +{ + _handle_trafic_exact(packet_size, packet_size); +} + +void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_size) +{ + tick(); + + calculate_times_struct cts ; calculate_times(packet_size, cts , false, -1); + calculate_times_struct cts2; calculate_times(packet_size, cts2, false, 5); + m_history[0].m_size += packet_size; + + std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; + std::string history_str = oss.str(); + + logger_handle_net("log/dr-monero/net/inreq-all.data",get_time_seconds(),packet_size); + _info_c( "net/" + m_nameshort , "Throttle " << m_name << ": packet of ~"<0) ? force_window : m_window_size) + ); + + if (!m_any_packet_yet) { + cts.window=0; cts.average=0; cts.delay=0; + cts.recomendetDataSize = m_network_minimal_segment; // should be overrided by caller anyway + return ; // no packet yet, I can not decide about sleep time + } + + network_time_seconds window_len = (the_window_size-1) * m_slot_size ; // -1 since current slot is not finished + window_len += (m_last_sample_time - time_to_slot(m_last_sample_time)); // add the time for current slot e.g. 13.7-13 = 0.7 + + auto time_passed = get_time_seconds() - m_start_time; + cts.window = std::max( std::min( window_len , time_passed ) , m_slot_size ) ; // window length resulting from size of history but limited by how long ago history was started, + // also at least slot size (e.g. 1 second) to not be ridiculous + // window_len e.g. 5.7 because takes into account current slot time + + size_t Epast = 0; // summ of traffic till now + for (auto sample : m_history) Epast += sample.m_size; + + const size_t E = Epast; + const size_t Enow = Epast + packet_size ; // including the data we're about to send now + + const double M = m_target_speed; // max + const double D1 = (Epast - M*cts.window) / M; // delay - how long to sleep to get back to target speed + const double D2 = (Enow - M*cts.window) / M; // delay - how long to sleep to get back to target speed (including current packet) + + auto O = get_current_overheat(); + auto Ouse = O * 0 ; // XXX TODO + cts.delay = (D1*0.80 + D2*0.20) + Ouse; // finall sleep depends on both with/without current packet + // update_overheat(); + cts.average = Epast/cts.window; // current avg. speed (for info) + + if (Epast <= 0) { + if (cts.delay>=0) cts.delay = 0; // no traffic in history so we will not wait + } + + double Wgood=-1; + { // how much data we recommend now to download + Wgood = the_window_size + 1; + cts.recomendetDataSize = M*cts.window - E; + } + + if (dbg) { + std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; + std::string history_str = oss.str(); + _dbg1_c( "net/"+m_nameshort+"_c" , + "dbg " << m_name << ": " + << "speed is A=" << std::setw(8) <100 elements + + std::vector< packet_info > m_history; // the history of bw usage + network_time_seconds m_last_sample_time; // time of last history[0] - so we know when to rotate the buffer + network_time_seconds m_start_time; // when we were created + bool m_any_packet_yet; // did we yet got any packet to count + + double m_overheat; // last overheat + double m_overheat_time; // time in seconds after epoch + + std::string m_name; // my name for debug and logs + std::string m_nameshort; // my name for debug and logs (used in log file name) + + // each sample is now 1 second + public: + network_throttle(const std::string &nameshort, const std::string &name, int window_size=-1); + virtual ~network_throttle(); + virtual void set_name(const std::string &name); + virtual void set_target_speed( network_speed_kbps target ); + virtual void set_target_kill( network_MB target ); + + // add information about events: + virtual void handle_trafic_exact(size_t packet_size); ///< count the new traffic/packet; the size is exact considering all network costs + virtual void handle_trafic_tcp(size_t packet_size); ///< count the new traffic/packet; the size is as TCP, we will consider MTU etc + virtual void handle_congestion(double overheat); ///< call this when congestion is detected; see example use + + virtual void tick(); ///< poke and update timers/history (recalculates, moves the history if needed, checks the real clock etc) + + virtual double get_time_seconds() const ; ///< timer that we use, time in seconds, monotionic + virtual double get_current_overheat() const; ///< did we detected congestion now. NOT USED NOW TODO + virtual void set_overheat(double lag); ///< did we detected congestion now. NOT USED NOW TODO. rename to add_overheat ? + + // time calculations: + virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const; ///< MAIN LOGIC (see base class for info) + + virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size); ///< increase the timer if needed, and get the package size + virtual network_time_seconds get_sleep_time(size_t packet_size) const; ///< gets the Delay (recommended Delay time) from calc. (not safe: only if time didnt change?) TODO + + virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported + virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const; ///< ditto, but for given windows time frame + //virtual void add_planned_transport(size_t size); + + private: + virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13 + virtual void _handle_trafic_exact(size_t packet_size, size_t orginal_size); + virtual void logger_handle_net(const std::string &filename, double time, size_t size); +}; + +/*** + * The complete set of traffic throttle for one typical connection +*/ +struct network_throttle_bw { + public: + network_throttle m_in; ///< for incomming traffic (this we can not controll directly as it depends of what others send to us - usually) + network_throttle m_inreq; ///< for requesting incomming traffic (this is exact usually) + network_throttle m_out; ///< for outgoing traffic that we just sent (this is exact usually) + + public: + network_throttle_bw(const std::string &name1); +}; + + + +} // namespace net_utils +} // namespace epee + + +#endif + + diff --git a/src/p2p/network_throttle.cpp b/src/p2p/network_throttle.cpp new file mode 100644 index 00000000..3d5edcdc --- /dev/null +++ b/src/p2p/network_throttle.cpp @@ -0,0 +1,121 @@ +/** +@file +@author rfree (current maintainer in monero.cc project) +@brief interface for throttling of connection (count and rate-limit speed etc) +@details
+
+Throttling work by:
+1) taking note of all traffic (hooks added e.g. to connection class) and measuring speed
+2) depending on that information we sleep before sending out data (or send smaller portions of data)
+3) depending on the information we can also sleep before sending requests or ask for smaller sets of data to download
+
+
+ +@image html images/net/rate1-down-1k.png +@image html images/net/rate1-down-full.png +@image html images/net/rate1-up-10k.png +@image html images/net/rate1-up-full.png +@image html images/net/rate2-down-100k.png +@image html images/net/rate2-down-10k.png +@image html images/net/rate2-down-50k.png +@image html images/net/rate2-down-full.png +@image html images/net/rate2-up-100k.png +@image html images/net/rate2-up-10k.png +@image html images/net/rate3-up-10k.png + + +*/ + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "../../src/p2p/network_throttle-detail.hpp" + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// network_throttle_manager +// ================================================================================================ + +// ================================================================================================ +// static: +std::mutex network_throttle_manager::m_lock_get_global_throttle_in; +std::mutex network_throttle_manager::m_lock_get_global_throttle_inreq; +std::mutex network_throttle_manager::m_lock_get_global_throttle_out; + +int network_throttle_manager::xxx; + + +// ================================================================================================ +// methods: +i_network_throttle & network_throttle_manager::get_global_throttle_in() { + + std::call_once(m_once_get_global_throttle_in, [] { m_obj_get_global_throttle_in.reset(new network_throttle("in/all","<<< global-IN",10)); } ); + return * m_obj_get_global_throttle_in; +} +std::once_flag network_throttle_manager::m_once_get_global_throttle_in; +std::unique_ptr network_throttle_manager::m_obj_get_global_throttle_in; + + + +i_network_throttle & network_throttle_manager::get_global_throttle_inreq() { + std::call_once(m_once_get_global_throttle_inreq, [] { m_obj_get_global_throttle_inreq.reset(new network_throttle("inreq/all", "<== global-IN-REQ",10)); } ); + return * m_obj_get_global_throttle_inreq; +} +std::once_flag network_throttle_manager::m_once_get_global_throttle_inreq; +std::unique_ptr network_throttle_manager::m_obj_get_global_throttle_inreq; + + +i_network_throttle & network_throttle_manager::get_global_throttle_out() { + std::call_once(m_once_get_global_throttle_out, [] { m_obj_get_global_throttle_out.reset(new network_throttle("out/all", ">>> global-OUT",10)); } ); + return * m_obj_get_global_throttle_out; +} +std::once_flag network_throttle_manager::m_once_get_global_throttle_out; +std::unique_ptr network_throttle_manager::m_obj_get_global_throttle_out; + + + + +network_throttle_bw::network_throttle_bw(const std::string &name1) + : m_in("in/"+name1, name1+"-DOWNLOAD"), m_inreq("inreq/"+name1, name1+"-DOWNLOAD-REQUESTS"), m_out("out/"+name1, name1+"-UPLOAD") +{ } + + + + +} // namespace +} // namespace + + + + + diff --git a/src/p2p/network_throttle.hpp b/src/p2p/network_throttle.hpp new file mode 100644 index 00000000..dc25a2c4 --- /dev/null +++ b/src/p2p/network_throttle.hpp @@ -0,0 +1,187 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief interface for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +/* rfree: throttle basic interface */ +/* rfree: also includes the manager for singeton/global such objects */ + + +#ifndef INCLUDED_p2p_network_throttle_hpp +#define INCLUDED_p2p_network_throttle_hpp + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include +#include +#include + +#include +#include +#include + +namespace epee +{ +namespace net_utils +{ + +// just typedefs to in code define the units used. TODO later it will be enforced that casts to other numericals are only explicit to avoid mistakes? use boost::chrono? +typedef double network_speed_kbps; +typedef double network_time_seconds; +typedef double network_MB; + +class i_network_throttle; + +/*** +@brief All information about given throttle - speed calculations +*/ +struct calculate_times_struct { + double average; + double window; + double delay; + double recomendetDataSize; +}; +typedef calculate_times_struct calculate_times_struct; + + +namespace cryptonote { class cryptonote_protocol_handler_base; }; // a friend class // TODO friend not working + +/*** +@brief Access to simple throttles, with singlton to access global network limits +*/ +class network_throttle_manager { + // provides global (singleton) in/inreq/out throttle access + + // [[note1]] see also http://www.nuonsoft.com/blog/2012/10/21/implementing-a-thread-safe-singleton-with-c11/ + // [[note2]] _inreq is the requested in traffic - we anticipate we will get in-bound traffic soon as result of what we do (e.g. that we sent network downloads requests) + + //protected: + public: // XXX + // [[note1]] + static std::once_flag m_once_get_global_throttle_in; + static std::once_flag m_once_get_global_throttle_inreq; // [[note2]] + static std::once_flag m_once_get_global_throttle_out; + static std::unique_ptr m_obj_get_global_throttle_in; + static std::unique_ptr m_obj_get_global_throttle_inreq; + static std::unique_ptr m_obj_get_global_throttle_out; + + static std::mutex m_lock_get_global_throttle_in; + static std::mutex m_lock_get_global_throttle_inreq; + static std::mutex m_lock_get_global_throttle_out; + + friend class cryptonote::cryptonote_protocol_handler_base; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! + friend class connection_basic; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! + friend class connection_basic_pimpl; // ditto + + static int xxx; + + public: + static i_network_throttle & get_global_throttle_in(); ///< singleton ; for friend class ; caller MUST use proper locks! like m_lock_get_global_throttle_in + static i_network_throttle & get_global_throttle_inreq(); ///< ditto ; use lock ... use m_lock_get_global_throttle_inreq obviously + static i_network_throttle & get_global_throttle_out(); ///< ditto ; use lock ... use m_lock_get_global_throttle_out obviously +}; + + + +/*** +@brief interface for the throttle, see the derivated class +*/ +class i_network_throttle { + public: + virtual void set_name(const std::string &name)=0; + virtual void set_target_speed( network_speed_kbps target )=0; + virtual void set_target_kill( network_MB target )=0; + + virtual void handle_trafic_exact(size_t packet_size) =0; // count the new traffic/packet; the size is exact considering all network costs + virtual void handle_trafic_tcp(size_t packet_size) =0; // count the new traffic/packet; the size is as TCP, we will consider MTU etc + virtual void handle_congestion(double overheat) =0; // call this when congestion is detected; see example use + virtual void tick() =0; // poke and update timers/history + + // time calculations: + + virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const =0; // assuming sending new package (or 0), calculate: + // Average, Window, Delay, Recommended data size ; also gets dbg=debug flag, and forced widnow size if >0 or -1 for not forcing window size + + // Average speed, Window size, recommended Delay to sleep now, Recommended size of data to send now + + virtual network_time_seconds get_sleep_time(size_t packet_size) const =0; // gets the D (recommended Delay time) from calc + virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size) =0; // ditto, but first tick the timer + + virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future + + virtual double get_time_seconds() const =0; // a timer + virtual double get_current_overheat() const =0; + virtual void set_overheat(double lag) =0; + virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0; + + +}; + + +// ... more in the -advanced.h file + + +} // namespace net_utils +} // namespace epee + + +#endif + + + diff --git a/src/simplewallet/CMakeLists.txt b/src/simplewallet/CMakeLists.txt index 84acbf29..a33ed0f3 100644 --- a/src/simplewallet/CMakeLists.txt +++ b/src/simplewallet/CMakeLists.txt @@ -50,6 +50,7 @@ target_link_libraries(simplewallet crypto common mnemonics + p2p ${UNBOUND_LIBRARY} ${UPNP_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} diff --git a/tests/core_proxy/CMakeLists.txt b/tests/core_proxy/CMakeLists.txt index e94d8d80..7d40d72b 100644 --- a/tests/core_proxy/CMakeLists.txt +++ b/tests/core_proxy/CMakeLists.txt @@ -38,6 +38,8 @@ add_executable(core_proxy target_link_libraries(core_proxy LINK_PRIVATE cryptonote_core + cryptonote_protocol + p2p ${UPNP_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index 568dfb2a..2b807564 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -81,5 +81,6 @@ namespace tests bool on_idle(){return true;} bool find_blockchain_supplement(const std::list& qblock_ids, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;} bool handle_get_objects(cryptonote::NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote::cryptonote_connection_context& context){return true;} + cryptonote::blockchain_storage &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class proxy_core."); } }; } diff --git a/tests/net_load_tests/CMakeLists.txt b/tests/net_load_tests/CMakeLists.txt index 89626811..6095146f 100644 --- a/tests/net_load_tests/CMakeLists.txt +++ b/tests/net_load_tests/CMakeLists.txt @@ -37,6 +37,8 @@ add_executable(net_load_tests_clt ${clt_headers}) target_link_libraries(net_load_tests_clt LINK_PRIVATE + otshell_utils + p2p ${GTEST_MAIN_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_DATE_TIME_LIBRARY} @@ -56,6 +58,8 @@ add_executable(net_load_tests_srv ${srv_headers}) target_link_libraries(net_load_tests_srv LINK_PRIVATE + otshell_utils + p2p ${GTEST_MAIN_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_DATE_TIME_LIBRARY} diff --git a/tests/net_load_tests/clt.cpp b/tests/net_load_tests/clt.cpp index 85cad917..7c9b86cb 100644 --- a/tests/net_load_tests/clt.cpp +++ b/tests/net_load_tests/clt.cpp @@ -44,7 +44,10 @@ #include "net_load_tests.h" +#include "../../contrib/otshell_utils/utils.hpp" + using namespace net_load_tests; +using namespace nOT::nUtils; namespace { diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index c480a312..41e30683 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -58,6 +58,7 @@ target_link_libraries(unit_tests cryptonote_core rpc wallet + p2p ${GTEST_MAIN_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_REGEX_LIBRARY} From 5ce4256e3d6ff2e1595750e3875865089e20a03b Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Thu, 12 Feb 2015 20:59:39 +0100 Subject: [PATCH 02/16] 2014 network limit 1.1 +utils +toc -doc -drmonero Update of the PR with network limits works very well for all speeds (but remember that low download speed can stop upload because we then slow down downloading of blockchain requests too) more debug options fixed pedantic warnings in our code should work again on Mac OS X and FreeBSD fixed warning about size_t tested on Debian, Ubuntu, Windows(testing now) TCP options and ToS (QoS) flag FIXED peer number limit FIXED some spikes in ingress/download FIXED problems when other up and down limit --- .gitignore | 2 +- CMakeLists.txt | 23 +-- .../epee/include/net/abstract_tcp_server2.h | 9 + .../epee/include/net/abstract_tcp_server2.inl | 68 +++++++- .../net/levin_protocol_handler_async.h | 26 ++- contrib/epee/include/net/net_utils_base.h | 10 +- .../include/storages/levin_abstract_invoke2.h | 4 +- contrib/otshell_utils/runoptions.cpp | 4 +- contrib/otshell_utils/runoptions.hpp | 4 +- contrib/otshell_utils/utils.cpp | 60 ++++--- contrib/otshell_utils/utils.hpp | 11 +- src/cryptonote_core/blockchain_storage.cpp | 18 ++ src/cryptonote_core/blockchain_storage.h | 1 + src/cryptonote_core/cryptonote_core.cpp | 31 +++- src/cryptonote_core/cryptonote_core.h | 6 + .../cryptonote_protocol_handler-base.cpp | 94 +--------- .../cryptonote_protocol_handler.h | 10 +- .../cryptonote_protocol_handler.inl | 148 ++++++++++++++-- src/daemon/daemon.cpp | 36 ++-- src/daemon/daemon_commands_handler.h | 54 +++++- src/p2p/connection_basic.cpp | 165 ++++++------------ src/p2p/connection_basic.hpp | 11 +- src/p2p/data_logger.cpp | 81 +++++++++ src/p2p/data_logger.hpp | 46 +++++ src/p2p/net_node.h | 12 +- src/p2p/net_node.inl | 66 ++++++- src/p2p/network_throttle-detail.cpp | 75 ++++---- src/p2p/network_throttle-detail.hpp | 10 +- src/p2p/network_throttle.hpp | 8 +- src/serialization/binary_archive.h | 4 +- 30 files changed, 714 insertions(+), 383 deletions(-) create mode 100644 src/p2p/data_logger.cpp create mode 100644 src/p2p/data_logger.hpp diff --git a/.gitignore b/.gitignore index 3aed0e11..755cb104 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ cscope.out cscope.in.out cscope.po.out - +external/miniupnpc/Makefile miniupnpcstrings.h version/ # Created by https://www.gitignore.io diff --git a/CMakeLists.txt b/CMakeLists.txt index bdb96746..84a48c11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,24 +50,17 @@ list(INSERT CMAKE_MODULE_PATH 0 if (NOT DEFINED ENV{DEVELOPER_LOCAL_TOOLS}) message(STATUS "Could not find DEVELOPER_LOCAL_TOOLS in env (not required)") - set(BOOST_IGNORE_SYSTEM_PATHS OFF) -elseif ("$ENV{DEVELOPER_LOCAL_TOOLS}" STREQUAL "1") + set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT OFF) +elseif ("$ENV{DEVELOPER_LOCAL_TOOLS}" EQUAL 1) message(STATUS "Found: env DEVELOPER_LOCAL_TOOLS = 1") - set(BOOST_IGNORE_SYSTEM_PATHS ON) + set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT ON) else() message(STATUS "Found: env DEVELOPER_LOCAL_TOOLS = 0") - set(BOOST_IGNORE_SYSTEM_PATHS OFF) + set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT OFF) endif() -#message(STATUS "BOOST_IGNORE_SYSTEM_PATHS defaults to ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}") -#option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost installation" ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}) -message(STATUS "BOOST_IGNORE_SYSTEM_PATHS: ${BOOST_IGNORE_SYSTEM_PATHS}") - -# Options (for external/otshell_utils/) -option(WITH_TERMCOLORS "Build with support for unix terminal console colors VT100" ON) -if (WITH_TERMCOLORS) - add_definitions( -DCFG_WITH_TERMCOLORS ) -endif () +message(STATUS "BOOST_IGNORE_SYSTEM_PATHS defaults to ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}") +option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost installation" ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}) set_property(GLOBAL PROPERTY USE_FOLDERS ON) enable_testing() @@ -162,9 +155,9 @@ else() else() set(ARCH_FLAG "-march=${ARCH}") endif() - set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-error=sign-compare -Wno-error=strict-aliasing -Wno-error=type-limits -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized") + set(WARNINGS "-Wall -pedantic -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-error=sign-compare -Wno-error=strict-aliasing -Wno-error=type-limits -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized") if(NOT MINGW) - set(WARNINGS "${WARNINGS} -Werror") + # set(WARNINGS "${WARNINGS} -Werror") # to allow pedantic but not stop compilation endif() if(CMAKE_C_COMPILER_ID STREQUAL "Clang") set(WARNINGS "${WARNINGS} -Wno-error=mismatched-tags -Wno-error=null-conversion -Wno-overloaded-shift-op-parentheses -Wno-error=shift-count-overflow -Wno-error=tautological-constant-out-of-range-compare -Wno-error=unused-private-field -Wno-error=unneeded-internal-declaration") diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 1e622321..3e6ea217 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -56,6 +56,7 @@ #include "syncobj.h" #include "../../../../src/p2p/connection_basic.hpp" #include "../../../../contrib/otshell_utils/utils.hpp" +#include "../../../../src/p2p/network_throttle-detail.hpp" #define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000 @@ -130,6 +131,7 @@ namespace net_utils /// Buffer for incoming data. boost::array buffer_; + //boost::array buffer_; t_connection_context context; i_connection_filter* &m_pfilter; @@ -143,6 +145,12 @@ namespace net_utils critical_section m_chunking_lock; // held while we add small chunks of the big do_send() to small do_send_chunk() t_server_role m_connection_type; + + // for calculate speed (last 60 sec) + network_throttle m_throttle_speed_in; + network_throttle m_throttle_speed_out; + std::mutex m_throttle_speed_in_mutex; + std::mutex m_throttle_speed_out_mutex; public: void setRPcStation(); @@ -285,6 +293,7 @@ namespace net_utils critical_section m_threads_lock; volatile uint32_t m_thread_index; // TODO change to std::atomic t_server_role type; + void detach_threads(); /// The next connection to be accepted connection_ptr new_connection_; diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 8dff192b..5855f7f8 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -52,6 +52,7 @@ #include "../../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal() #include "../../../../contrib/otshell_utils/utils.hpp" +#include "../../../../src/p2p/data_logger.hpp" using namespace nOT::nUtils; // TODO PRAGMA_WARNING_PUSH @@ -75,7 +76,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) connection_basic(io_service, ref_sock_count, sock_number), m_protocol_handler(this, config, context), m_pfilter( pfilter ), - m_connection_type(NET) + m_connection_type(NET), + m_throttle_speed_in("speed_in", "throttle_speed_in"), + m_throttle_speed_out("speed_out", "throttle_speed_out") { _info_c("net/sleepRPC", "connection constructor set m_connection_type="<::start()", false); @@ -240,6 +246,32 @@ PRAGMA_WARNING_DISABLE_VS(4355) if (!e) { + { + CRITICAL_REGION_LOCAL(m_throttle_speed_in_mutex); + m_throttle_speed_in.handle_trafic_exact(bytes_transferred); + context.m_current_speed_down = m_throttle_speed_in.get_current_speed(); + } + + { + CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_in ); + epee::net_utils::network_throttle_manager::network_throttle_manager::get_global_throttle_in().handle_trafic_exact(bytes_transferred * 1024); + } + double delay=0; // will be calculated + do + { + { //_scope_dbg1("CRITICAL_REGION_LOCAL"); + CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in ); + delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred ); // decission from global + } + + delay *= 0.5; + if (delay > 0) { + long int ms = (long int)(delay * 100); + epee::net_utils::data_logger::get_instance().add_data("sleep_down", ms); + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } + } while(delay > 0); + //_info("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred); logger_handle_net_read(bytes_transferred); context.m_last_recv = time(NULL); @@ -398,6 +430,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) return false; if(m_was_shutdown) return false; + { + CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex); + m_throttle_speed_out.handle_trafic_exact(cb); + context.m_current_speed_up = m_throttle_speed_out.get_current_speed(); + } //_info("[sock " << socket_.native_handle() << "] SEND " << cb); context.m_last_send = time(NULL); @@ -405,8 +442,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) //some data should be wrote to stream //request complete - do_send_handler_start( ptr , cb ); // (((H))) - + sleep_before_packet(cb, 1, 1); epee::critical_region_t send_guard(m_send_que_lock); // *** critical *** long int retry=0; const long int retry_limit = 5*4; @@ -440,8 +476,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) { // active operation should be in progress, nothing to do, just wait last operation callback auto size_now = cb; _info_c("net/out/size", "do_send() NOW just queues: packet="< void boosted_tcp_server::send_stop_signal() { + if (::cryptonote::core::get_fast_exit() == true) + { + detach_threads(); + } m_stop_signal_sent = true; TRY_ENTRY(); io_service_.stop(); @@ -988,6 +1029,15 @@ POP_WARNINGS return true; CATCH_ENTRY_L0("boosted_tcp_server::connect_async", false); } + //--------------------------------------------------------------------------------- + template + void boosted_tcp_server::detach_threads() + { + for (auto thread : m_threads) + thread->detach(); + } + + } // namespace } // namespace PRAGMA_WARNING_POP diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 5e5e803f..92f75161 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -81,7 +81,7 @@ public: async_protocol_handler_config():m_pcommands_handler(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE) {} - void del_connections(size_t count); + void del_out_connections(size_t count); }; @@ -670,10 +670,30 @@ void async_protocol_handler_config::del_connection(async_p } //------------------------------------------------------------------------------------------ template -void async_protocol_handler_config::del_connections(size_t count) // TODO +void async_protocol_handler_config::del_out_connections(size_t count) { + std::vector out_connections; CRITICAL_REGION_BEGIN(m_connects_lock); - m_connects.clear(); + for (auto& c: m_connects) + { + if (!c.second->m_connection_context.m_is_income) + out_connections.push_back(c.first); + } + + if (out_connections.size() == 0) + return; + + // close random out connections + unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); + shuffle(out_connections.begin(), out_connections.end(), std::default_random_engine(seed)); + while (count > 0 && out_connections.size() > 0) + { + close(*out_connections.begin()); + del_connection(m_connects.at(*out_connections.begin())); + out_connections.erase(out_connections.begin()); + --count; + } + CRITICAL_REGION_END(); } //------------------------------------------------------------------------------------------ diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 90e35278..f963e774 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -55,6 +55,8 @@ namespace net_utils time_t m_last_send; uint64_t m_recv_cnt; uint64_t m_send_cnt; + double m_current_speed_down; + double m_current_speed_up; connection_context_base(boost::uuids::uuid connection_id, long remote_ip, int remote_port, bool is_income, @@ -68,7 +70,9 @@ namespace net_utils m_last_recv(last_recv), m_last_send(last_send), m_recv_cnt(recv_cnt), - m_send_cnt(send_cnt) + m_send_cnt(send_cnt), + m_current_speed_down(0), + m_current_speed_up(0) {} connection_context_base(): m_connection_id(), @@ -79,7 +83,9 @@ namespace net_utils m_last_recv(0), m_last_send(0), m_recv_cnt(0), - m_send_cnt(0) + m_send_cnt(0), + m_current_speed_down(0), + m_current_speed_up(0) {} connection_context_base& operator=(const connection_context_base& a) diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 1b32c51d..73ede1b1 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -185,7 +185,7 @@ namespace epee } return res; - }; + } template int buff_to_t_adapter(t_owner* powner, int command, const std::string& in_buff, callback_t cb, t_context& context) @@ -199,7 +199,7 @@ namespace epee boost::value_initialized in_struct; static_cast(in_struct).load(strg); return cb(command, in_struct, context); - }; + } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ int invoke(int command, const std::string& in_buff, std::string& buff_out, context_type& context) \ diff --git a/contrib/otshell_utils/runoptions.cpp b/contrib/otshell_utils/runoptions.cpp index 28e7ceb5..ffd37eae 100644 --- a/contrib/otshell_utils/runoptions.cpp +++ b/contrib/otshell_utils/runoptions.cpp @@ -7,7 +7,7 @@ namespace nOT { -INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces // (no debug - this is the default) // +nodebug (no debug) @@ -64,6 +64,6 @@ void cRunOptions::Normalize() { cRunOptions gRunOptions; // (extern) -}; // namespace OT +} // namespace OT diff --git a/contrib/otshell_utils/runoptions.hpp b/contrib/otshell_utils/runoptions.hpp index f3306283..219d3b50 100644 --- a/contrib/otshell_utils/runoptions.hpp +++ b/contrib/otshell_utils/runoptions.hpp @@ -10,7 +10,7 @@ Template for new files, replace word "template" and later delete this line here. namespace nOT { -INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces /** Global options to run this program main() Eg used for developer's special options like +setdemo +setdebug. This is NOT for all the other options that are parsed and executed by program. */ @@ -50,7 +50,7 @@ class cRunOptions { extern cRunOptions gRunOptions; -}; // namespace nOT +} // namespace nOT diff --git a/contrib/otshell_utils/utils.cpp b/contrib/otshell_utils/utils.cpp index 489fb307..1d26075c 100644 --- a/contrib/otshell_utils/utils.cpp +++ b/contrib/otshell_utils/utils.cpp @@ -26,8 +26,7 @@ #elif defined(__unix__) || defined(__posix) || defined(__linux) || defined(__darwin) || defined(__APPLE__) || defined(__clang__) #define OS_TYPE_POSIX #else - #warning "Compiler/OS platform is not recognized" - #warning "Just assuming it will work as POSIX then" + #warning "Compiler/OS platform is not recognized. Just assuming it will work as POSIX then" #define OS_TYPE_POSIX #endif @@ -44,7 +43,7 @@ namespace nOT { namespace nUtils { -INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces myexception::myexception(const char * what) : std::runtime_error(what) @@ -78,26 +77,37 @@ std::string & trim(std::string &s) { return ltrim(rtrim(s)); } -std::string get_current_time() -{ +std::string get_current_time() { + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + time_t time_now = std::chrono::system_clock::to_time_t(now); + std::chrono::high_resolution_clock::duration duration = now.time_since_epoch(); + int64_t micro = std::chrono::duration_cast(duration).count(); + + // std::localtime() - This function may not be thread-safe. + #ifdef OS_TYPE_WINDOWS + struct tm * tm_pointer = std::localtime( &time_now ); // thread-safe on mingw-w64 (thread local variable) and on MSVC btw + // http://stackoverflow.com/questions/18551409/localtime-r-support-on-mingw + // tm_pointer points to thread-local data, memory is owned/managed by the system/library + #else + // linux, freebsd, have this + struct tm tm_object; // automatic storage duration http://en.cppreference.com/w/cpp/language/storage_duration + struct tm * tm_pointer = & tm_object; // just point to our data + auto x = localtime_r( &time_now , tm_pointer ); // modifies our own (this thread) data in tm_object, this is safe http://linux.die.net/man/3/localtime_r + if (x != tm_pointer) return "(internal error in get_current_time)"; // redundant check in case of broken implementation of localtime_r + #endif + // tm_pointer now points to proper time data, and that memory is automatically managed + if (!tm_pointer) return "(internal error in get_current_time - NULL)"; // redundant check in case of broken implementation of used library methods + std::stringstream stream; - struct tm * date; - - std::chrono::high_resolution_clock::time_point now = std::chrono::high_resolution_clock::now(); - time_t time_now; - time_now = std::chrono::high_resolution_clock::to_time_t(now); - date = std::localtime(& time_now); - - char date_buff[32]; - std::strftime(date_buff, sizeof(date_buff), "%d-%b-%Y %H:%M:%S.", date); - stream << date_buff; - - std::chrono::high_resolution_clock::duration duration = now.time_since_epoch(); - int64_t micro = std::chrono::duration_cast(duration).count(); - micro %= 1000000; - stream << std::setfill('0') << std::setw(3) << micro; - - return stream.str(); + stream << std::setfill('0') + << std::setw(2) << tm_pointer->tm_year+1900 + << '-' << std::setw(2) << tm_pointer->tm_mon+1 + << '-' << std::setw(2) << tm_pointer->tm_mday + << ' ' << std::setw(2) << tm_pointer->tm_hour + << ':' << std::setw(2) << tm_pointer->tm_min + << ':' << std::setw(2) << tm_pointer->tm_sec + << '.' << std::setw(6) << (micro%1000000); // 6 because microseconds + return stream.str(); } cNullstream g_nullstream; // extern a stream that does nothing (eats/discards data) @@ -213,7 +223,7 @@ void cDebugScopeGuard::Assign(const string &chan, const int level, const string mMsg=msg; } -}; // namespace nDetail +} // namespace nDetail // ==================================================================== @@ -591,10 +601,10 @@ string stringToColor(const string &hash) { // algorthms -}; // namespace nUtil +} // namespace nUtil -}; // namespace OT +} // namespace OT // global namespace diff --git a/contrib/otshell_utils/utils.hpp b/contrib/otshell_utils/utils.hpp index 6cfd11ee..bb984320 100644 --- a/contrib/otshell_utils/utils.hpp +++ b/contrib/otshell_utils/utils.hpp @@ -14,7 +14,7 @@ #endif #ifndef CFG_WITH_TERMCOLORS - #error "You requested to turn off terminal colors (CFG_WITH_TERMCOLORS), however currently they are hardcoded (this option to turn them off is not yet implemented)." + //#error "You requested to turn off terminal colors (CFG_WITH_TERMCOLORS), however currently they are hardcoded (this option to turn them off is not yet implemented)." #endif ///Macros related to automatic deduction of class name etc; @@ -35,7 +35,7 @@ class myexception : public std::runtime_error { }; /// @macro Use this macro INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 as a shortcut for various using std::string etc. -INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces // ====================================================================================== /// text trimming functions (they do mutate the passes string); they trim based on std::isspace. also return it's reference again @@ -87,6 +87,7 @@ extern std::mutex gLoggerGuard; #define _warn(VAR) _debug_level( 90,VAR) // some problem #define _erro(VAR) _debug_level(100,VAR) // error - report + #define _dbg3_c(C,VAR) _debug_level_c(C, 20,VAR) #define _dbg2_c(C,VAR) _debug_level_c(C, 30,VAR) #define _dbg1_c(C,VAR) _debug_level_c(C, 40,VAR) // details @@ -140,7 +141,7 @@ class cDebugScopeGuard { const char* DbgShortenCodeFileName(const char *s); ///< Returns a pointer to some part of the string that was given, skipping directory names, for log/debug -}; // namespace nDetail +} // namespace nDetail // ========== logger ========== @@ -423,9 +424,9 @@ class value_init { template value_init::value_init() : data(INIT) { } -}; // namespace nUtils +} // namespace nUtils -}; // namespace nOT +} // namespace nOT // global namespace diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index 78419121..fe80d75a 100644 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -178,6 +178,22 @@ bool blockchain_storage::store_genesis_block(bool testnet) { return true; } //------------------------------------------------------------------ +void blockchain_storage::logger_handle(long int ms) +{ + std::ofstream log_file; + log_file.open("log/dr-monero/blockchain_log.data", std::ofstream::out | std::ofstream::app); + log_file.precision(7); + + using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto m_ms = duration_cast< milliseconds >( time_from_epoh ).count(); + double ms_f = m_ms; + ms_f /= 1000.; + + log_file << ms_f << " " << ms << std::endl; +} +//------------------------------------------------------------------ bool blockchain_storage::store_blockchain() { m_is_blockchain_storing = true; @@ -1760,6 +1776,8 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size << ", " << block_processing_time << "("<< target_calculating_time << "/" << longhash_calculating_time << ")ms"); + logger_handle(block_processing_time); + bvc.m_added_to_main_chain = true; /*if(!m_orphanes_reorganize_in_work) review_orphaned_blocks_with_new_block_id(id, true);*/ diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index 38bdfbce..6456689b 100644 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -249,6 +249,7 @@ namespace cryptonote bool complete_timestamps_vector(uint64_t start_height, std::vector& timestamps); bool update_next_comulative_size_limit(); bool store_genesis_block(bool testnet); + void logger_handle(long int ms); }; diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index b8b5dc00..c8daa351 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -187,11 +187,34 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::deinit() { - m_miner.stop(); - m_mempool.deinit(); - m_blockchain_storage.deinit(); + m_miner.stop(); + m_mempool.deinit(); + if (!m_fast_exit) + { + m_blockchain_storage.deinit(); + } return true; } + //----------------------------------------------------------------------------------------------- + void core::set_fast_exit() + { + m_fast_exit = true; + } + //----------------------------------------------------------------------------------------------- + bool core::get_fast_exit() + { + return m_fast_exit; + } + //----------------------------------------------------------------------------------------------- + void core::no_check_blocks() + { + m_check_blocks = false; + } + //----------------------------------------------------------------------------------------------- + bool core::get_check_blocks() + { + return m_check_blocks; + } //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block) { @@ -595,4 +618,6 @@ namespace cryptonote { raise(SIGTERM); } + + std::atomic core::m_fast_exit(false); } diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 748f2b66..9218aef0 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -75,6 +75,10 @@ namespace cryptonote bool init(const boost::program_options::variables_map& vm, bool testnet); bool set_genesis_block(const block& b); bool deinit(); + static void set_fast_exit(); + static bool get_fast_exit(); + void no_check_blocks(); + bool get_check_blocks(); uint64_t get_current_blockchain_height(); bool get_blockchain_top(uint64_t& heeight, crypto::hash& top_id); bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs); @@ -146,6 +150,8 @@ namespace cryptonote bool on_update_blocktemplate_interval(); bool check_tx_inputs_keyimages_diff(const transaction& tx); void graceful_exit(); + static std::atomic m_fast_exit; + bool m_check_blocks = true; tx_memory_pool m_mempool; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp index 6b25cb68..b5a5ceea 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -104,11 +104,11 @@ namespace cryptonote { double cryptonote_protocol_handler_base::estimate_one_block_size() noexcept { // for estimating size of blocks to downloa const double size_min = 500; // XXX 500 - const int history_len = 20; // how many blocks to average over + //const int history_len = 20; // how many blocks to average over double avg=0; try { - avg = get_avg_block_size(history_len); + avg = get_avg_block_size(/*history_len*/); } catch (...) { } avg = std::max( size_min , avg); return avg; @@ -120,96 +120,6 @@ cryptonote_protocol_handler_base::cryptonote_protocol_handler_base() { cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() { } -void cryptonote_protocol_handler_base::handler_request_blocks_now(size_t &count_limit) { - using namespace epee::net_utils; - size_t est_req_size=0; // how much data are we now requesting (to be soon send to us) - - const auto count_limit_default = count_limit; - - bool allowed_now = false; // are we now allowed to request or are we limited still - // long int size_limit; - - while (!allowed_now) { - /* if ( ::cryptonote::core::get_is_stopping() ) { // TODO fast exit - _fact("ABORT sleep (before sending requeset) due to stopping"); - break; - }*/ - - //LOG_PRINT_RED("[DBG]" << get_avg_block_size(1), LOG_LEVEL_0); - //{ - long int size_limit1=0, size_limit2=0; - //LOG_PRINT_RED("calculating REQUEST size:", LOG_LEVEL_0); - { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); - network_throttle_manager::get_global_throttle_in().tick(); - size_limit1 = network_throttle_manager::get_global_throttle_in().get_recommended_size_of_planned_transport(); - } - { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); - network_throttle_manager::get_global_throttle_inreq().tick(); - size_limit2 = network_throttle_manager::get_global_throttle_inreq().get_recommended_size_of_planned_transport(); - } - - long int one_block_estimated_size = estimate_one_block_size(); - long int limit_small = std::min( size_limit1 , size_limit2 ); - long int size_limit = limit_small/3 + size_limit1/3 + size_limit2/3; - if (limit_small <= 0) size_limit = 0; - const double estimated_peers = 1.2; // how many peers/threads we want to talk to, in order to not grab entire b/w by 1 thread - const double knob = 1.000; - size_limit /= (estimated_peers / estimated_peers) * knob; - _note_c("net/req-calc" , "calculating REQUEST size:" << size_limit1 << " " << size_limit2 << " small=" << limit_small << " final size_limit="<& ids) { using namespace epee::net_utils; LOG_PRINT_L0("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index b3393928..0be864df 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -46,6 +46,7 @@ #include "cryptonote_core/cryptonote_stat_info.h" #include "cryptonote_core/verification_context.h" #include +#include PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) @@ -61,11 +62,10 @@ namespace cryptonote public: cryptonote_protocol_handler_base(); virtual ~cryptonote_protocol_handler_base(); - void handler_request_blocks_now(size_t & count_limit); // before asking for blocks, can adjust the limit of download void handler_request_blocks_history(std::list& ids); // before asking for list of objects, we can change the list still void handler_response_blocks_now(size_t packet_size); - virtual double get_avg_block_size( size_t count) const = 0; + virtual double get_avg_block_size() = 0; virtual double estimate_one_block_size() noexcept; // for estimating size of blocks to download virtual std::ofstream& get_logreq() const =0; @@ -129,10 +129,12 @@ namespace cryptonote nodetool::i_p2p_endpoint* m_p2p; std::atomic m_syncronized_connections_count; std::atomic m_synchronized; + bool m_one_request = true; // static std::ofstream m_logreq; - - double get_avg_block_size(size_t count) const; + std::mutex m_buffer_mutex; + double get_avg_block_size(); + boost::circular_buffer m_avg_buffer = boost::circular_buffer(10); template bool post_notify(typename t_parametr::request& arg, cryptonote_connection_context& context) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index aebfcd10..0130cb38 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -41,6 +41,7 @@ #include "cryptonote_core/cryptonote_format_utils.h" #include "profile_tools.h" #include "../../contrib/otshell_utils/utils.hpp" +#include "../../src/p2p/network_throttle-detail.hpp" using namespace nOT::nUtils; namespace cryptonote @@ -115,28 +116,66 @@ namespace cryptonote void t_cryptonote_protocol_handler::log_connections() { std::stringstream ss; + ss.precision(1); + + double down_sum = 0.0; + double down_curr_sum = 0.0; + double up_sum = 0.0; + double up_curr_sum = 0.0; ss << std::setw(30) << std::left << "Remote Host" << std::setw(20) << "Peer id" - << std::setw(25) << "Recv/Sent (inactive,sec)" + << std::setw(30) << "Recv/Sent (inactive,sec)" << std::setw(25) << "State" - << std::setw(20) << "Livetime(seconds)" << ENDL; + << std::setw(20) << "Livetime(sec)" + << std::setw(12) << "Down (kB/s)" + << std::setw(14) << "Down(now)" + << std::setw(10) << "Up (kB/s)" + << std::setw(13) << "Up(now)" + << ENDL; uint32_t ip; m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id) { + bool local_ip = false; ip = ntohl(cntxt.m_remote_ip); + // TODO: local ip in calss A, B + if (ip > 3232235520 && ip < 3232301055) // 192.168.x.x + local_ip = true; + auto connection_time = time(NULL) - cntxt.m_started; ss << std::setw(30) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") + epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port) << std::setw(20) << std::hex << peer_id - << std::setw(25) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" + << std::setw(30) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" << std::setw(25) << get_protocol_state_string(cntxt.m_state) << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) - << std::setw(10) << (ip > 3232235520 && ip < 3232301055 ? " [LAN]" : "") //TODO: local ip in calss A, B - << ENDL; + << std::setw(12) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_recv_cnt / connection_time / 1024) + << std::setw(14) << std::fixed << cntxt.m_current_speed_down / 1024 + << std::setw(10) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_send_cnt / connection_time / 1024) + << std::setw(13) << std::fixed << cntxt.m_current_speed_up / 1024 + << (local_ip ? "[LAN]" : "") + << std::left << (ip == 2130706433 ? "[LOCALHOST]" : "") // 127.0.0.1 + << ENDL; + + if (connection_time > 0) + { + down_sum += (cntxt.m_recv_cnt / connection_time / 1024); + up_sum += (cntxt.m_send_cnt / connection_time / 1024); + } + + down_curr_sum += (cntxt.m_current_speed_down / 1024); + up_curr_sum += (cntxt.m_current_speed_up / 1024); + return true; }); - LOG_PRINT_L0("Connections: " << ENDL << ss.str()); + ss << ENDL + << std::setw(125) << " " + << std::setw(12) << down_sum + << std::setw(14) << down_curr_sum + << std::setw(10) << up_sum + << std::setw(13) << up_curr_sum + << ENDL; + LOG_PRINT_L0("Connections: " << ENDL << ss.str()); } //------------------------------------------------------------------------------------------------------------------------ // Returns a list of connection_info objects describing each open p2p connection @@ -332,8 +371,22 @@ namespace cryptonote template - double t_cryptonote_protocol_handler::get_avg_block_size( size_t count) const { - return m_core.get_blockchain_storage().get_avg_block_size(count); + double t_cryptonote_protocol_handler::get_avg_block_size() { + // return m_core.get_blockchain_storage().get_avg_block_size(count); // this does not count too well the actuall network-size of data we need to download + + CRITICAL_REGION_LOCAL(m_buffer_mutex); + double avg = 0; + if (m_avg_buffer.size() == 0) { + _warn("m_avg_buffer.size() == 0"); + return 500; + } + + const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option + long int dbg_repeat=0; + do { + for (auto element : m_avg_buffer) avg += element; + } while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one + return avg / m_avg_buffer.size(); } @@ -341,6 +394,41 @@ namespace cryptonote int t_cryptonote_protocol_handler::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { LOG_PRINT_CCONTEXT_L2("NOTIFY_RESPONSE_GET_OBJECTS"); + + // calculate size of request - mainly for logging/debug + size_t size = 0; + for (auto element : arg.txs) + size += element.size(); + + for (auto element : arg.blocks) + { + size += element.block.size(); + for (auto tx : element.txs) + size += tx.size(); + } + + for (auto element : arg.missed_ids) + size += sizeof(element.data); + + size += sizeof(arg.current_blockchain_height); + { + CRITICAL_REGION_LOCAL(m_buffer_mutex); + m_avg_buffer.push_back(size); + + const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option + long int dbg_repeat=0; + do { + m_avg_buffer.push_back(666); // a test value + m_avg_buffer.erase_end(1); + } while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one + } + /*using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto sec = duration_cast< seconds >( time_from_epoh ).count();*/ + + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + if(context.m_last_response_height > arg.current_blockchain_height) { LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height @@ -430,8 +518,9 @@ namespace cryptonote //process block TIME_MEASURE_START(block_process_time); block_verification_context bvc = boost::value_initialized(); - - m_core.handle_incoming_block(block_entry.block, bvc, false); + + if (m_core.get_check_blocks()) + m_core.handle_incoming_block(block_entry.block, bvc, false); if(bvc.m_verifivation_failed) { @@ -448,10 +537,21 @@ namespace cryptonote TIME_MEASURE_FINISH(block_process_time); LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms"); + + std::ofstream log_file; + log_file.open("log/dr-monero/get_objects_calc_time.data", std::ofstream::out | std::ofstream::app); + log_file.precision(7); + + using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto m_ms = duration_cast< milliseconds >( time_from_epoh ).count(); + double ms_f = m_ms; + ms_f /= 1000.; + + log_file << static_cast(ms_f) << " " << block_process_time + transactions_process_time << std::endl; } } - size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; - handler_request_blocks_now(count_limit); // XXX request_missing_objects(context, true); return 1; } @@ -480,6 +580,15 @@ namespace cryptonote template bool t_cryptonote_protocol_handler::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks) { + //if (!m_one_request == false) + //return true; + m_one_request = false; + // save request size to log (dr monero) + /*using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto sec = duration_cast< seconds >( time_from_epoh ).count();*/ + if(context.m_needed_objects.size()) { //we know objects that we need, request this objects @@ -487,11 +596,8 @@ namespace cryptonote size_t count = 0; auto it = context.m_needed_objects.begin(); - size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; - //handler_request_blocks_now( count_limit ); // change the limit, sleep(?) XXX - // XXX - count_limit=200; // XXX - _note_c("net/req-calc" , "Setting count_limit: " << count_limit); + size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; + _note_c("net/req-calc" , "Setting count_limit: " << count_limit); while(it != context.m_needed_objects.end() && count < BLOCKS_SYNCHRONIZING_DEFAULT_COUNT) { if( !(check_having_blocks && m_core.have_block(*it))) @@ -504,6 +610,8 @@ namespace cryptonote } LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() << "requested blocks count=" << count << " / " << count_limit); + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + post_notify(req, context); }else if(context.m_last_response_height < context.m_remote_blockchain_height-1) {//we have to fetch more objects ids, request blockchain entry @@ -511,6 +619,12 @@ namespace cryptonote NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized(); m_core.get_short_chain_history(r.block_ids); handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + + //std::string blob; // for calculate size of request + //epee::serialization::store_t_to_binary(r, blob); + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + LOG_PRINT_CCONTEXT_L0("r = " << 200); + LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify(r, context); }else diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 9f2ae316..27a1129d 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -69,7 +69,9 @@ namespace , "Run on testnet. The wallet must be launched with --testnet flag." , false }; - const command_line::arg_descriptor arg_dns_checkpoints = {"enforce-dns-checkpointing", "checkpoints from DNS server will be enforced", false}; + const command_line::arg_descriptor arg_dns_checkpoints = {"enforce-dns-checkpointing", "checkpoints from DNS server will be enforced", false}; + const command_line::arg_descriptor arg_test_drop_download = {"test-drop-download", "For network testing, drop downloaded blocks instead checking/adding them to blockchain. Can fake-download blocks very fast."}; + const command_line::arg_descriptor arg_save_graph = {"save-graph", "Save data for dr monero", false}; } bool command_line_preprocessor(const boost::program_options::variables_map& vm) @@ -99,6 +101,8 @@ bool command_line_preprocessor(const boost::program_options::variables_map& vm) else if (log_space::get_set_log_detalisation_level(false) != new_log_level) { log_space::get_set_log_detalisation_level(true, new_log_level); + int otshell_utils_log_level = 100 - (new_log_level * 25); + gCurrentLogger.setDebugLevel(otshell_utils_log_level); LOG_PRINT_L0("LOG_LEVEL set to " << new_log_level); } @@ -107,7 +111,7 @@ bool command_line_preprocessor(const boost::program_options::variables_map& vm) int main(int argc, char* argv[]) { - + string_tools::set_module_name_and_folder(argv[0]); #ifdef WIN32 _CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF ); @@ -137,6 +141,8 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_console); command_line::add_arg(desc_cmd_sett, arg_testnet_on); command_line::add_arg(desc_cmd_sett, arg_dns_checkpoints); + command_line::add_arg(desc_cmd_sett, arg_test_drop_download); + command_line::add_arg(desc_cmd_sett, arg_save_graph); cryptonote::core::init_options(desc_cmd_sett); cryptonote::core_rpc_server::init_options(desc_cmd_sett); @@ -231,7 +237,17 @@ int main(int argc, char* argv[]) cryptonote::core_rpc_server rpc_server {ccore, p2psrv, testnet_mode}; cprotocol.set_p2p_endpoint(&p2psrv); ccore.set_cryptonote_protocol(&cprotocol); - daemon_cmmands_handler dch(p2psrv, testnet_mode); + std::shared_ptr dch(new daemon_cmmands_handler(p2psrv, testnet_mode)); + if(command_line::has_arg(vm, arg_save_graph)) + p2psrv.set_save_graph(true); + + //initialize core here + LOG_PRINT_L0("Initializing core..."); + res = ccore.init(vm, testnet_mode); + CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core"); + if (command_line::get_arg(vm, arg_test_drop_download)) + ccore.no_check_blocks(); + LOG_PRINT_L0("Core initialized OK"); //initialize objects LOG_PRINT_L0("Initializing P2P server..."); @@ -248,17 +264,11 @@ int main(int argc, char* argv[]) res = rpc_server.init(vm); CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core RPC server."); LOG_PRINT_GREEN("Core RPC server initialized OK on port: " << rpc_server.get_binded_port(), LOG_LEVEL_0); - - //initialize core here - LOG_PRINT_L0("Initializing core..."); - res = ccore.init(vm, testnet_mode); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core"); - LOG_PRINT_L0("Core initialized OK"); - + // start components if(!command_line::has_arg(vm, arg_console)) { - dch.start_handling(); + dch->start_handling(); } LOG_PRINT_L0("Starting core RPC server..."); @@ -267,7 +277,7 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Core RPC server started ok"); tools::signal_handler::install([&dch, &p2psrv] { - dch.stop_handling(); + dch->stop_handling(); p2psrv.send_stop_signal(); }); @@ -276,6 +286,8 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("P2P net loop stopped"); //stop components + dch->stop_handling(); + dch.reset(); LOG_PRINT_L0("Stopping core rpc server..."); rpc_server.send_stop_signal(); rpc_server.timed_wait_server_stop(5000); diff --git a/src/daemon/daemon_commands_handler.h b/src/daemon/daemon_commands_handler.h index 7cba4ec5..6afbbb07 100644 --- a/src/daemon/daemon_commands_handler.h +++ b/src/daemon/daemon_commands_handler.h @@ -79,6 +79,10 @@ public: m_cmd_binder.set_handler("limit_up", boost::bind(&daemon_cmmands_handler::limit_up, this, _1), "Set upload limit [kB/s]"); m_cmd_binder.set_handler("limit_down", boost::bind(&daemon_cmmands_handler::limit_down, this, _1), "Set download limit [kB/s]"); m_cmd_binder.set_handler("limit", boost::bind(&daemon_cmmands_handler::limit, this, _1), "Set download and upload limit [kB/s]"); + m_cmd_binder.set_handler("fast_exit", boost::bind(&daemon_cmmands_handler::fast_exit, this, _1), "Exit"); + m_cmd_binder.set_handler("test_drop_download", boost::bind(&daemon_cmmands_handler::test_drop_download, this, _1), "For network testing, drop downloaded blocks instead checking/adding them to blockchain. Can fake-download blocks very fast."); + m_cmd_binder.set_handler("start_save_graph", boost::bind(&daemon_cmmands_handler::start_save_graph, this, _1), ""); + m_cmd_binder.set_handler("stop_save_graph", boost::bind(&daemon_cmmands_handler::stop_save_graph, this, _1), ""); } bool start_handling() @@ -240,6 +244,8 @@ private: } log_space::log_singletone::get_set_log_detalisation_level(true, l); + int otshell_utils_log_level = 100 - (l * 25); + gCurrentLogger.setDebugLevel(otshell_utils_log_level); return true; } @@ -406,7 +412,7 @@ private: //-------------------------------------------------------------------------------- bool out_peers_limit(const std::vector& args) { if(args.size()!=1) { - std::cout << "Usage: limit_down " << ENDL; + std::cout << "Usage: out_peers " << ENDL; return true; } @@ -420,13 +426,26 @@ private: return false; } + using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto ms = duration_cast< milliseconds >( time_from_epoh ).count(); + double ms_f = ms; + ms_f /= 1000.; + + std::ofstream limitFile("log/dr-monero/peers_limit.info", std::ios::app); + limitFile.precision(7); + limitFile << ms_f << " " << static_cast(limit) << std::endl; if (m_srv.m_config.m_net_config.connections_count > limit) { - int count = m_srv.m_config.m_net_config.connections_count - limit; m_srv.m_config.m_net_config.connections_count = limit; - m_srv.delete_connections(count); - } - else + if (m_srv.m_number_of_out_peers > limit) + { + int count = m_srv.m_number_of_out_peers - limit; + m_srv.delete_connections(count); + } + } + else m_srv.m_config.m_net_config.connections_count = limit; return true; @@ -527,4 +546,29 @@ private: return true; } + //-------------------------------------------------------------------------------- + bool fast_exit(const std::vector& args) + { + m_srv.get_payload_object().get_core().set_fast_exit(); + m_srv.send_stop_signal(); + return true; + } + //-------------------------------------------------------------------------------- + bool test_drop_download(const std::vector& args) + { + m_srv.get_payload_object().get_core().no_check_blocks(); + return true; + } + //-------------------------------------------------------------------------------- + bool start_save_graph(const std::vector& args) + { + m_srv.set_save_graph(true); + return true; + } + //-------------------------------------------------------------------------------- + bool stop_save_graph(const std::vector& args) + { + m_srv.set_save_graph(false); + return true; + } }; diff --git a/src/p2p/connection_basic.cpp b/src/p2p/connection_basic.cpp index 35b0d4c8..0e2fd594 100644 --- a/src/p2p/connection_basic.cpp +++ b/src/p2p/connection_basic.cpp @@ -78,6 +78,7 @@ #include "../../contrib/epee/include/net/abstract_tcp_server2.h" #include "../../contrib/otshell_utils/utils.hpp" +#include "data_logger.hpp" using namespace nOT::nUtils; // TODO: @@ -146,31 +147,31 @@ connection_basic::connection_basic(boost::asio::io_service& io_service, std::ato { ++ref_sock_count; // increase the global counter mI->m_peer_number = sock_number.fetch_add(1); // use, and increase the generated number - _note("Spawned connection p2p#"<m_peer_number<<" currently we have sockets count:" << m_ref_sock_count); + + string remote_addr_str = "?"; + try { remote_addr_str = socket_.remote_endpoint().address().to_string(); } catch(...){} ; + + _note("Spawned connection p2p#"<m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_ref_sock_count); boost::filesystem::create_directories("log/dr-monero/net/"); - /*boost::asio::SettableSocketOption option;// = new boost::asio::SettableSocketOption(); - option.level(IPPROTO_IP); - option.name(IP_TOS); - option.value(&tos); - option.size = sizeof(tos); - socket_.set_option(option);*/ - // TODO socket options } connection_basic::~connection_basic() { - _note("Destructing connection p2p#"<m_peer_number); + string remote_addr_str = "?"; + try { remote_addr_str = socket_.remote_endpoint().address().to_string(); } catch(...){} ; + _note("Destructing connection p2p#"<m_peer_number << " to " << remote_addr_str); } void connection_basic::set_rate_up_limit(uint64_t limit) { - save_limit_to_file(limit); + + // TODO remove __SCALING_FACTOR... + const double SCALING_FACTOR = 2.1; // to acheve the best performance + limit *= SCALING_FACTOR; { - // TODO remove __SCALING_FACTOR... - const double SCALING_FACTOR = 2.25; // to acheve the best performance - limit *= SCALING_FACTOR; CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); network_throttle_manager::get_global_throttle_out().set_target_speed(limit); + network_throttle_manager::get_global_throttle_out().set_real_target_speed(limit / SCALING_FACTOR); } - // connection_basic_pimpl::m_throttle_global.m_out.set_target_speed(limit); + save_limit_to_file(limit); } void connection_basic::set_rate_down_limit(uint64_t limit) { @@ -186,36 +187,30 @@ void connection_basic::set_rate_down_limit(uint64_t limit) { save_limit_to_file(limit); } -void connection_basic::set_rate_limit(uint64_t limit) { - // TODO -} -void connection_basic::set_kill_limit (uint64_t limit) { - { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); - network_throttle_manager::get_global_throttle_in().set_target_kill(limit); - } - - { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); - network_throttle_manager::get_global_throttle_out().set_target_kill(limit); - } - - { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); - network_throttle_manager::get_global_throttle_inreq().set_target_kill(limit); - } -} void connection_basic::save_limit_to_file(int limit) { // saving limit to file - std::ofstream file; - file.open("log/dr-monero/limit.info"); - file << limit; -} + if (!epee::net_utils::data_logger::m_save_graph) + return; + std::ofstream file_up, file_down; + file_up.open("log/dr-monero/limit_up.info", std::ofstream::out | std::ofstream::app); + file_up.precision(8); + file_down.open("log/dr-monero/limit_down.info", std::ofstream::out | std::ofstream::app); + file_down.precision(8); + using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto s = duration_cast< seconds >( time_from_epoh ).count(); -void connection_basic::set_rate_autodetect(uint64_t limit) { - // TODO - LOG_PRINT_L0("inside connection_basic we set autodetect (this is additional notification).."); + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + file_up << s << " " << network_throttle_manager::get_global_throttle_out().get_terget_speed() / 1024 << "\n"; + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + file_down << s << " " << network_throttle_manager::get_global_throttle_in().get_terget_speed() / 1024 << "\n"; + } } void connection_basic::set_tos_flag(int tos) { @@ -230,39 +225,30 @@ void connection_basic::sleep_before_packet(size_t packet_size, int phase, int q double delay=0; // will be calculated do { // rate limiting - //XXX - /*if (::cryptonote::core::get_is_stopping()) { - _dbg1("We are stopping - so abort sleep"); - return; - }*/ if (m_was_shutdown) { _dbg2("m_was_shutdown - so abort sleep"); return; } { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global } - delay *= 0.50; - delay = 0; // XXX if (delay > 0) { - //delay += rand2*0.1; - long int ms = (long int)(delay * 1000); - _info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="< 0); // XXX LATER XXX { CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); - network_throttle_manager::get_global_throttle_out().handle_trafic_tcp( packet_size ); // increase counter - global - //epee::critical_region_t guard(m_throttle_global_lock); // *** critical *** - //m_throttle_global.m_out.handle_trafic_tcp( packet_size ); // increase counter - global + network_throttle_manager::get_global_throttle_out().handle_trafic_exact( packet_size * 700); // increase counter - global } } @@ -271,34 +257,12 @@ void connection_basic::set_start_time() { m_start_time = network_throttle_manager::get_global_throttle_out().get_time_seconds(); } -void connection_basic::do_send_handler_start(const void* ptr , size_t cb ) { - _fact_c("net/out/size", "*** do_sen() called for packet="< max sending time - //if (sending_time > 0.1) network_throttle_manager::get_global_throttle_out().set_overheat(sending_time); // TODO - -} - void connection_basic::do_send_handler_write_from_queue( const boost::system::error_code& e, size_t cb, int q_len ) { sleep_before_packet(cb,2,q_len); _info_c("net/out/size", "handler_write (after write, from queue="<m_throttle_global_lock)> guard(mI->m_throttle_global_lock); // *** critical *** - // mI->m_throttle_global.m_in.handle_trafic_tcp( packet_size ); // increase counter - global - } -} - -void connection_basic::logger_handle_net_peer(size_t size, bool io) { // network data written - // TODO OPTIMIZE! do NOT reopen idiotically :) - std::ostringstream oss; - std::string filename; - if (io) { // write - double time = network_throttle_manager::get_global_throttle_in().get_time_seconds() ; - oss << "log/dr-monero/net/in-peer-" << (mI->m_peer_number) << ".dat" << std::ends; - filename = oss.str(); - network_throttle_manager::get_global_throttle_out().logger_handle_net(filename,time,size); - } - else { // read - double time = network_throttle_manager::get_global_throttle_out().get_time_seconds() ; - oss << "log/dr-monero/net/out-peer-" << (mI->m_peer_number) << ".dat" << std::ends; - filename = oss.str(); - network_throttle_manager::get_global_throttle_in().logger_handle_net(filename,time,size); - } -} - void connection_basic::logger_handle_net_read(size_t size) { // network data read - std::string filename = "log/dr-monero/net/in-all.data"; - - double time = network_throttle_manager::get_global_throttle_in().get_time_seconds() ; - network_throttle_manager::get_global_throttle_in().logger_handle_net(filename, time, size); - logger_handle_net_peer(size,0); + size /= 1024; + epee::net_utils::data_logger::get_instance().add_data("download", size); } void connection_basic::logger_handle_net_write(size_t size) { - std::string filename = "log/dr-monero/net/out-all.data"; - double time = network_throttle_manager::get_global_throttle_out().get_time_seconds() ; - network_throttle_manager::get_global_throttle_out().logger_handle_net(filename, time, size); - logger_handle_net_peer(size,1); - + size /= 1024; + epee::net_utils::data_logger::get_instance().add_data("upload", size); } double connection_basic::get_sleep_time(size_t cb) { @@ -356,6 +285,10 @@ double connection_basic::get_sleep_time(size_t cb) { return t; } +void connection_basic::set_save_graph(bool save_graph) { + epee::net_utils::data_logger::m_save_graph = save_graph; +} + } // namespace } // namespace diff --git a/src/p2p/connection_basic.hpp b/src/p2p/connection_basic.hpp index 1b5a2c8a..e9fdc3ad 100644 --- a/src/p2p/connection_basic.hpp +++ b/src/p2p/connection_basic.hpp @@ -99,17 +99,11 @@ class connection_basic { // not-templated base class for rapid developmet of som virtual ~connection_basic(); // various handlers to be called from connection class: - void do_send_handler_start(const void * ptr , size_t cb); - void do_send_handler_delayed(const void * ptr , size_t cb); void do_send_handler_write(const void * ptr , size_t cb); - void do_send_handler_stop(const void * ptr , size_t cb); - void do_send_handler_after_write( const boost::system::error_code& e, size_t cb ); // from handle_write void do_send_handler_write_from_queue(const boost::system::error_code& e, size_t cb , int q_len); // from handle_write, sending next part - void do_read_handler_start(const boost::system::error_code& e, std::size_t bytes_transferred); // from read, after read completion void logger_handle_net_write(size_t size); // network data written void logger_handle_net_read(size_t size); // network data read - void logger_handle_net_peer(size_t size, bool io); void set_start_time(); @@ -117,9 +111,6 @@ class connection_basic { // not-templated base class for rapid developmet of som static void set_rate_up_limit(uint64_t limit); static void set_rate_down_limit(uint64_t limit); - static void set_rate_limit(uint64_t limit); - static void set_rate_autodetect(uint64_t limit); - static void set_kill_limit (uint64_t limit); // config misc static void set_tos_flag(int tos); // ToS / QoS flag @@ -129,6 +120,8 @@ class connection_basic { // not-templated base class for rapid developmet of som void sleep_before_packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) static void save_limit_to_file(int limit); ///< for dr-monero static double get_sleep_time(size_t cb); + + static void set_save_graph(bool save_graph); }; } // nameserver diff --git a/src/p2p/data_logger.cpp b/src/p2p/data_logger.cpp new file mode 100644 index 00000000..6a8eb25b --- /dev/null +++ b/src/p2p/data_logger.cpp @@ -0,0 +1,81 @@ +#include "data_logger.hpp" + +#include +#include + +namespace epee +{ +namespace net_utils +{ + data_logger &data_logger::get_instance() + { + static data_logger instance; + return instance; + } + + data_logger::data_logger() + { + //create timer + std::shared_ptr logger_thread(new std::thread([&]() + { + while (true) + { + std::this_thread::sleep_for(std::chrono::seconds(1)); + saveToFile(); + } + })); + logger_thread->detach(); + + mFilesMap["peers"] = data_logger::fileData("log/dr-monero/peers.data"); + mFilesMap["download"] = data_logger::fileData("log/dr-monero/net/in-all.data"); + mFilesMap["upload"] = data_logger::fileData("log/dr-monero/net/out-all.data"); + mFilesMap["request"] = data_logger::fileData("log/dr-monero/net/req-all.data"); + mFilesMap["sleep_down"] = data_logger::fileData("log/dr-monero/down_sleep_log.data"); + mFilesMap["sleep_up"] = data_logger::fileData("log/dr-monero/up_sleep_log.data"); + + } + + void data_logger::add_data(std::string filename, unsigned int data) + { + if (mFilesMap.find(filename) == mFilesMap.end()) + return; // TODO: exception + + mFilesMap[filename].mDataToSave += data; + } + + double data_logger::fileData::get_current_time() + { + using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto ms = duration_cast< milliseconds >( time_from_epoh ).count(); + double ms_f = ms; + return ms_f / 1000.; + } + + data_logger::fileData::fileData(std::string pFile) + { + mFile = std::make_shared (pFile); + } + + void data_logger::fileData::save() + { + if (!data_logger::m_save_graph) + return; + *mFile << static_cast(get_current_time()) << " " << mDataToSave << std::endl; + } + + void data_logger::saveToFile() + { + std::lock_guard lock(mSaveMutex); + for (auto &element : mFilesMap) + { + element.second.save(); + element.second.mDataToSave = 0; + } + } + +std::atomic data_logger::m_save_graph(false); + +} // namespace +} // namespace diff --git a/src/p2p/data_logger.hpp b/src/p2p/data_logger.hpp new file mode 100644 index 00000000..2b8503df --- /dev/null +++ b/src/p2p/data_logger.hpp @@ -0,0 +1,46 @@ +#ifndef INCLUDED_p2p_data_logger_hpp +#define INCLUDED_p2p_data_logger_hpp + +#include +#include +#include +#include +#include +#include +#include + +namespace epee +{ +namespace net_utils +{ + + class data_logger { + public: + static data_logger &get_instance(); + data_logger(const data_logger &ob) = delete; + data_logger(data_logger &&ob) = delete; + void add_data(std::string filename, unsigned int data); + static std::atomic m_save_graph; + private: + data_logger(); + class fileData + { + public: + fileData(){} + fileData(std::string pFile); + + std::shared_ptr mFile; + long int mDataToSave = 0; + static double get_current_time(); + void save(); + }; + + std::map mFilesMap; + std::mutex mSaveMutex; + void saveToFile(); + }; + +} // namespace +} // namespace + +#endif diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 48737193..ea7d5c38 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -86,7 +86,10 @@ namespace nodetool m_no_igd(false), m_hide_my_port(false), m_network_id(std::move(network_id)) - {} + { + m_number_of_out_peers = 0; + m_save_graph = false; + } static void init_options(boost::program_options::options_description& desc); @@ -225,6 +228,12 @@ namespace nodetool public: config m_config; // TODO was private, add getters? + std::atomic m_number_of_out_peers; + void set_save_graph(bool save_graph) + { + m_save_graph = save_graph; + epee::net_utils::connection_basic::set_save_graph(save_graph); + } private: std::string m_config_folder; @@ -237,6 +246,7 @@ namespace nodetool bool m_allow_local_ip; bool m_hide_my_port; bool m_no_igd; + std::atomic m_save_graph; //critical_section m_connections_lock; //connections_indexed_container m_connections; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index ce70e241..60eed1f3 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -46,6 +46,7 @@ #include "net/local_ip.h" #include "crypto/crypto.h" #include "storages/levin_abstract_invoke2.h" +#include "data_logger.hpp" // We have to look for miniupnpc headers in different places, dependent on if its compiled or external #ifdef UPNP_STATIC @@ -85,8 +86,8 @@ namespace nodetool const command_line::arg_descriptor > arg_p2p_seed_node = {"seed-node", "Connect to a node to retrieve peer addresses, and disconnect"}; const command_line::arg_descriptor arg_p2p_hide_my_port = {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true}; - const command_line::arg_descriptor arg_no_igd = {"no-igd", "Disable UPnP port mapping"}; - const command_line::arg_descriptor arg_out_peers = {"out-peers", "set max limit of out peers", -1}; + const command_line::arg_descriptor arg_no_igd = {"no-igd", "Disable UPnP port mapping"}; + const command_line::arg_descriptor arg_out_peers = {"out-peers", "set max limit of out peers", -1}; const command_line::arg_descriptor arg_tos_flag = {"tos-flag", "set TOS flag", -1}; const command_line::arg_descriptor arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", -1}; @@ -289,6 +290,31 @@ namespace nodetool std::vector> dns_results; dns_results.resize(m_seed_nodes_list.size()); + + std::shared_ptr peersLoggerThread (new std::thread([&]() + { + unsigned int number_of_peers; + while (1) + { + if (m_save_graph) + { + //number_of_peers = m_net_server.get_config_object().get_connections_count(); + number_of_peers = 0; + m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if(!cntxt.m_is_income) + ++number_of_peers; + return true; + }); // lambda + + m_number_of_out_peers = number_of_peers; + epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers); + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + })); // lambda + + peersLoggerThread->detach(); std::list dns_threads; uint64_t result_index = 0; @@ -487,6 +513,7 @@ namespace nodetool { m_peerlist.deinit(); m_net_server.deinit_server(); + return store_config(); } //----------------------------------------------------------------------------------- @@ -697,6 +724,16 @@ namespace nodetool template bool node_server::try_to_connect_and_handshake_with_new_peer(const net_address& na, bool just_take_peerlist, uint64_t last_seen_stamp, bool white) { + if (m_number_of_out_peers == m_config.m_net_config.connections_count) // out peers limit + { + return false; + } + else if (m_number_of_out_peers > m_config.m_net_config.connections_count) + { + m_net_server.get_config_object().del_out_connections(1); + m_number_of_out_peers --; // atomic variable, update time = 1s + return false; + } LOG_PRINT_L1("Connecting to " << epee::string_tools::get_ip_string_from_int32(na.ip) << ":" << epee::string_tools::num_to_string_fast(na.port) << "(white=" << white << ", last_seen: " << (last_seen_stamp ? epee::misc_utils::get_time_interval_string(time(NULL) - last_seen_stamp):"never") @@ -784,16 +821,22 @@ namespace nodetool ++try_count; - if(is_peer_used(pe)) + _note("Considering connecting (out) to peer: " << pe.id << " " << epee::string_tools::get_ip_string_from_int32(pe.adr.ip) << ":" << boost::lexical_cast(pe.adr.port)); + + if(is_peer_used(pe)) { + _note("Peer is used"); continue; + } LOG_PRINT_L1("Selected peer: " << pe.id << " " << epee::string_tools::get_ip_string_from_int32(pe.adr.ip) << ":" << boost::lexical_cast(pe.adr.port) << "[white=" << use_white_list << "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never")); - if(!try_to_connect_and_handshake_with_new_peer(pe.adr, false, pe.last_seen, use_white_list)) + if(!try_to_connect_and_handshake_with_new_peer(pe.adr, false, pe.last_seen, use_white_list)) { + _note("Handshake failed"); continue; + } return true; } @@ -1336,20 +1379,31 @@ namespace nodetool template bool node_server::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max) { + using namespace std::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto ms = duration_cast< milliseconds >( time_from_epoh ).count(); + double ms_f = ms; + ms_f /= 1000.; + + std::ofstream limitFile("log/dr-monero/peers_limit.info", std::ios::app); + limitFile.precision(7); if(max == -1) { m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT; + if (m_save_graph) + limitFile << static_cast(ms_f) << " " << P2P_DEFAULT_CONNECTIONS_COUNT << std::endl; return true; } m_config.m_net_config.connections_count = max; - LOG_PRINT_RED_L0("connections_count: " << m_config.m_net_config.connections_count); + limitFile << static_cast(ms_f) << " " << max << std::endl; return true; } template void node_server::delete_connections(size_t count) { - m_net_server.get_config_object().del_connections(count); + m_net_server.get_config_object().del_out_connections(count); } template diff --git a/src/p2p/network_throttle-detail.cpp b/src/p2p/network_throttle-detail.cpp index 6ea3076a..6b2ee698 100644 --- a/src/p2p/network_throttle-detail.cpp +++ b/src/p2p/network_throttle-detail.cpp @@ -78,6 +78,7 @@ #include "../../src/p2p/network_throttle-detail.hpp" #include "../../contrib/otshell_utils/utils.hpp" +#include "data_logger.hpp" using namespace nOT::nUtils; // ################################################################################################ @@ -152,8 +153,6 @@ network_throttle::network_throttle(const std::string &nameshort, const std::stri m_any_packet_yet = false; m_slot_size = 1.0; // hard coded in few places m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle - m_target_MB = 0; - } void network_throttle::set_name(const std::string &name) @@ -163,16 +162,20 @@ void network_throttle::set_name(const std::string &name) void network_throttle::set_target_speed( network_speed_kbps target ) { - m_target_speed = target; + m_target_speed = target * 1024; _note_c("net/"+m_nameshort, "Setting LIMIT: " << target << " kbps"); + set_real_target_speed(target); } -void network_throttle::set_target_kill( network_MB target ) +void network_throttle::set_real_target_speed( network_speed_kbps real_target ) { - _note_c("net/"+m_nameshort, "Setting KILL: " << target << " MB hard limit"); - m_target_MB = target; + m_real_target_speed = real_target * 1024; } +network_speed_kbps network_throttle::get_terget_speed() +{ + return m_real_target_speed / 1024; +} void network_throttle::tick() { @@ -187,7 +190,7 @@ void network_throttle::tick() // TODO optimize when moving few slots at once while ( (!m_any_packet_yet) || (last_sample_time_slot < current_sample_time_slot)) { - LOG_PRINT_L4("Moving counter buffer by 1 second " << last_sample_time_slot << " < " << current_sample_time_slot << " (last time " << m_last_sample_time<<")"); + _dbg3("Moving counter buffer by 1 second " << last_sample_time_slot << " < " << current_sample_time_slot << " (last time " << m_last_sample_time<<")"); // rotate buffer for (size_t i=m_history.size()-1; i>=1; --i) m_history[i] = m_history[i-1]; m_history[0] = packet_info(); @@ -217,7 +220,6 @@ void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_s std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; std::string history_str = oss.str(); - logger_handle_net("log/dr-monero/net/inreq-all.data",get_time_seconds(),packet_size); _info_c( "net/" + m_nameshort , "Throttle " << m_name << ": packet of ~"<(time) << " " << static_cast(size/1024) << "\n"; file.close(); } mutex.unlock(); } @@ -257,27 +258,11 @@ void network_throttle::logger_handle_net(const std::string &filename, double tim // fine tune this to decide about sending speed: network_time_seconds network_throttle::get_sleep_time(size_t packet_size) const { - //_scope_mark(""); double D2=0; calculate_times_struct cts = { 0, 0, 0, 0}; - //calculate_times(packet_size, cts, false, m_window_size/2); D2=cts.delay; - //calculate_times(packet_size, cts, true, m_window_size/2); D2=cts.delay; calculate_times(packet_size, cts, true, m_window_size); D2=cts.delay; return D2; } -double network_throttle::get_current_overheat() const { - auto now = get_time_seconds(); - auto diff = now - m_overheat_time; - auto overheat = m_overheat - diff; - overheat = std::max(m_overheat, 0.); - return overheat; -} - -void network_throttle::set_overheat(double lag) { - m_overheat += lag; - m_overheat_time = get_time_seconds(); - LOG_PRINT_L0("Lag: " << lag << ", overheat: " << m_overheat ); -} // MAIN LOGIC: void network_throttle::calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const @@ -310,9 +295,7 @@ void network_throttle::calculate_times(size_t packet_size, calculate_times_struc const double D1 = (Epast - M*cts.window) / M; // delay - how long to sleep to get back to target speed const double D2 = (Enow - M*cts.window) / M; // delay - how long to sleep to get back to target speed (including current packet) - auto O = get_current_overheat(); - auto Ouse = O * 0 ; // XXX TODO - cts.delay = (D1*0.80 + D2*0.20) + Ouse; // finall sleep depends on both with/without current packet + cts.delay = (D1*0.80 + D2*0.20); // finall sleep depends on both with/without current packet // update_overheat(); cts.average = Epast/cts.window; // current avg. speed (for info) @@ -329,13 +312,13 @@ void network_throttle::calculate_times(size_t packet_size, calculate_times_struc if (dbg) { std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; std::string history_str = oss.str(); - _dbg1_c( "net/"+m_nameshort+"_c" , - "dbg " << m_name << ": " + _info_c( "net/"+m_nameshort+"_c" , + (cts.delay > 0 ? "SLEEP" : "") + << "dbg " << m_name << ": " << "speed is A=" << std::setw(8) <( time_from_epoh ).count(); @@ -368,14 +351,28 @@ size_t network_throttle::get_recommended_size_of_planned_transport_window(double size_t network_throttle::get_recommended_size_of_planned_transport() const { size_t R1=0,R2=0,R3=0; R1 = get_recommended_size_of_planned_transport_window( -1 ); - R2 = get_recommended_size_of_planned_transport_window( m_window_size/2); - R3 = get_recommended_size_of_planned_transport_window( 8 ); + R2 = get_recommended_size_of_planned_transport_window(m_window_size / 2); + R3 = get_recommended_size_of_planned_transport_window( 5 ); auto RM = std::min(R1, std::min(R2,R3)); - const double a1=70, a2=10, a3=10, am=10; // weight of the various windows in decisssion + const double a1=20, a2=10, a3=10, am=10; // weight of the various windows in decisssion // TODO 70 => 20 return (R1*a1 + R2*a2 + R3*a3 + RM*am) / (a1+a2+a3+am); } +double network_throttle::get_current_speed() const { + unsigned int bytes_transferred = 0; + if (m_history.size() == 0 || m_slot_size == 0) + return 0; + + auto it = m_history.begin(); + while (it < m_history.end() - 1) + { + bytes_transferred += it->m_size; + it ++; + } + + return bytes_transferred / ((m_history.size() - 1) * m_slot_size); +} } // namespace } // namespace diff --git a/src/p2p/network_throttle-detail.hpp b/src/p2p/network_throttle-detail.hpp index 9d492c53..063dac85 100644 --- a/src/p2p/network_throttle-detail.hpp +++ b/src/p2p/network_throttle-detail.hpp @@ -54,7 +54,7 @@ class network_throttle : public i_network_throttle { network_speed_kbps m_target_speed; - network_MB m_target_MB; + network_speed_kbps m_real_target_speed; size_t m_network_add_cost; // estimated add cost of headers size_t m_network_minimal_segment; // estimated minimal cost of sending 1 byte to round up to size_t m_network_max_segment; // recommended max size of 1 TCP transmission @@ -80,18 +80,16 @@ class network_throttle : public i_network_throttle { virtual ~network_throttle(); virtual void set_name(const std::string &name); virtual void set_target_speed( network_speed_kbps target ); - virtual void set_target_kill( network_MB target ); + virtual void set_real_target_speed( network_speed_kbps real_target ); // only for throttle_out + virtual network_speed_kbps get_terget_speed(); // add information about events: virtual void handle_trafic_exact(size_t packet_size); ///< count the new traffic/packet; the size is exact considering all network costs virtual void handle_trafic_tcp(size_t packet_size); ///< count the new traffic/packet; the size is as TCP, we will consider MTU etc - virtual void handle_congestion(double overheat); ///< call this when congestion is detected; see example use virtual void tick(); ///< poke and update timers/history (recalculates, moves the history if needed, checks the real clock etc) virtual double get_time_seconds() const ; ///< timer that we use, time in seconds, monotionic - virtual double get_current_overheat() const; ///< did we detected congestion now. NOT USED NOW TODO - virtual void set_overheat(double lag); ///< did we detected congestion now. NOT USED NOW TODO. rename to add_overheat ? // time calculations: virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const; ///< MAIN LOGIC (see base class for info) @@ -101,7 +99,7 @@ class network_throttle : public i_network_throttle { virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const; ///< ditto, but for given windows time frame - //virtual void add_planned_transport(size_t size); + virtual double get_current_speed() const; private: virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13 diff --git a/src/p2p/network_throttle.hpp b/src/p2p/network_throttle.hpp index dc25a2c4..add4daa8 100644 --- a/src/p2p/network_throttle.hpp +++ b/src/p2p/network_throttle.hpp @@ -100,7 +100,7 @@ struct calculate_times_struct { typedef calculate_times_struct calculate_times_struct; -namespace cryptonote { class cryptonote_protocol_handler_base; }; // a friend class // TODO friend not working +namespace cryptonote { class cryptonote_protocol_handler_base; } // a friend class // TODO friend not working /*** @brief Access to simple throttles, with singlton to access global network limits @@ -146,11 +146,11 @@ class i_network_throttle { public: virtual void set_name(const std::string &name)=0; virtual void set_target_speed( network_speed_kbps target )=0; - virtual void set_target_kill( network_MB target )=0; + virtual void set_real_target_speed(network_speed_kbps real_target)=0; + virtual network_speed_kbps get_terget_speed()=0; virtual void handle_trafic_exact(size_t packet_size) =0; // count the new traffic/packet; the size is exact considering all network costs virtual void handle_trafic_tcp(size_t packet_size) =0; // count the new traffic/packet; the size is as TCP, we will consider MTU etc - virtual void handle_congestion(double overheat) =0; // call this when congestion is detected; see example use virtual void tick() =0; // poke and update timers/history // time calculations: @@ -166,8 +166,6 @@ class i_network_throttle { virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future virtual double get_time_seconds() const =0; // a timer - virtual double get_current_overheat() const =0; - virtual void set_overheat(double lag) =0; virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0; diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index 6e00ad66..5cd4988e 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -42,8 +42,8 @@ #include "warnings.h" /* I have no clue what these lines means */ -PUSH_WARNINGS; -DISABLE_VS_WARNINGS(4244); +PUSH_WARNINGS +DISABLE_VS_WARNINGS(4244) //TODO: fix size_t warning in x32 platform From 39fc63f48dd570920d04a4e5f1eb7aab5f11d5c4 Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Thu, 12 Feb 2015 21:34:00 +0100 Subject: [PATCH 03/16] removed not needed --- src/cryptonote_protocol/cryptonote_protocol_handler.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 0be864df..ada26967 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -45,7 +45,6 @@ #include "cryptonote_core/connection_context.h" #include "cryptonote_core/cryptonote_stat_info.h" #include "cryptonote_core/verification_context.h" -#include #include PUSH_WARNINGS From 0f06dca83197e317384609c8c7e1413289a1be03 Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Thu, 12 Feb 2015 21:53:52 +0100 Subject: [PATCH 04/16] fixed size_t on windows thought it was already fixed, apparently commit got lost somewhere --- contrib/epee/include/net/abstract_tcp_server2.inl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 5855f7f8..3dff6da5 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -357,7 +357,10 @@ PRAGMA_WARNING_DISABLE_VS(4355) const t_safe chunksize_max = chunksize_good * 2 ; const bool allow_split = (m_connection_type == RPC) ? false : true; // TODO config - if (allow_split && (cb > chunksize_max)) { + ASRT(! (chunksize_max<0) ); // make sure it is unsigned before removin sign with cast: + long long unsigned int chunksize_max_unsigned = static_cast( chunksize_max ) ; + + if (allow_split && (cb > chunksize_max_unsigned)) { { // LOCK: chunking epee::critical_region_t send_guard(m_chunking_lock); // *** critical *** @@ -380,7 +383,10 @@ PRAGMA_WARNING_DISABLE_VS(4355) ASRT(len<=chunksize_good); // pos=8; len=4; all=10; len=3; - ASRT(len>0); ASRT(len < std::numeric_limits::max()); // yeap we want strong < then max size, to be sure + ASRT(! (len<0) ); // check before we cast away sign: + unsigned long long int len_unsigned = static_cast( len ); + ASRT(len>0); // (redundand) + ASRT(len_unsigned < std::numeric_limits::max()); // yeap we want strong < then max size, to be sure void *chunk_start = ((char*)ptr) + pos; _fact_c("net/out/size","chunk_start="< mutex_guard( nOT::nUtils::gLoggerGuard ); \ + part=1; \ + try { \ + ++nOT::nUtils::gLoggerGuardDepth_Get(); \ + std::ostringstream oss; \ + oss << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \ + std::string as_string = oss.str(); \ +/* int counter = nOT::nUtils::gLoggerGuardDepth_Get(); if (counter!=1) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: recursion, counter="< arg_ip = {"ip", "set ip"}; diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index fe80d75a..4e669daa 100644 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -50,6 +50,7 @@ #include "cryptonote_core/checkpoints_create.h" //#include "serialization/json_archive.h" #include "../../contrib/otshell_utils/utils.hpp" +#include "../../src/p2p/data_logger.hpp" using namespace cryptonote; @@ -178,22 +179,6 @@ bool blockchain_storage::store_genesis_block(bool testnet) { return true; } //------------------------------------------------------------------ -void blockchain_storage::logger_handle(long int ms) -{ - std::ofstream log_file; - log_file.open("log/dr-monero/blockchain_log.data", std::ofstream::out | std::ofstream::app); - log_file.precision(7); - - using namespace boost::chrono; - auto point = steady_clock::now(); - auto time_from_epoh = point.time_since_epoch(); - auto m_ms = duration_cast< milliseconds >( time_from_epoh ).count(); - double ms_f = m_ms; - ms_f /= 1000.; - - log_file << ms_f << " " << ms << std::endl; -} -//------------------------------------------------------------------ bool blockchain_storage::store_blockchain() { m_is_blockchain_storing = true; @@ -1776,7 +1761,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size << ", " << block_processing_time << "("<< target_calculating_time << "/" << longhash_calculating_time << ")ms"); - logger_handle(block_processing_time); + epee::net_utils::data_logger::get_instance().add_data("blockchain_processing_time", block_processing_time); bvc.m_added_to_main_chain = true; /*if(!m_orphanes_reorganize_in_work) diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index 6456689b..38bdfbce 100644 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -249,7 +249,6 @@ namespace cryptonote bool complete_timestamps_vector(uint64_t start_height, std::vector& timestamps); bool update_next_comulative_size_limit(); bool store_genesis_block(bool testnet); - void logger_handle(long int ms); }; diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index c8daa351..49ce6306 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -206,14 +206,30 @@ namespace cryptonote return m_fast_exit; } //----------------------------------------------------------------------------------------------- - void core::no_check_blocks() + void core::test_drop_download() { - m_check_blocks = false; + m_test_drop_download = false; } //----------------------------------------------------------------------------------------------- - bool core::get_check_blocks() + void core::test_drop_download_height(uint64_t height) { - return m_check_blocks; + m_test_drop_download_height = height; + } + //----------------------------------------------------------------------------------------------- + bool core::get_test_drop_download() + { + return m_test_drop_download; + } + //----------------------------------------------------------------------------------------------- + bool core::get_test_drop_download_height() + { + if (m_test_drop_download_height == 0) + return true; + + if (get_blockchain_storage().get_current_blockchain_height() <= m_test_drop_download_height) + return true; + + return false; } //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block) diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 9218aef0..795a6d21 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -77,8 +77,10 @@ namespace cryptonote bool deinit(); static void set_fast_exit(); static bool get_fast_exit(); - void no_check_blocks(); - bool get_check_blocks(); + void test_drop_download(); + void test_drop_download_height(uint64_t height); + bool get_test_drop_download(); + bool get_test_drop_download_height(); uint64_t get_current_blockchain_height(); bool get_blockchain_top(uint64_t& heeight, crypto::hash& top_id); bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs); @@ -151,8 +153,8 @@ namespace cryptonote bool check_tx_inputs_keyimages_diff(const transaction& tx); void graceful_exit(); static std::atomic m_fast_exit; - bool m_check_blocks = true; - + bool m_test_drop_download = true; + uint64_t m_test_drop_download_height = 0; tx_memory_pool m_mempool; blockchain_storage m_blockchain_storage; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp index b5a5ceea..614ee8fa 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -128,7 +128,7 @@ void cryptonote_protocol_handler_base::handler_request_blocks_history(std::list< // TODO } -void cryptonote_protocol_handler_base::handler_response_blocks_now(size_t packet_size) { _scope_mark(""); +void cryptonote_protocol_handler_base::handler_response_blocks_now(size_t packet_size) { _scope_dbg1(""); using namespace epee::net_utils; double delay=0; // will be calculated _dbg1("Packet size: " << packet_size); diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index ada26967..571c36dc 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -45,11 +45,14 @@ #include "cryptonote_core/connection_context.h" #include "cryptonote_core/cryptonote_stat_info.h" #include "cryptonote_core/verification_context.h" +// #include #include PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) +#define LOCALHOST_INT 2130706433 + namespace cryptonote { diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 0130cb38..023dd03a 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -42,6 +42,7 @@ #include "profile_tools.h" #include "../../contrib/otshell_utils/utils.hpp" #include "../../src/p2p/network_throttle-detail.hpp" +#include "../../src/p2p/data_logger.hpp" using namespace nOT::nUtils; namespace cryptonote @@ -154,10 +155,10 @@ namespace cryptonote << std::setw(10) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_send_cnt / connection_time / 1024) << std::setw(13) << std::fixed << cntxt.m_current_speed_up / 1024 << (local_ip ? "[LAN]" : "") - << std::left << (ip == 2130706433 ? "[LOCALHOST]" : "") // 127.0.0.1 + << std::left << (ip == LOCALHOST_INT ? "[LOCALHOST]" : "") // 127.0.0.1 << ENDL; - if (connection_time > 0) + if (connection_time > 1) { down_sum += (cntxt.m_recv_cnt / connection_time / 1024); up_sum += (cntxt.m_send_cnt / connection_time / 1024); @@ -397,11 +398,9 @@ namespace cryptonote // calculate size of request - mainly for logging/debug size_t size = 0; - for (auto element : arg.txs) - size += element.size(); + for (auto element : arg.txs) size += element.size(); - for (auto element : arg.blocks) - { + for (auto element : arg.blocks) { size += element.block.size(); for (auto tx : element.txs) size += tx.size(); @@ -491,66 +490,65 @@ namespace cryptonote return 1; } + { m_core.pause_mine(); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler( boost::bind(&t_core::resume_mine, &m_core)); LOG_PRINT_CCONTEXT_YELLOW( "Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() , LOG_LEVEL_0); - BOOST_FOREACH(const block_complete_entry& block_entry, arg.blocks) - { - //process transactions - TIME_MEASURE_START(transactions_process_time); - BOOST_FOREACH(auto& tx_blob, block_entry.txs) - { - tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(tx_blob, tvc, true); - if(tvc.m_verifivation_failed) - { - LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " - << epee::string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); - m_p2p->drop_connection(context); - return 1; - } - } - TIME_MEASURE_FINISH(transactions_process_time); + + if (m_core.get_test_drop_download() && m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing + + + BOOST_FOREACH(const block_complete_entry& block_entry, arg.blocks) + { + // process transactions + TIME_MEASURE_START(transactions_process_time); + BOOST_FOREACH(auto& tx_blob, block_entry.txs) + { + tx_verification_context tvc = AUTO_VAL_INIT(tvc); + m_core.handle_incoming_tx(tx_blob, tvc, true); + if(tvc.m_verifivation_failed) + { + LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " + << epee::string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); + m_p2p->drop_connection(context); + return 1; + } + } + TIME_MEASURE_FINISH(transactions_process_time); - //process block - TIME_MEASURE_START(block_process_time); - block_verification_context bvc = boost::value_initialized(); - - if (m_core.get_check_blocks()) - m_core.handle_incoming_block(block_entry.block, bvc, false); + // process block + + TIME_MEASURE_START(block_process_time); + block_verification_context bvc = boost::value_initialized(); + + m_core.handle_incoming_block(block_entry.block, bvc, false); // <--- process block - if(bvc.m_verifivation_failed) - { - LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection"); - m_p2p->drop_connection(context); - return 1; - } - if(bvc.m_marked_as_orphaned) - { - LOG_PRINT_CCONTEXT_L1("Block received at sync phase was marked as orphaned, dropping connection"); - m_p2p->drop_connection(context); - return 1; - } + if(bvc.m_verifivation_failed) + { + LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection"); + m_p2p->drop_connection(context); + return 1; + } + if(bvc.m_marked_as_orphaned) + { + LOG_PRINT_CCONTEXT_L1("Block received at sync phase was marked as orphaned, dropping connection"); + m_p2p->drop_connection(context); + return 1; + } - TIME_MEASURE_FINISH(block_process_time); - LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms"); - - std::ofstream log_file; - log_file.open("log/dr-monero/get_objects_calc_time.data", std::ofstream::out | std::ofstream::app); - log_file.precision(7); + TIME_MEASURE_FINISH(block_process_time); + LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms"); + + epee::net_utils::data_logger::get_instance().add_data("calc_time", block_process_time + transactions_process_time); + + } // each download block - using namespace boost::chrono; - auto point = steady_clock::now(); - auto time_from_epoh = point.time_since_epoch(); - auto m_ms = duration_cast< milliseconds >( time_from_epoh ).count(); - double ms_f = m_ms; - ms_f /= 1000.; - - log_file << static_cast(ms_f) << " " << block_process_time + transactions_process_time << std::endl; - } + } // if not DISCARD BLOCK + + } request_missing_objects(context, true); return 1; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 27a1129d..8497efbf 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -34,6 +34,7 @@ #include "include_base_utils.h" #include "version.h" +#include "../../contrib/epee/include/syncobj.h" using namespace epee; @@ -57,6 +58,8 @@ using namespace epee; namespace po = boost::program_options; +unsigned int epee::g_test_dbg_lock_sleep = 0; + namespace { const command_line::arg_descriptor arg_config_file = {"config-file", "Specify configuration file", std::string(CRYPTONOTE_NAME ".conf")}; @@ -69,9 +72,11 @@ namespace , "Run on testnet. The wallet must be launched with --testnet flag." , false }; - const command_line::arg_descriptor arg_dns_checkpoints = {"enforce-dns-checkpointing", "checkpoints from DNS server will be enforced", false}; - const command_line::arg_descriptor arg_test_drop_download = {"test-drop-download", "For network testing, drop downloaded blocks instead checking/adding them to blockchain. Can fake-download blocks very fast."}; - const command_line::arg_descriptor arg_save_graph = {"save-graph", "Save data for dr monero", false}; + const command_line::arg_descriptor arg_dns_checkpoints = {"enforce-dns-checkpointing", "checkpoints from DNS server will be enforced", false}; + const command_line::arg_descriptor arg_test_drop_download = {"test-drop-download", "For net tests: in download, discard ALL blocks instead checking/saving them (very fast)"}; + const command_line::arg_descriptor arg_test_drop_download_height = {"test-drop-download-height", "Like test-drop-download but disards only after around certain height", 0}; + const command_line::arg_descriptor arg_save_graph = {"save-graph", "Save data for dr monero", false}; + const command_line::arg_descriptor test_dbg_lock_sleep = {"test-dbg-lock-sleep", "Sleep time in ms, defaults to 0 (off), used to debug before/after locking mutex. Values 100 to 1000 are good for tests.", 0}; } bool command_line_preprocessor(const boost::program_options::variables_map& vm) @@ -111,7 +116,6 @@ bool command_line_preprocessor(const boost::program_options::variables_map& vm) int main(int argc, char* argv[]) { - string_tools::set_module_name_and_folder(argv[0]); #ifdef WIN32 _CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF ); @@ -119,7 +123,10 @@ int main(int argc, char* argv[]) log_space::get_set_log_detalisation_level(true, LOG_LEVEL_0); log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); LOG_PRINT_L0("Starting..."); - + + nOT::nUtils::cFilesystemUtils::CreateDirTree("log/dr-monero/net/"); + // _warn_c("log/test","Starting program"); // TODO + TRY_ENTRY(); boost::filesystem::path default_data_path {tools::get_default_data_dir()}; @@ -142,7 +149,9 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_testnet_on); command_line::add_arg(desc_cmd_sett, arg_dns_checkpoints); command_line::add_arg(desc_cmd_sett, arg_test_drop_download); + command_line::add_arg(desc_cmd_sett, arg_test_drop_download_height); command_line::add_arg(desc_cmd_sett, arg_save_graph); + command_line::add_arg(desc_cmd_sett, test_dbg_lock_sleep); cryptonote::core::init_options(desc_cmd_sett); cryptonote::core_rpc_server::init_options(desc_cmd_sett); @@ -241,12 +250,16 @@ int main(int argc, char* argv[]) if(command_line::has_arg(vm, arg_save_graph)) p2psrv.set_save_graph(true); + epee::g_test_dbg_lock_sleep = command_line::get_arg(vm, test_dbg_lock_sleep); + //initialize core here LOG_PRINT_L0("Initializing core..."); res = ccore.init(vm, testnet_mode); CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core"); if (command_line::get_arg(vm, arg_test_drop_download)) - ccore.no_check_blocks(); + ccore.test_drop_download(); + + ccore.test_drop_download_height(command_line::get_arg(vm, arg_test_drop_download_height)); LOG_PRINT_L0("Core initialized OK"); //initialize objects diff --git a/src/daemon/daemon_commands_handler.h b/src/daemon/daemon_commands_handler.h index 6afbbb07..c23df044 100644 --- a/src/daemon/daemon_commands_handler.h +++ b/src/daemon/daemon_commands_handler.h @@ -426,27 +426,21 @@ private: return false; } - using namespace boost::chrono; - auto point = steady_clock::now(); - auto time_from_epoh = point.time_since_epoch(); - auto ms = duration_cast< milliseconds >( time_from_epoh ).count(); - double ms_f = ms; - ms_f /= 1000.; - - std::ofstream limitFile("log/dr-monero/peers_limit.info", std::ios::app); - limitFile.precision(7); - limitFile << ms_f << " " << static_cast(limit) << std::endl; if (m_srv.m_config.m_net_config.connections_count > limit) { m_srv.m_config.m_net_config.connections_count = limit; - if (m_srv.m_number_of_out_peers > limit) + epee::net_utils::data_logger::get_instance().add_data("peers_limit", m_srv.m_config.m_net_config.connections_count); + if (m_srv.m_current_number_of_out_peers > limit) { - int count = m_srv.m_number_of_out_peers - limit; + int count = m_srv.m_current_number_of_out_peers - limit; m_srv.delete_connections(count); } } else + { m_srv.m_config.m_net_config.connections_count = limit; + epee::net_utils::data_logger::get_instance().add_data("peers_limit", m_srv.m_config.m_net_config.connections_count); + } return true; } @@ -556,7 +550,7 @@ private: //-------------------------------------------------------------------------------- bool test_drop_download(const std::vector& args) { - m_srv.get_payload_object().get_core().no_check_blocks(); + m_srv.get_payload_object().get_core().test_drop_download(); return true; } //-------------------------------------------------------------------------------- diff --git a/src/miner/simpleminer.cpp b/src/miner/simpleminer.cpp index fafe6b3f..6212f88f 100644 --- a/src/miner/simpleminer.cpp +++ b/src/miner/simpleminer.cpp @@ -41,6 +41,7 @@ using namespace epee; namespace po = boost::program_options; +unsigned int epee::g_test_dbg_lock_sleep = 0; int main(int argc, char** argv) { diff --git a/src/p2p/connection_basic.cpp b/src/p2p/connection_basic.cpp index 0e2fd594..4a4a3238 100644 --- a/src/p2p/connection_basic.cpp +++ b/src/p2p/connection_basic.cpp @@ -152,7 +152,7 @@ connection_basic::connection_basic(boost::asio::io_service& io_service, std::ato try { remote_addr_str = socket_.remote_endpoint().address().to_string(); } catch(...){} ; _note("Spawned connection p2p#"<m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_ref_sock_count); - boost::filesystem::create_directories("log/dr-monero/net/"); + //boost::filesystem::create_directories("log/dr-monero/net/"); } connection_basic::~connection_basic() { @@ -192,24 +192,15 @@ void connection_basic::save_limit_to_file(int limit) { // saving limit to file if (!epee::net_utils::data_logger::m_save_graph) return; - std::ofstream file_up, file_down; - file_up.open("log/dr-monero/limit_up.info", std::ofstream::out | std::ofstream::app); - file_up.precision(8); - file_down.open("log/dr-monero/limit_down.info", std::ofstream::out | std::ofstream::app); - file_down.precision(8); - using namespace boost::chrono; - auto point = steady_clock::now(); - auto time_from_epoh = point.time_since_epoch(); - auto s = duration_cast< seconds >( time_from_epoh ).count(); { CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); - file_up << s << " " << network_throttle_manager::get_global_throttle_out().get_terget_speed() / 1024 << "\n"; + epee::net_utils::data_logger::get_instance().add_data("upload_limit", network_throttle_manager::get_global_throttle_out().get_terget_speed() / 1024); } { CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); - file_down << s << " " << network_throttle_manager::get_global_throttle_in().get_terget_speed() / 1024 << "\n"; + epee::net_utils::data_logger::get_instance().add_data("download_limit", network_throttle_manager::get_global_throttle_in().get_terget_speed() / 1024); } } diff --git a/src/p2p/data_logger.cpp b/src/p2p/data_logger.cpp index 6a8eb25b..77f647be 100644 --- a/src/p2p/data_logger.cpp +++ b/src/p2p/data_logger.cpp @@ -1,7 +1,9 @@ #include "data_logger.hpp" #include +#include #include +#include "../../contrib/otshell_utils/utils.hpp" namespace epee { @@ -32,7 +34,16 @@ namespace net_utils mFilesMap["request"] = data_logger::fileData("log/dr-monero/net/req-all.data"); mFilesMap["sleep_down"] = data_logger::fileData("log/dr-monero/down_sleep_log.data"); mFilesMap["sleep_up"] = data_logger::fileData("log/dr-monero/up_sleep_log.data"); + mFilesMap["calc_time"] = data_logger::fileData("log/dr-monero/get_objects_calc_time.data"); + mFilesMap["blockchain_processing_time"] = data_logger::fileData("log/dr-monero/blockchain_log.data"); + mFilesMap["peers_limit"] = data_logger::fileData("log/dr-monero/peers_limit.info"); + mFilesMap["download_limit"] = data_logger::fileData("log/dr-monero/limit_down.info"); + mFilesMap["upload_limit"] = data_logger::fileData("log/dr-monero/limit_up.info"); + + mFilesMap["peers_limit"].mLimitFile = true; + mFilesMap["download_limit"].mLimitFile = true; + mFilesMap["upload_limit"].mLimitFile = true; } void data_logger::add_data(std::string filename, unsigned int data) @@ -40,7 +51,14 @@ namespace net_utils if (mFilesMap.find(filename) == mFilesMap.end()) return; // TODO: exception - mFilesMap[filename].mDataToSave += data; + + nOT::nUtils::cFilesystemUtils::CreateDirTree("log/dr-monero/net/"); + + std::lock_guard lock(mSaveMutex); + if (mFilesMap[filename].mLimitFile) + mFilesMap[filename].mDataToSave = data; + else + mFilesMap[filename].mDataToSave += data; } double data_logger::fileData::get_current_time() @@ -56,23 +74,27 @@ namespace net_utils data_logger::fileData::fileData(std::string pFile) { mFile = std::make_shared (pFile); + mPath = pFile; } void data_logger::fileData::save() { if (!data_logger::m_save_graph) return; + mFile->open(mPath, std::ios::app); *mFile << static_cast(get_current_time()) << " " << mDataToSave << std::endl; + mFile->close(); } void data_logger::saveToFile() { - std::lock_guard lock(mSaveMutex); - for (auto &element : mFilesMap) - { - element.second.save(); - element.second.mDataToSave = 0; - } + std::lock_guard lock(mSaveMutex); + for (auto &element : mFilesMap) + { + element.second.save(); + if (!element.second.mLimitFile) + element.second.mDataToSave = 0; + } } std::atomic data_logger::m_save_graph(false); diff --git a/src/p2p/data_logger.hpp b/src/p2p/data_logger.hpp index 2b8503df..50beb847 100644 --- a/src/p2p/data_logger.hpp +++ b/src/p2p/data_logger.hpp @@ -27,12 +27,15 @@ namespace net_utils { public: fileData(){} + fileData(const fileData &ob) = delete; fileData(std::string pFile); std::shared_ptr mFile; long int mDataToSave = 0; static double get_current_time(); void save(); + std::string mPath; + bool mLimitFile = false; }; std::map mFilesMap; diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index ea7d5c38..5417ffa5 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -87,7 +87,7 @@ namespace nodetool m_hide_my_port(false), m_network_id(std::move(network_id)) { - m_number_of_out_peers = 0; + m_current_number_of_out_peers = 0; m_save_graph = false; } @@ -228,7 +228,7 @@ namespace nodetool public: config m_config; // TODO was private, add getters? - std::atomic m_number_of_out_peers; + std::atomic m_current_number_of_out_peers; void set_save_graph(bool save_graph) { m_save_graph = save_graph; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 60eed1f3..a015763b 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -296,20 +296,18 @@ namespace nodetool unsigned int number_of_peers; while (1) { - if (m_save_graph) + //number_of_peers = m_net_server.get_config_object().get_connections_count(); + number_of_peers = 0; + m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - //number_of_peers = m_net_server.get_config_object().get_connections_count(); - number_of_peers = 0; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) - { - if(!cntxt.m_is_income) - ++number_of_peers; - return true; - }); // lambda + if(!cntxt.m_is_income) + ++number_of_peers; + return true; + }); // lambda + + m_current_number_of_out_peers = number_of_peers; + epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers); - m_number_of_out_peers = number_of_peers; - epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers); - } std::this_thread::sleep_for(std::chrono::seconds(1)); } })); // lambda @@ -724,14 +722,14 @@ namespace nodetool template bool node_server::try_to_connect_and_handshake_with_new_peer(const net_address& na, bool just_take_peerlist, uint64_t last_seen_stamp, bool white) { - if (m_number_of_out_peers == m_config.m_net_config.connections_count) // out peers limit + if (m_current_number_of_out_peers == m_config.m_net_config.connections_count) // out peers limit { return false; } - else if (m_number_of_out_peers > m_config.m_net_config.connections_count) + else if (m_current_number_of_out_peers > m_config.m_net_config.connections_count) { m_net_server.get_config_object().del_out_connections(1); - m_number_of_out_peers --; // atomic variable, update time = 1s + m_current_number_of_out_peers --; // atomic variable, update time = 1s return false; } LOG_PRINT_L1("Connecting to " << epee::string_tools::get_ip_string_from_int32(na.ip) << ":" @@ -1378,25 +1376,14 @@ namespace nodetool template bool node_server::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max) - { - using namespace std::chrono; - auto point = steady_clock::now(); - auto time_from_epoh = point.time_since_epoch(); - auto ms = duration_cast< milliseconds >( time_from_epoh ).count(); - double ms_f = ms; - ms_f /= 1000.; - - std::ofstream limitFile("log/dr-monero/peers_limit.info", std::ios::app); - limitFile.precision(7); + { if(max == -1) { m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT; - if (m_save_graph) - limitFile << static_cast(ms_f) << " " << P2P_DEFAULT_CONNECTIONS_COUNT << std::endl; + epee::net_utils::data_logger::get_instance().add_data("peers_limit", m_config.m_net_config.connections_count); return true; } - + epee::net_utils::data_logger::get_instance().add_data("peers_limit", max); m_config.m_net_config.connections_count = max; - limitFile << static_cast(ms_f) << " " << max << std::endl; return true; } diff --git a/src/p2p/network_throttle-detail.cpp b/src/p2p/network_throttle-detail.cpp index 6b2ee698..7426e6dc 100644 --- a/src/p2p/network_throttle-detail.cpp +++ b/src/p2p/network_throttle-detail.cpp @@ -220,7 +220,7 @@ void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_s std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; std::string history_str = oss.str(); - _info_c( "net/" + m_nameshort , "Throttle " << m_name << ": packet of ~"< 0 ? "SLEEP" : "") << "dbg " << m_name << ": " << "speed is A=" << std::setw(8) < Date: Tue, 24 Feb 2015 20:12:56 +0100 Subject: [PATCH 06/16] 2014 network limit 1.3 fix log/path/data +utils +toc -doc -drmonero Fixed the windows path, and improved logging and data (for graph) logging, fixed some locks and added more checks. Still there is a locking error, not added by my patches, but present in master version (locking of map/list of peers). --- .gitignore | 1 - contrib/otshell_utils/utils.cpp | 168 +++++++++++++++++++++------- contrib/otshell_utils/utils.hpp | 16 ++- src/common/util.cpp | 2 +- src/daemon/daemon.cpp | 6 +- src/p2p/connection_basic.cpp | 1 + src/p2p/data_logger.cpp | 143 +++++++++++++++-------- src/p2p/data_logger.hpp | 54 ++++++--- src/p2p/network_throttle.cpp | 1 - tests/core_proxy/core_proxy.cpp | 3 + tests/core_proxy/core_proxy.h | 2 + tests/core_tests/CMakeLists.txt | 2 + tests/core_tests/chaingen_main.cpp | 2 + tests/functional_tests/main.cpp | 2 + tests/net_load_tests/CMakeLists.txt | 2 + tests/net_load_tests/clt.cpp | 3 + tests/net_load_tests/srv.cpp | 4 +- tests/performance_tests/main.cpp | 2 + tests/unit_tests/main.cpp | 2 + 19 files changed, 306 insertions(+), 110 deletions(-) diff --git a/.gitignore b/.gitignore index 755cb104..18653c23 100644 --- a/.gitignore +++ b/.gitignore @@ -59,7 +59,6 @@ version/ ### CMake ### CMakeCache.txt CMakeFiles -Makefile cmake_install.cmake install_manifest.txt *.cmake diff --git a/contrib/otshell_utils/utils.cpp b/contrib/otshell_utils/utils.cpp index 04326080..ff39d15e 100644 --- a/contrib/otshell_utils/utils.cpp +++ b/contrib/otshell_utils/utils.cpp @@ -160,7 +160,7 @@ std::unique_ptr make_unique( Args&& ...args ) #endif // ==================================================================== -char cFilesystemUtils::GetDirSeparator() { +char cFilesystemUtils::GetDirSeparatorSys() { // TODO nicer os detection? #if defined(OS_TYPE_POSIX) return '/'; @@ -171,26 +171,49 @@ char cFilesystemUtils::GetDirSeparator() { #endif } +char cFilesystemUtils::GetDirSeparatorInter() { + return '/'; +} + +string cFilesystemUtils::FileInternalToSystem(const std::string &name) { + string ret; + ret.resize(name.size()); + std::replace_copy(name.begin(), name.end(), ret.begin(), + GetDirSeparatorInter() , GetDirSeparatorSys()); + return ret; +} + +string cFilesystemUtils::FileSystemToInternal(const std::string &name) { + string ret; + ret.reserve(name.size()); + std::replace_copy(name.begin(), name.end(), ret.begin(), + GetDirSeparatorSys() , GetDirSeparatorInter()); + return ret; +} + bool cFilesystemUtils::CreateDirTree(const std::string & dir, bool only_below) { const bool dbg=false; //struct stat st; - const char dirch = cFilesystemUtils::GetDirSeparator(); + const char dirchS = cFilesystemUtils::GetDirSeparatorSys(); + const char dirchI = cFilesystemUtils::GetDirSeparatorInter(); std::istringstream iss(dir); - string part, sofar=""; + string partI; // current par is in internal format (though it should not matter since it doesn't contain any slashes). eg "bar" + string sofarS=""; // sofarS - the so far created dir part is in SYSTEM format. eg "foo/bar" if (dir.size()<1) return false; // illegal name // dir[0] is valid from here - if (only_below && (dir[0]==dirch)) return false; // no jumping to top (on any os) - while (getline(iss,part,dirch)) { - if (dbg) cout << '['<= mLevel) && (mStream)) { + if ((level >= mLevel) && (mStream)) { // TODO now disabling mStream also disables writting to any channel ostream & output = SelectOutput(level,channel); output << icon(level) << ' '; std::thread::id this_id = std::this_thread::get_id(); @@ -278,33 +321,76 @@ std::string cLogger::GetLogBaseDir() const { return "log"; } -void cLogger::OpenNewChannel(const std::string & channel) { - size_t last_split = channel.find_last_of(cFilesystemUtils::GetDirSeparator()); - // log/test/aaa - // ^----- last_split - string dir = GetLogBaseDir() + cFilesystemUtils::GetDirSeparator() + channel.substr(0, last_split); - string basefile = channel.substr(last_split+1) + ".log"; - string fname = dir + cFilesystemUtils::GetDirSeparator() + cFilesystemUtils::GetDirSeparator() + basefile; - _dbg1("Starting debug to channel file: " + fname + " in directory ["+dir+"]"); - bool dirok = cFilesystemUtils::CreateDirTree(dir); - if (!dirok) { const string msg = "In logger failed to open directory (" + dir +")."; _erro(msg); throw std::runtime_error(msg); } - std::ofstream * thefile = new std::ofstream( fname.c_str() ); - *thefile << "====== (Log opened: " << fname << ") ======" << endl; - mChannels.insert( std::pair(channel , thefile ) ); -} - -std::ostream & cLogger::SelectOutput(int level, const std::string & channel) { - if (channel=="") return *mStream; - auto obj = mChannels.find(channel); - if (obj == mChannels.end()) { // new channel - OpenNewChannel(channel); - return SelectOutput(level,channel); +void cLogger::OpenNewChannel(const std::string & channel) noexcept { + try { + std::cerr<<"openning channel for channel="<second; + catch (const std::exception &except) { + SetStreamBroken(OT_CODE_STAMP + " Got exception when opening debug channel: " + ToStr(except.what())); + } + catch (...) { + SetStreamBroken(OT_CODE_STAMP + " Got not-standard exception when opening debug channel."); } } +void cLogger::OpenNewChannel_(const std::string & channel) { // channel=="net/sleep" + size_t last_split = channel.find_last_of(cFilesystemUtils::GetDirSeparatorInter()); + + string fname_system; // the full file name in system format + + if (last_split==string::npos) { // The channel name has no directory, eg channel=="test" + string dir = GetLogBaseDir(); + string basefile = channel + ".log"; + string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile; + fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <- + } + else { // there is a directory eg channel=="net/sleep" + // net/sleep + // ^----- last_split + string dir = GetLogBaseDir() + cFilesystemUtils::GetDirSeparatorInter() + channel.substr(0, last_split); + string basefile = channel.substr(last_split+1) + ".log"; + string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile; + fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <- + bool dirok = cFilesystemUtils::CreateDirTree(dir); + if (!dirok) { string err = "In logger failed to open directory (" + dir +") for channel (" + channel +")"; throw std::runtime_error(err); } + } + + std::ofstream * thefile = new std::ofstream( fname_system.c_str() ); // file system + *thefile << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl; + cerr << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl; + mChannels.insert( std::pair(channel , thefile ) ); // <- created the channel mapping +} + +std::ostream & cLogger::SelectOutput(int level, const std::string & channel) noexcept { + try { + if (mIsBroken) return *mStreamBrokenDebug; + if (channel=="") return *mStream; + + auto obj = mChannels.find(channel); + if (obj == mChannels.end()) { // not found - need to make new channel + OpenNewChannel(channel); // <- create channel + obj = mChannels.find(channel); // find again + if (obj == mChannels.end()) { // still not found! something is wrong + SetStreamBroken( OT_CODE_STAMP + " WARNING: can not get stream for channel="+ToStr(channel)+" level="+ToStr(channel) ); + return *mStreamBrokenDebug; + } + } + auto the_stream_ptr = obj->second; + ASRT(the_stream_ptr); + return *the_stream_ptr; // <--- RETURN + } + catch (std::exception &except) { + SetStreamBroken( OT_CODE_STAMP + " Got exception: " + ToStr(except.what()) ); + return *mStreamBrokenDebug; + } + catch (...) { + SetStreamBroken( OT_CODE_STAMP + " Got not-standard exception."); + return *mStreamBrokenDebug; + } + + // dead code +} void cLogger::setOutStreamFile(const string &fname) { // switch to using this file _mark("WILL SWITCH DEBUG NOW to file: " << fname); diff --git a/contrib/otshell_utils/utils.hpp b/contrib/otshell_utils/utils.hpp index 83e6b822..35b464b4 100644 --- a/contrib/otshell_utils/utils.hpp +++ b/contrib/otshell_utils/utils.hpp @@ -185,6 +185,7 @@ const char* DbgShortenCodeFileName(const char *s); ///< Returns a pointer to som /*** @brief Class to write debug into. Used it by calling the debug macros _dbg1(...) _info(...) _erro(...) etc, NOT directly! @author rfree (maintainer) +@thread this class is NOT thread safe and must used only by one thread at once (use it via ot_debug_macros like _info macro they do proper locking) */ class cLogger { public: @@ -201,15 +202,21 @@ class cLogger { std::string endline() const; ///< returns string to be written at end of message protected: + void SetStreamBroken(); ///< call in case of internal error in logger (e.g. can not open a file) + void SetStreamBroken(const std::string &msg); ///< same but with error message + unique_ptr mOutfile; std::ostream * mStream; ///< pointing only! can point to our own mOutfile, or maye to global null stream + std::ostream * mStreamBrokenDebug; ///< pointing only! this is a pointer to some stream that should be used when normal debugging is broken eg std::cerr + bool mIsBroken; ///< is the debugging system broken (this should be set when internal problems occur and should cause fallback to std::cerr) std::map< std::string , std::ofstream * > mChannels; // the ofstream objects are owned by this class int mLevel; ///< current debug level - std::ostream & SelectOutput(int level, const std::string & channel); - void OpenNewChannel(const std::string & channel); + std::ostream & SelectOutput(int level, const std::string & channel) noexcept; ///< returns a proper stream for this level and channel (always usable string) + void OpenNewChannel(const std::string & channel) noexcept; ///< tries to prepare this channel. does NOT guarantee to created mChannels[] entry! + void OpenNewChannel_(const std::string & channel); ///< internal function, will throw in case of problems std::string GetLogBaseDir() const; std::map< std::thread::id , int > mThread2Number; // change long thread IDs into a short nice number to show @@ -360,7 +367,10 @@ eSubjectType String2SubjectType(const string & type); class cFilesystemUtils { // if we do not want to use boost in given project (or we could optionally write boost here later) public: static bool CreateDirTree(const std::string & dir, bool only_below=false); - static char GetDirSeparator(); // eg '/' or '\' + static char GetDirSeparatorSys(); /// < eg '/' or '\' + static char GetDirSeparatorInter(); /// < internal is '/' + static string FileInternalToSystem(const std::string &name); ///< converts from internal file name string to system file name string + static string FileSystemToInternal(const std::string &name); ///< converts from system file name string to internal file name string }; diff --git a/src/common/util.cpp b/src/common/util.cpp index 907a87ce..94c1e042 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -326,7 +326,7 @@ std::string get_nix_version_display_string() std::string config_folder; #ifdef WIN32 - config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "/" + CRYPTONOTE_NAME; + config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "\\" + CRYPTONOTE_NAME; #else std::string pathRet; char* pszHome = getenv("HOME"); diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 8497efbf..8d584af5 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -125,8 +125,9 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Starting..."); nOT::nUtils::cFilesystemUtils::CreateDirTree("log/dr-monero/net/"); - // _warn_c("log/test","Starting program"); // TODO - + _warn_c("test","Starting program (a test message)"); + _warn_c("main/program","Starting program"); + TRY_ENTRY(); boost::filesystem::path default_data_path {tools::get_default_data_dir()}; @@ -319,6 +320,7 @@ int main(int argc, char* argv[]) ccore.set_cryptonote_protocol(NULL); cprotocol.set_p2p_endpoint(NULL); + epee::net_utils::data_logger::get_instance().kill_instance(); LOG_PRINT("Node stopped.", LOG_LEVEL_0); return 0; diff --git a/src/p2p/connection_basic.cpp b/src/p2p/connection_basic.cpp index 4a4a3238..ed15c098 100644 --- a/src/p2p/connection_basic.cpp +++ b/src/p2p/connection_basic.cpp @@ -272,6 +272,7 @@ void connection_basic::logger_handle_net_write(size_t size) { } double connection_basic::get_sleep_time(size_t cb) { + CRITICAL_REGION_LOCAL(epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_out); auto t = network_throttle_manager::get_global_throttle_out().get_sleep_time(cb); return t; } diff --git a/src/p2p/data_logger.cpp b/src/p2p/data_logger.cpp index 77f647be..d62af133 100644 --- a/src/p2p/data_logger.cpp +++ b/src/p2p/data_logger.cpp @@ -1,4 +1,5 @@ #include "data_logger.hpp" +#include #include #include @@ -9,25 +10,25 @@ namespace epee { namespace net_utils { - data_logger &data_logger::get_instance() - { - static data_logger instance; - return instance; + data_logger &data_logger::get_instance() { + std::call_once(m_singleton, + [] { + _info_c("dbg/data","Creating singleton of data_logger"); + if (m_state != data_logger_state::state_before_init) { _erro_c("dbg/data","Internal error in singleton"); throw std::runtime_error("data_logger singleton"); } + m_state = data_logger_state::state_during_init; + m_obj.reset(new data_logger()); + m_state = data_logger_state::state_ready_to_use; + } + ); + return * m_obj; } - data_logger::data_logger() - { - //create timer - std::shared_ptr logger_thread(new std::thread([&]() - { - while (true) - { - std::this_thread::sleep_for(std::chrono::seconds(1)); - saveToFile(); - } - })); - logger_thread->detach(); + data_logger::data_logger() { + _warn_c("dbg/data","Starting data logger (for graphs data)"); + if (m_state != data_logger_state::state_during_init) { _erro_c("dbg/data","Singleton ctor state"); throw std::runtime_error("data_logger ctor state"); } + std::lock_guard lock(mMutex); // lock + // prepare all the files for given data channels: mFilesMap["peers"] = data_logger::fileData("log/dr-monero/peers.data"); mFilesMap["download"] = data_logger::fileData("log/dr-monero/net/in-all.data"); mFilesMap["upload"] = data_logger::fileData("log/dr-monero/net/out-all.data"); @@ -44,25 +45,80 @@ namespace net_utils mFilesMap["peers_limit"].mLimitFile = true; mFilesMap["download_limit"].mLimitFile = true; mFilesMap["upload_limit"].mLimitFile = true; + + // do NOT modify mFilesMap below this point, since there is no locking for this used (yet) + + _note_c("dbg/data","Creating thread for data logger"); // create timer thread + m_thread_maybe_running=true; + std::shared_ptr logger_thread(new std::thread([&]() { + _note_c("dbg/data","Inside thread for data logger"); + while (m_state == data_logger_state::state_during_init) { // wait for creation to be done (in other thread, in singleton) before actually running + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + _note_c("dbg/data","Inside thread for data logger - going into main loop"); + while (m_state == data_logger_state::state_ready_to_use) { // run as long as we are not closing the single object + std::this_thread::sleep_for(std::chrono::seconds(1)); + saveToFile(); // save all the pending data + } + _note_c("dbg/data","Inside thread for data logger - done the main loop"); + m_thread_maybe_running=false; + })); + logger_thread->detach(); + _info_c("dbg/data","Data logger constructed"); + } + + data_logger::~data_logger() { + _note_c("dbg/data","Destructor of the data logger"); + { + std::lock_guard lock(mMutex); + m_state = data_logger_state::state_dying; + } + _info_c("dbg/data","State was set to dying"); + while(m_thread_maybe_running) { // wait for the thread to exit + std::this_thread::sleep_for(std::chrono::seconds(1)); + _info_c("dbg/data","Waiting for background thread to exit"); + } + _info_c("dbg/data","Thread exited"); + } + + void data_logger::kill_instance() { + m_state = m_state = data_logger_state::state_dying; + m_obj.reset(); } - void data_logger::add_data(std::string filename, unsigned int data) - { - if (mFilesMap.find(filename) == mFilesMap.end()) - return; // TODO: exception - + void data_logger::add_data(std::string filename, unsigned int data) { + std::lock_guard lock(mMutex); + if (m_state != data_logger_state::state_ready_to_use) { _info_c("dbg/data","Data logger is not ready, returning."); return; } + + if (mFilesMap.find(filename) == mFilesMap.end()) { // no such file/counter + _erro_c("dbg/data","Trying to use not opened data file filename="< lock(mSaveMutex); - if (mFilesMap[filename].mLimitFile) + if (mFilesMap[filename].mLimitFile) { // this holds a number (that is not additive) - e.g. the limit setting mFilesMap[filename].mDataToSave = data; - else - mFilesMap[filename].mDataToSave += data; + } else { + mFilesMap[filename].mDataToSave += data; // this holds a number that should be sum of all accumulated samples + } } + + void data_logger::saveToFile() { + _dbg2_c("dbg/data","saving to files"); + std::lock_guard lock(mMutex); + if (m_state != data_logger_state::state_ready_to_use) { _info_c("dbg/data","Data logger is not ready, returning."); return; } + nOT::nUtils::cFilesystemUtils::CreateDirTree("log/dr-monero/net/"); + for (auto &element : mFilesMap) + { + element.second.save(); + if (!element.second.mLimitFile) element.second.mDataToSave = 0; + } + } + + // the inner class: - double data_logger::fileData::get_current_time() - { + double data_logger::fileData::get_current_time() { using namespace boost::chrono; auto point = steady_clock::now(); auto time_from_epoh = point.time_since_epoch(); @@ -71,33 +127,28 @@ namespace net_utils return ms_f / 1000.; } - data_logger::fileData::fileData(std::string pFile) - { + data_logger::fileData::fileData(std::string pFile) { + _dbg3_c("dbg/data","opening data file named pFile="<close(); } - void data_logger::saveToFile() - { - std::lock_guard lock(mSaveMutex); - for (auto &element : mFilesMap) - { - element.second.save(); - if (!element.second.mLimitFile) - element.second.mDataToSave = 0; - } - } -std::atomic data_logger::m_save_graph(false); +data_logger_state data_logger::m_state(data_logger_state::state_before_init); ///< (static) state of the singleton object +std::atomic data_logger::m_save_graph(false); // (static) +std::atomic data_logger::m_thread_maybe_running(false); // (static) +std::once_flag data_logger::m_singleton; // (static) +std::unique_ptr data_logger::m_obj; // (static) } // namespace } // namespace + diff --git a/src/p2p/data_logger.hpp b/src/p2p/data_logger.hpp index 50beb847..21591216 100644 --- a/src/p2p/data_logger.hpp +++ b/src/p2p/data_logger.hpp @@ -13,34 +13,60 @@ namespace epee { namespace net_utils { + +enum class data_logger_state { state_before_init, state_during_init, state_ready_to_use, state_dying }; +/*** +@note: use it ONLY via singleton! It will be spawned then, and will auto destruct on program exit. +@note: do call ::kill_instance() before exiting main, at end of main. But before make sure no one else (e.g. no other threads) will try to use this/singleton +@note: it is not allowed to use this class from code "runnig before or after main", e.g. from ctors of static objects, because of static-creation-order races +@note: on creation (e.g. from singleton), it spawns a thread that saves all data in background +*/ class data_logger { public: - static data_logger &get_instance(); - data_logger(const data_logger &ob) = delete; - data_logger(data_logger &&ob) = delete; - void add_data(std::string filename, unsigned int data); - static std::atomic m_save_graph; + static data_logger &get_instance(); ///< singleton + static void kill_instance(); ///< call this before ending main to allow more gracefull shutdown of the main singleton and it's background thread + ~data_logger(); ///< destr, will be called when singleton is killed when global m_obj dies. will kill theads etc + private: - data_logger(); - class fileData - { + data_logger(); ///< constructor is private, use only via singleton get_instance + + public: + data_logger(const data_logger &ob) = delete; // use only one per program + data_logger(data_logger &&ob) = delete; + data_logger & operator=(const data_logger&) = delete; + data_logger & operator=(data_logger&&) = delete; + + void add_data(std::string filename, unsigned int data); ///< use this to append data here. Use it only the singleton. It locks itself. + + static std::atomic m_save_graph; ///< global setting flag, should we save all the data or not (can disable logging graphs data) + + private: + static std::once_flag m_singleton; ///< to guarantee singleton creates the object exactly once + static data_logger_state m_state; ///< state of the singleton object + static std::atomic m_thread_maybe_running; ///< is the background thread (more or less) running, or is it fully finished + static std::unique_ptr m_obj; ///< the singleton object. Only use it via get_instance(). Can be killed by kill_instance() + + /*** + * one graph/file with data + */ + class fileData { public: - fileData(){} + fileData() = default; fileData(const fileData &ob) = delete; fileData(std::string pFile); std::shared_ptr mFile; - long int mDataToSave = 0; + long int mDataToSave = 0; ///< sum of the data (in current interval, will be counted from 0 on next interval) static double get_current_time(); void save(); std::string mPath; - bool mLimitFile = false; + bool mLimitFile = false; ///< this holds a number (that is not additive) - e.g. the limit setting }; - std::map mFilesMap; - std::mutex mSaveMutex; - void saveToFile(); + std::map mFilesMap; + std::mutex mMutex; + void saveToFile(); ///< write data to the target files. do not use this directly }; } // namespace diff --git a/src/p2p/network_throttle.cpp b/src/p2p/network_throttle.cpp index 3d5edcdc..7bc89881 100644 --- a/src/p2p/network_throttle.cpp +++ b/src/p2p/network_throttle.cpp @@ -78,7 +78,6 @@ int network_throttle_manager::xxx; // ================================================================================================ // methods: i_network_throttle & network_throttle_manager::get_global_throttle_in() { - std::call_once(m_once_get_global_throttle_in, [] { m_obj_get_global_throttle_in.reset(new network_throttle("in/all","<<< global-IN",10)); } ); return * m_obj_get_global_throttle_in; } diff --git a/tests/core_proxy/core_proxy.cpp b/tests/core_proxy/core_proxy.cpp index de8f45fc..879f83eb 100644 --- a/tests/core_proxy/core_proxy.cpp +++ b/tests/core_proxy/core_proxy.cpp @@ -62,6 +62,8 @@ using namespace crypto; BOOST_CLASS_VERSION(nodetool::node_server >, 1); +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char* argv[]) { @@ -148,6 +150,7 @@ int main(int argc, char* argv[]) LOG_PRINT("Node stopped.", LOG_LEVEL_0); + epee::net_utils::data_logger::get_instance().kill_instance(); return 0; CATCH_ENTRY_L0("main", 1); diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index 2b807564..b40c5b21 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -82,5 +82,7 @@ namespace tests bool find_blockchain_supplement(const std::list& qblock_ids, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;} bool handle_get_objects(cryptonote::NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote::cryptonote_connection_context& context){return true;} cryptonote::blockchain_storage &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class proxy_core."); } + bool get_test_drop_download() {return true;} + bool get_test_drop_download_height() {return true;} }; } diff --git a/tests/core_tests/CMakeLists.txt b/tests/core_tests/CMakeLists.txt index ac536b29..2b9e0cf8 100644 --- a/tests/core_tests/CMakeLists.txt +++ b/tests/core_tests/CMakeLists.txt @@ -60,6 +60,8 @@ add_executable(coretests target_link_libraries(coretests LINK_PRIVATE cryptonote_core + p2p + ${Boost_CHRONO_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index 0ec4d458..a9ef8b00 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -44,6 +44,8 @@ namespace const command_line::arg_descriptor arg_test_transactions = {"test_transactions", ""}; } +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char* argv[]) { TRY_ENTRY(); diff --git a/tests/functional_tests/main.cpp b/tests/functional_tests/main.cpp index fda64303..a653e7b1 100644 --- a/tests/functional_tests/main.cpp +++ b/tests/functional_tests/main.cpp @@ -55,6 +55,8 @@ namespace const command_line::arg_descriptor arg_test_repeat_count = {"test_repeat_count", "", 1}; } +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char* argv[]) { TRY_ENTRY(); diff --git a/tests/net_load_tests/CMakeLists.txt b/tests/net_load_tests/CMakeLists.txt index 6095146f..acd7c9ac 100644 --- a/tests/net_load_tests/CMakeLists.txt +++ b/tests/net_load_tests/CMakeLists.txt @@ -39,6 +39,7 @@ target_link_libraries(net_load_tests_clt LINK_PRIVATE otshell_utils p2p + cryptonote_core ${GTEST_MAIN_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_DATE_TIME_LIBRARY} @@ -60,6 +61,7 @@ target_link_libraries(net_load_tests_srv LINK_PRIVATE otshell_utils p2p + cryptonote_core ${GTEST_MAIN_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_DATE_TIME_LIBRARY} diff --git a/tests/net_load_tests/clt.cpp b/tests/net_load_tests/clt.cpp index 7c9b86cb..6307f18e 100644 --- a/tests/net_load_tests/clt.cpp +++ b/tests/net_load_tests/clt.cpp @@ -623,6 +623,8 @@ TEST_F(net_load_test_clt, permament_open_and_close_and_connections_closed_by_ser ASSERT_EQ(RESERVED_CONN_CNT, m_tcp_server.get_config_object().get_connections_count()); } +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char** argv) { epee::debug::get_set_enable_assert(true, false); @@ -631,5 +633,6 @@ int main(int argc, char** argv) epee::log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); ::testing::InitGoogleTest(&argc, argv); + epee::net_utils::data_logger::get_instance().kill_instance(); return RUN_ALL_TESTS(); } diff --git a/tests/net_load_tests/srv.cpp b/tests/net_load_tests/srv.cpp index edb106ea..582c9efd 100644 --- a/tests/net_load_tests/srv.cpp +++ b/tests/net_load_tests/srv.cpp @@ -213,6 +213,8 @@ namespace }; } +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char** argv) { //set up logging options @@ -232,6 +234,6 @@ int main(int argc, char** argv) if (!tcp_server.run_server(thread_count, true)) return 2; - + epee::net_utils::data_logger::get_instance().kill_instance(); return 0; } diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index 724b3638..588cf652 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -42,6 +42,8 @@ #include "generate_key_image_helper.h" #include "is_out_to_acc.h" +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char** argv) { set_process_affinity(1); diff --git a/tests/unit_tests/main.cpp b/tests/unit_tests/main.cpp index e187bf52..471895b8 100644 --- a/tests/unit_tests/main.cpp +++ b/tests/unit_tests/main.cpp @@ -32,6 +32,8 @@ #include "include_base_utils.h" +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char** argv) { epee::debug::get_set_enable_assert(true, false); From f79821ac7ebc269b18dd2edd8ac0cb022861ad11 Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Tue, 24 Feb 2015 21:02:48 +0100 Subject: [PATCH 07/16] fix locking in count-peers thread (2) --- src/daemon/daemon.cpp | 2 +- src/p2p/data_logger.cpp | 8 +++++++- src/p2p/net_node.h | 11 ++++++++++- src/p2p/net_node.inl | 19 ++++++++++--------- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 8d584af5..4a93a0db 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -320,7 +320,7 @@ int main(int argc, char* argv[]) ccore.set_cryptonote_protocol(NULL); cprotocol.set_p2p_endpoint(NULL); - epee::net_utils::data_logger::get_instance().kill_instance(); + epee::net_utils::data_logger::kill_instance(); LOG_PRINT("Node stopped.", LOG_LEVEL_0); return 0; diff --git a/src/p2p/data_logger.cpp b/src/p2p/data_logger.cpp index d62af133..69e50141 100644 --- a/src/p2p/data_logger.cpp +++ b/src/p2p/data_logger.cpp @@ -20,6 +20,12 @@ namespace net_utils m_state = data_logger_state::state_ready_to_use; } ); + + if (m_state != data_logger_state::state_ready_to_use) { + _erro ("trying to use not working data_logger"); + throw std::runtime_error("data_logger ctor state"); + } + return * m_obj; } @@ -82,7 +88,7 @@ namespace net_utils } void data_logger::kill_instance() { - m_state = m_state = data_logger_state::state_dying; + m_state = data_logger_state::state_dying; m_obj.reset(); } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 5417ffa5..5b034ce2 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -89,6 +89,7 @@ namespace nodetool { m_current_number_of_out_peers = 0; m_save_graph = false; + is_closing = false; } static void init_options(boost::program_options::options_description& desc); @@ -209,6 +210,13 @@ namespace nodetool bool set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit); bool set_rate_limit(const boost::program_options::variables_map& vm, uint64_t limit); + void kill() { ///< will be called e.g. from deinit() + _info("Killing the net_node"); + is_closing = true; + mPeersLoggerThread->join(); // make sure the thread finishes + _info("Joined extra background net_node threads"); + } + //debug functions std::string print_connections_container(); @@ -247,7 +255,8 @@ namespace nodetool bool m_hide_my_port; bool m_no_igd; std::atomic m_save_graph; - + std::atomic is_closing; + std::unique_ptr mPeersLoggerThread; //critical_section m_connections_lock; //connections_indexed_container m_connections; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index a015763b..afc6436f 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -291,17 +291,17 @@ namespace nodetool std::vector> dns_results; dns_results.resize(m_seed_nodes_list.size()); - std::shared_ptr peersLoggerThread (new std::thread([&]() + // creating thread to log number of connections + mPeersLoggerThread.reset(new std::thread([&]() { - unsigned int number_of_peers; - while (1) - { + _note("Thread monitor number of peers - start"); + while (!is_closing) + { // main loop of thread //number_of_peers = m_net_server.get_config_object().get_connections_count(); - number_of_peers = 0; + unsigned int number_of_peers = 0; m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - if(!cntxt.m_is_income) - ++number_of_peers; + if (!cntxt.m_is_income) ++number_of_peers; return true; }); // lambda @@ -309,10 +309,10 @@ namespace nodetool epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers); std::this_thread::sleep_for(std::chrono::seconds(1)); - } + } // main loop of thread + _note("Thread monitor number of peers - done"); })); // lambda - peersLoggerThread->detach(); std::list dns_threads; uint64_t result_index = 0; @@ -509,6 +509,7 @@ namespace nodetool template bool node_server::deinit() { + kill(); m_peerlist.deinit(); m_net_server.deinit_server(); From c511abf0058b8c1d020255faaad418b0ffd2eb26 Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Wed, 1 Apr 2015 19:00:45 +0200 Subject: [PATCH 08/16] remerged; commands JSON. logging upgrade. doxygen --- CMakeLists.txt | 40 ++++ .../net/levin_protocol_handler_async.h | 1 - contrib/otshell_utils/utils.cpp | 102 +++++++-- contrib/otshell_utils/utils.hpp | 34 ++- src/common/command_line.cpp | 3 + src/common/command_line.h | 3 + src/common/dns_utils.cpp | 19 +- src/cryptonote_core/cryptonote_core.cpp | 5 + .../cryptonote_protocol_defs.h | 14 ++ .../cryptonote_protocol_handler.inl | 37 ++++ src/daemon/CMakeLists.txt | 3 + src/daemon/command_parser_executor.cpp | 37 ++++ src/daemon/command_parser_executor.h | 7 + src/daemon/command_server.cpp | 24 ++- src/daemon/daemon.cpp | 10 +- src/daemon/daemon_commands_handler.h | 202 ++++-------------- src/daemon/main.cpp | 10 +- src/daemon/rpc_command_executor.cpp | 152 +++++++++++-- src/daemon/rpc_command_executor.h | 8 +- src/p2p/data_logger.cpp | 19 +- src/p2p/data_logger.hpp | 1 + src/p2p/net_node.h | 19 +- src/p2p/net_node.inl | 60 +++--- src/p2p/network_throttle-detail.cpp | 9 +- src/rpc/CMakeLists.txt | 1 + src/rpc/core_rpc_server.cpp | 39 ++++ src/rpc/core_rpc_server.h | 8 + src/rpc/core_rpc_server_commands_defs.h | 74 +++++++ src/simplewallet/simplewallet.cpp | 2 + 29 files changed, 701 insertions(+), 242 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c633016..392b04de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,9 +62,24 @@ endif() message(STATUS "BOOST_IGNORE_SYSTEM_PATHS defaults to ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}") option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost installation" ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}) + +if (NOT DEFINED ENV{DEVELOPER_LIBUNBOUND_OLD}) + message(STATUS "Could not find DEVELOPER_LIBUNBOUND_OLD in env (not required)") +elseif ("$ENV{DEVELOPER_LIBUNBOUND_OLD}" EQUAL 1) + message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD = 1, will use the work around") + add_definitions(-DDEVELOPER_LIBUNBOUND_OLD) +elseif ("$ENV{DEVELOPER_LIBUNBOUND_OLD}" EQUAL 0) + message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD = 0") +else() + message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD with bad value. Will NOT use the work around") +endif() + set_property(GLOBAL PROPERTY USE_FOLDERS ON) enable_testing() +option(BUILD_DOCUMENTATION "Build the Doxygen documentation." ON) + + # Check if we're on FreeBSD so we can exclude the local miniupnpc (it should be installed from ports instead) # CMAKE_SYSTEM_NAME checks are commonly known, but specifically taken from libsdl's CMakeLists if(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD.*") @@ -276,3 +291,28 @@ add_subdirectory(src) if(BUILD_TESTS) add_subdirectory(tests) endif() + + + +if(BUILD_DOCUMENTATION) + set(DOC_GRAPHS "YES" CACHE STRING "Create dependency graphs (needs graphviz)") + set(DOC_FULLGRAPHS "NO" CACHE STRING "Create call/callee graphs (large)") + + find_program(DOT_PATH dot) + + if (DOT_PATH STREQUAL "DOT_PATH-NOTFOUND") + message("Doxygen: graphviz not found - graphs disabled") + set(DOC_GRAPHS "NO") + endif() + + find_package(Doxygen) + if(DOXYGEN_FOUND) + configure_file("cmake/Doxyfile.in" "Doxyfile" @ONLY) + configure_file("cmake/Doxygen.extra.css.in" "Doxygen.extra.css" @ONLY) + add_custom_target(doc + ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating API documentation with Doxygen.." VERBATIM) + endif() +endif() + diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 92f75161..dacbfd4d 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -690,7 +690,6 @@ void async_protocol_handler_config::del_out_connections(si { close(*out_connections.begin()); del_connection(m_connects.at(*out_connections.begin())); - out_connections.erase(out_connections.begin()); --count; } diff --git a/contrib/otshell_utils/utils.cpp b/contrib/otshell_utils/utils.cpp index ff39d15e..3ca84da8 100644 --- a/contrib/otshell_utils/utils.cpp +++ b/contrib/otshell_utils/utils.cpp @@ -40,6 +40,7 @@ #error "Compiler/OS platform detection failed - not supported" #endif + namespace nOT { namespace nUtils { @@ -246,6 +247,13 @@ bool cFilesystemUtils::CreateDirTree(const std::string & dir, bool only_below) { namespace nDetail { +struct channel_use_info { ///< feedback information about using (e.g. opening) given debug channel - used internally by logging system +/// TODO not yet used in code +/// e.g. used to write into channel net/in/all that given message was a first logged message from never-before-logged thread or PID etc + bool m_was_interesting; ///< anything interesting happened when using the channel? + std::vector m_extra_msg; ///< any additional messages about this channel use +}; + cDebugScopeGuard::cDebugScopeGuard() : mLevel(-1) { } @@ -269,13 +277,18 @@ cLogger::cLogger() : mStream(NULL), mStreamBrokenDebug(NULL), mIsBroken(true), // before constructor finishes -mLevel(85), -mThread2Number_Biggest(0) // the CURRENT biggest value (no thread yet in map) +mLevel(70), +mThread2Number_Biggest(0), // the CURRENT biggest value (no thread yet in map) +mPid2Number_Biggest(0) { mStream = & std::cout; - mStreamBrokenDebug = & std::cerr; - Thread2Number( std::this_thread::get_id() ); // convert current id to short number, useful to reserve a number so that main thread is usually called 1 + mStreamBrokenDebug = & std::cerr; // the backup stream + *mStreamBrokenDebug << "Creating the logger system" << endl; mIsBroken=false; // ok, constr. succeeded, so string is not broken now + + // this is here, because it could be using logging itself to log creation of first thread/PID etc + Thread2Number( std::this_thread::get_id() ); // convert current id to short number, useful to reserve a number so that main thread is usually called 1 + Pid2Number( getpid() ); // add this proces ID as first one } cLogger::~cLogger() { @@ -291,7 +304,9 @@ void cLogger::SetStreamBroken() { } void cLogger::SetStreamBroken(const std::string &msg) { + _dbg_dbg("Stream is broken (msg: " << msg << ")"); if (!mIsBroken) { // if not already marked as broken + _dbg_dbg("(It was not broken before)"); std::cerr << OT_CODE_STAMP << "WARNING: due to debug stream problem ("<= mLevel) && (mStream)) { // TODO now disabling mStream also disables writting to any channel - ostream & output = SelectOutput(level,channel); - output << icon(level) << ' '; - std::thread::id this_id = std::this_thread::get_id(); - output << "{" << Thread2Number(this_id) << "} "; - return output; - } + _dbg_dbg("level="<0) output << " {p" << nicePid << "}"; + output << ' '; + return output; // <--- return + } else _dbg_dbg("Not writting: No mStream"); + } else _dbg_dbg("Not writting: Too low level level="<= mLevel="<(channel , thefile ) ); // <- created the channel mapping } std::ostream & cLogger::SelectOutput(int level, const std::string & channel) noexcept { try { - if (mIsBroken) return *mStreamBrokenDebug; - if (channel=="") return *mStream; + if (mIsBroken) { + _dbg_dbg("The stream is broken mIsBroken="<second; + _dbg_dbg("Found the stream file for channel="<= 100) return cc::back::red + ToStr(cc::fore::black) + ToStr("ERROR ") + ToStr(cc::fore::lightyellow) + " " ; + #if defined(OS_TYPE_POSIX) + if (level >= 100) return cc::back::lightred + ToStr(cc::fore::lightyellow) + ToStr("ERROR ") + ToStr(cc::fore::lightyellow) + " " ; if (level >= 90) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("Warn ") + ToStr(cc::fore::red)+ " " ; if (level >= 80) return cc::back::lightmagenta + ToStr(cc::fore::black) + ToStr("MARK "); //+ zkr::cc::console + ToStr(cc::fore::lightmagenta)+ " "; - if (level >= 75) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("FACT ") + zkr::cc::console + ToStr(cc::fore::lightyellow)+ " "; + if (level >= 75) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("FACT ") + zkr::cc::console + ToStr(cc::fore::lightyellow)+ " "; if (level >= 70) return cc::fore::green + ToStr("Note "); if (level >= 50) return cc::fore::cyan + ToStr("info "); if (level >= 40) return cc::fore::lightwhite + ToStr("dbg "); if (level >= 30) return cc::fore::lightblue + ToStr("dbg "); if (level >= 20) return cc::fore::blue + ToStr("dbg "); + #elif defined(OS_TYPE_WINDOWS) + if (level >= 100) return ToStr("ERROR "); + if (level >= 90) return ToStr("Warn "); + if (level >= 80) return ToStr("MARK "); + if (level >= 75) return ToStr("FACT "); + if (level >= 70) return ToStr("Note "); + if (level >= 50) return ToStr("info "); + if (level >= 40) return ToStr("dbg "); + if (level >= 30) return ToStr("dbg "); + if (level >= 20) return ToStr("dbg "); + #endif + return " "; } std::string cLogger::endline() const { + #if defined(OS_TYPE_POSIX) return ToStr("") + zkr::cc::console + ToStr("\n"); // TODO replan to avoid needles converting back and forth char*, string etc + #elif defined(OS_TYPE_WINDOWS) + return ToStr("\n"); + #endif } int cLogger::Thread2Number(const std::thread::id id) { @@ -451,13 +506,24 @@ int cLogger::Thread2Number(const std::thread::id id) { if (found == mThread2Number.end()) { // new one mThread2Number_Biggest++; mThread2Number[id] = mThread2Number_Biggest; + _mark_c("dbg/main", "This is a new thread (used in debug), thread id="< & gLoggerGuardDepth_Get(); // getter for the global singleton o // detect stream e.g. operator<< error #define _debug_level(LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \ + _dbg_dbg("WRITE DEBUG: LEVEL="< mutex_guard( nOT::nUtils::gLoggerGuard ); \ @@ -93,6 +108,7 @@ std::atomic & gLoggerGuardDepth_Get(); // getter for the global singleton o // info for code below: oss object is normal stack variable, using it does not need lock protection #define _debug_level_c(CHANNEL,LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \ + _dbg_dbg("WRITE DEBUG: LEVEL="< mutex_guard( nOT::nUtils::gLoggerGuard ); \ @@ -102,9 +118,11 @@ std::atomic & gLoggerGuardDepth_Get(); // getter for the global singleton o std::ostringstream oss; \ oss << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \ std::string as_string = oss.str(); \ + _dbg_dbg("START will write to log LEVEL="< mThread2Number; // change long thread IDs into a short nice number to show - int mThread2Number_Biggest; // current biggest value held there (biggest key) - works as growing-only counter basically - int Thread2Number(const std::thread::id id); // convert the system's thread id into a nice short our id; make one if new thread + std::map< std::thread::id , int > mThread2Number; ///< change long thread IDs into a short nice number to show + int mThread2Number_Biggest; ///< current biggest value held there (biggest key) - works as growing-only counter basically + int Thread2Number(const std::thread::id id); ///< convert the system's thread id into a nice short our id; make one if new thread + + std::map< t_anypid , int > mPid2Number; ///< change long proces PID into a short nice number to show + int mPid2Number_Biggest; ///< current biggest value held there (biggest key) - works as growing-only counter basically + int Pid2Number(const t_anypid id); ///< convert the system's PID id into a nice short our id; make one if new thread }; diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index 36d9905b..d2cd75e5 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -48,4 +48,7 @@ namespace command_line const arg_descriptor arg_version = {"version", "Output version information"}; const arg_descriptor arg_data_dir = {"data-dir", "Specify data directory"}; const arg_descriptor arg_testnet_data_dir = {"testnet-data-dir", "Specify testnet data directory"}; + const arg_descriptor arg_test_drop_download = {"test-drop-download", "For net tests: in download, discard ALL blocks instead checking/saving them (very fast)"}; + const arg_descriptor arg_test_drop_download_height = {"test-drop-download-height", "Like test-drop-download but disards only after around certain height", 0}; + const arg_descriptor arg_test_dbg_lock_sleep = {"test-dbg-lock-sleep", "Sleep time in ms, defaults to 0 (off), used to debug before/after locking mutex. Values 100 to 1000 are good for tests."}; } diff --git a/src/common/command_line.h b/src/common/command_line.h index 5176f0f6..ae79f0a0 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -204,4 +204,7 @@ namespace command_line extern const arg_descriptor arg_version; extern const arg_descriptor arg_data_dir; extern const arg_descriptor arg_testnet_data_dir; + extern const arg_descriptor arg_test_drop_download; + extern const arg_descriptor arg_test_drop_download_height; + extern const arg_descriptor arg_test_dbg_lock_sleep; } diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 4ab93cce..38b88023 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -168,7 +168,24 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) ub_ctx_resolvconf(m_data->m_ub_context, &empty_string); ub_ctx_hosts(m_data->m_ub_context, &empty_string); - ub_ctx_add_ta(m_data->m_ub_context, ::get_builtin_ds()); + #ifdef DEVELOPER_LIBUNBOUND_OLD + #warning "Using the work around for old libunbound" + { // work around for bug https://www.nlnetlabs.nl/bugs-script/show_bug.cgi?id=515 needed for it to compile on e.g. Debian 7 + char * ds_copy = NULL; // this will be the writable copy of string that bugged version of libunbound requires + try { + char * ds_copy = strdup( ::get_builtin_ds() ); + ub_ctx_add_ta(m_data->m_ub_context, ds_copy); + } catch(...) { // probably not needed but to work correctly in every case... + if (ds_copy) { free(ds_copy); ds_copy=NULL; } // for the strdup + throw ; + } + if (ds_copy) { free(ds_copy); ds_copy=NULL; } // for the strdup + } + #else + // normal version for fixed libunbound + ub_ctx_add_ta(m_data->m_ub_context, ::get_builtin_ds() ); + #endif + } DNSResolver::~DNSResolver() diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 505feb56..9be5eca9 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -145,6 +145,11 @@ namespace cryptonote set_enforce_dns_checkpoints(command_line::get_arg(vm, daemon_args::arg_dns_checkpoints)); + test_drop_download_height(command_line::get_arg(vm, command_line::arg_test_drop_download_height)); + + if (command_line::get_arg(vm, command_line::arg_test_drop_download) == true) + test_drop_download(); + return true; } //----------------------------------------------------------------------------------------------- diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index f761274c..7e019b53 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -46,6 +46,8 @@ namespace cryptonote struct connection_info { bool incoming; + bool localhost; + bool local_ip; std::string ip; std::string port; @@ -62,8 +64,16 @@ namespace cryptonote uint64_t live_time; + uint64_t avg_download; + uint64_t current_download; + + uint64_t avg_upload; + uint64_t current_upload; + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(incoming) + KV_SERIALIZE(localhost) + KV_SERIALIZE(local_ip) KV_SERIALIZE(ip) KV_SERIALIZE(port) KV_SERIALIZE(peer_id) @@ -73,6 +83,10 @@ namespace cryptonote KV_SERIALIZE(send_idle_time) KV_SERIALIZE(state) KV_SERIALIZE(live_time) + KV_SERIALIZE(avg_download) + KV_SERIALIZE(current_download) + KV_SERIALIZE(avg_upload) + KV_SERIALIZE(current_upload) END_KV_SERIALIZE_MAP() }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 023dd03a..1cf66521 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -210,6 +210,42 @@ namespace cryptonote cnx.live_time = timestamp - cntxt.m_started; + uint32_t ip; + ip = ntohl(cntxt.m_remote_ip); + if (ip == LOCALHOST_INT) + { + cnx.localhost = true; + } + else + { + cnx.localhost = false; + } + + if (ip > 3232235520 && ip < 3232301055) // 192.168.x.x + { + cnx.local_ip = true; + } + else + { + cnx.local_ip = false; + } + + auto connection_time = time(NULL) - cntxt.m_started; + if (connection_time == 0) + { + cnx.avg_download = 0; + cnx.avg_upload = 0; + } + + else + { + cnx.avg_download = cntxt.m_recv_cnt / connection_time / 1024; + cnx.avg_upload = cntxt.m_send_cnt / connection_time / 1024; + } + + cnx.current_download = cntxt.m_current_speed_down / 1024; + cnx.current_upload = cntxt.m_current_speed_up / 1024; + connections.push_back(cnx); return true; @@ -543,6 +579,7 @@ namespace cryptonote LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms"); epee::net_utils::data_logger::get_instance().add_data("calc_time", block_process_time + transactions_process_time); + epee::net_utils::data_logger::get_instance().add_data("block_processing", 1); } // each download block diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 4de8b82b..f9c2af32 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -77,6 +77,9 @@ target_link_libraries(daemon cryptonote_core crypto common + otshell_utils + p2p + cryptonote_protocol daemonizer ${Boost_CHRONO_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 8f023da9..fd7b40be 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -296,4 +296,41 @@ bool t_command_parser_executor::set_limit_down(const std::vector& a return m_executor.set_limit_down(limit); } + +bool t_command_parser_executor::fast_exit(const std::vector& args) +{ + if (!args.empty()) return false; + return m_executor.fast_exit(); +} + +bool t_command_parser_executor::out_peers(const std::vector& args) +{ + if (args.empty()) return false; + + unsigned int limit; + try { + limit = std::stoi(args[0]); + } + + catch(std::invalid_argument& ex) { + _erro("stoi exception"); + return false; + } + + return m_executor.out_peers(limit); +} + +bool t_command_parser_executor::start_save_graph(const std::vector& args) +{ + if (!args.empty()) return false; + return m_executor.start_save_graph(); +} + +bool t_command_parser_executor::stop_save_graph(const std::vector& args) +{ + if (!args.empty()) return false; + return m_executor.stop_save_graph(); +} + + } // namespace daemonize diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index 07d2e70a..27ffabc1 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -93,6 +93,13 @@ public: bool set_limit_down(const std::vector& args); + bool fast_exit(const std::vector& args); + + bool out_peers(const std::vector& args); + + bool start_save_graph(const std::vector& args); + + bool stop_save_graph(const std::vector& args); }; } // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 601b12d5..f0f4cd67 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -150,15 +150,35 @@ t_command_server::t_command_server( , "limit - Set download and upload limit" ); m_command_lookup.set_handler( - "limit-up" + "limit_up" , std::bind(&t_command_parser_executor::set_limit_up, &m_parser, p::_1) , "limit - Set upload limit" ); m_command_lookup.set_handler( - "limit-down" + "limit_down" , std::bind(&t_command_parser_executor::set_limit_down, &m_parser, p::_1) , "limit - Set download limit" ); + m_command_lookup.set_handler( + "fast_exit" + , std::bind(&t_command_parser_executor::fast_exit, &m_parser, p::_1) + , "Exit" + ); + m_command_lookup.set_handler( + "out_peers" + , std::bind(&t_command_parser_executor::out_peers, &m_parser, p::_1) + , "Set max limit of out peers" + ); + m_command_lookup.set_handler( + "start_save_graph" + , std::bind(&t_command_parser_executor::start_save_graph, &m_parser, p::_1) + , "Start save data for dr monero" + ); + m_command_lookup.set_handler( + "stop_save_graph" + , std::bind(&t_command_parser_executor::stop_save_graph, &m_parser, p::_1) + , "Stop save data for dr monero" + ); } bool t_command_server::process_command_str(const std::string& cmd) diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index ec12c281..7931ba03 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -1,5 +1,5 @@ -// Copyright (c) 2014, The Monero Project -// +// Copyright (c) 2014-2015, The Monero Project +// // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -38,10 +38,16 @@ #include "daemon/command_server.h" #include "misc_log_ex.h" #include "version.h" +#include "../../contrib/epee/include/syncobj.h" + +using namespace epee; + #include #include #include +unsigned int epee::g_test_dbg_lock_sleep = 0; + namespace daemonize { struct t_internals { diff --git a/src/daemon/daemon_commands_handler.h b/src/daemon/daemon_commands_handler.h index 7baca596..215cf26d 100644 --- a/src/daemon/daemon_commands_handler.h +++ b/src/daemon/daemon_commands_handler.h @@ -1,7 +1,35 @@ -// Copyright (c) 2012-2013 The Cryptonote developers -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +/* This isn't a header file, may want to refactor this... */ #pragma once #include @@ -44,10 +72,14 @@ public: m_cmd_binder.set_handler("save", boost::bind(&daemon_cmmands_handler::save, this, _1), "Save blockchain"); m_cmd_binder.set_handler("set_log", boost::bind(&daemon_cmmands_handler::set_log, this, _1), "set_log - Change current log detalization level, is a number 0-4"); m_cmd_binder.set_handler("diff", boost::bind(&daemon_cmmands_handler::diff, this, _1), "Show difficulty"); - m_cmd_binder.set_handler("limit-up", boost::bind(&daemon_cmmands_handler::limit_up, this, _1), "Set upload limit"); - m_cmd_binder.set_handler("limit-down", boost::bind(&daemon_cmmands_handler::limit_down, this, _1), "Set download limit"); - m_cmd_binder.set_handler("limit", boost::bind(&daemon_cmmands_handler::limit, this, _1), "Set download and upload limit"); m_cmd_binder.set_handler("out_peers", boost::bind(&daemon_cmmands_handler::out_peers_limit, this, _1), "Set max limit of out peers"); + m_cmd_binder.set_handler("limit_up", boost::bind(&daemon_cmmands_handler::limit_up, this, _1), "Set upload limit [kB/s]"); + m_cmd_binder.set_handler("limit_down", boost::bind(&daemon_cmmands_handler::limit_down, this, _1), "Set download limit [kB/s]"); + m_cmd_binder.set_handler("limit", boost::bind(&daemon_cmmands_handler::limit, this, _1), "Set download and upload limit [kB/s]"); + m_cmd_binder.set_handler("fast_exit", boost::bind(&daemon_cmmands_handler::fast_exit, this, _1), "Exit"); + m_cmd_binder.set_handler("test_drop_download", boost::bind(&daemon_cmmands_handler::test_drop_download, this, _1), "For network testing, drop downloaded blocks instead checking/adding them to blockchain. Can fake-download blocks very fast."); + m_cmd_binder.set_handler("start_save_graph", boost::bind(&daemon_cmmands_handler::start_save_graph, this, _1), ""); + m_cmd_binder.set_handler("stop_save_graph", boost::bind(&daemon_cmmands_handler::stop_save_graph, this, _1), ""); } bool start_handling() @@ -328,6 +360,8 @@ private: PUSH_WARNINGS DISABLE_GCC_WARNING(maybe-uninitialized) log_space::log_singletone::get_set_log_detalisation_level(true, l); + int otshell_utils_log_level = 100 - (l * 25); + gCurrentLogger.setDebugLevel(otshell_utils_log_level); POP_WARNINGS return true; @@ -492,160 +526,4 @@ POP_WARNINGS m_srv.get_payload_object().get_core().get_miner().stop(); return true; } - //-------------------------------------------------------------------------------- - bool out_peers_limit(const std::vector& args) { - if(args.size()!=1) { - std::cout << "Usage: out_peers " << ENDL; - return true; - } - - unsigned int limit; - try { - limit = std::stoi(args[0]); - } - - catch(std::invalid_argument& ex) { - _erro("stoi exception"); - return false; - } - - if (m_srv.m_config.m_net_config.connections_count > limit) - { - m_srv.m_config.m_net_config.connections_count = limit; - epee::net_utils::data_logger::get_instance().add_data("peers_limit", m_srv.m_config.m_net_config.connections_count); - if (m_srv.m_current_number_of_out_peers > limit) - { - int count = m_srv.m_current_number_of_out_peers - limit; - m_srv.delete_connections(count); - } - } - else - { - m_srv.m_config.m_net_config.connections_count = limit; - epee::net_utils::data_logger::get_instance().add_data("peers_limit", m_srv.m_config.m_net_config.connections_count); - } - - return true; - } - //-------------------------------------------------------------------------------- - bool limit_up(const std::vector& args) - { - if(args.size()!=1) { - std::cout << "Usage: limit_up " << ENDL; - return false; - } - - int limit; - try { - limit = std::stoi(args[0]); - } - catch(std::invalid_argument& ex) { - return false; - } - - if (limit==-1) { - limit=128; - //this->islimitup=false; - } - - limit *= 1024; - - - //nodetool::epee::net_utils::connection >::set_rate_up_limit( limit ); - epee::net_utils::connection_basic::set_rate_up_limit( limit ); - std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; - - return true; - } - - //-------------------------------------------------------------------------------- - bool limit_down(const std::vector& args) - { - - if(args.size()!=1) { - std::cout << "Usage: limit_down " << ENDL; - return true; - } - - int limit; - try { - limit = std::stoi(args[0]); - } - - catch(std::invalid_argument& ex) { - return false; - } - - if (limit==-1) { - limit=128; - //this->islimitup=false; - } - - limit *= 1024; - - - //nodetool::epee::net_utils::connection >::set_rate_up_limit( limit ); - epee::net_utils::connection_basic::set_rate_down_limit( limit ); - std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; - - return true; - } - - //-------------------------------------------------------------------------------- - bool limit(const std::vector& args) - { - if(args.size()!=1) { - std::cout << "Usage: limit_down " << ENDL; - return true; - } - - int limit; - try { - limit = std::stoi(args[0]); - } - catch(std::invalid_argument& ex) { - return false; - } - - if (limit==-1) { - limit=128; - //this->islimitup=false; - } - - limit *= 1024; - - - //nodetool::epee::net_utils::connection >::set_rate_up_limit( limit ); - epee::net_utils::connection_basic::set_rate_down_limit( limit ); - epee::net_utils::connection_basic::set_rate_up_limit( limit ); - std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; - std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; - - return true; - } - //-------------------------------------------------------------------------------- - bool fast_exit(const std::vector& args) - { - m_srv.get_payload_object().get_core().set_fast_exit(); - m_srv.send_stop_signal(); - return true; - } - //-------------------------------------------------------------------------------- - bool test_drop_download(const std::vector& args) - { - m_srv.get_payload_object().get_core().test_drop_download(); - return true; - } - //-------------------------------------------------------------------------------- - bool start_save_graph(const std::vector& args) - { - m_srv.set_save_graph(true); - return true; - } - //-------------------------------------------------------------------------------- - bool stop_save_graph(const std::vector& args) - { - m_srv.set_save_graph(false); - return true; - } }; diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 5d8baf49..6f7511e2 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -71,7 +71,10 @@ int main(int argc, char const * argv[]) command_line::add_arg(visible_options, command_line::arg_testnet_data_dir, default_testnet_data_dir.string()); bf::path default_conf = default_data_dir / std::string(CRYPTONOTE_NAME ".conf"); command_line::add_arg(visible_options, daemon_args::arg_config_file, default_conf.string()); - + command_line::add_arg(visible_options, command_line::arg_test_drop_download); + command_line::add_arg(visible_options, command_line::arg_test_dbg_lock_sleep); + command_line::add_arg(visible_options, command_line::arg_test_drop_download_height); + // Settings bf::path default_log = default_data_dir / std::string(CRYPTONOTE_NAME ".log"); command_line::add_arg(core_settings, daemon_args::arg_log_file, default_log.string()); @@ -127,6 +130,8 @@ int main(int argc, char const * argv[]) std::cout << "OS: " << tools::get_os_version_string() << ENDL; return 0; } + + epee::g_test_dbg_lock_sleep = command_line::get_arg(vm, command_line::arg_test_dbg_lock_sleep); bool testnet_mode = command_line::get_arg(vm, daemon_args::arg_testnet_on); @@ -209,6 +214,8 @@ int main(int argc, char const * argv[]) else if (epee::log_space::get_set_log_detalisation_level(false) != new_log_level) { epee::log_space::get_set_log_detalisation_level(true, new_log_level); + int otshell_utils_log_level = 100 - (new_log_level * 25); + gCurrentLogger.setDebugLevel(otshell_utils_log_level); LOG_PRINT_L0("LOG_LEVEL set to " << new_log_level); } } @@ -223,6 +230,7 @@ int main(int argc, char const * argv[]) , log_file_path.parent_path().string().c_str() ); } + _erro("Test error"); return daemonizer::daemonize(argc, argv, daemonize::t_executor{}, vm); } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index f06f4854..46900b07 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -32,6 +32,7 @@ #include "common/scoped_message_writer.h" #include "daemon/rpc_command_executor.h" #include "rpc/core_rpc_server_commands_defs.h" +#include "cryptonote_core/cryptonote_core.h" #include #include @@ -267,11 +268,38 @@ bool t_rpc_command_executor::print_connections() { } } + tools::msg_writer() << std::setw(30) << std::left << "Remote Host" + << std::setw(20) << "Peer id" + << std::setw(30) << "Recv/Sent (inactive,sec)" + << std::setw(25) << "State" + << std::setw(20) << "Livetime(sec)" + << std::setw(12) << "Down (kB/s)" + << std::setw(14) << "Down(now)" + << std::setw(10) << "Up (kB/s)" + << std::setw(13) << "Up(now)" + << std::endl; + for (auto & info : res.connections) { - std::string address = info.ip + ":" + info.port; - std::string in_out = info.incoming ? "INC" : "OUT"; - tools::msg_writer() << boost::format("%-25s peer_id: %-25s %s") % address % info.peer_id % in_out; + std::string address = info.incoming ? "INC " : "OUT "; + address += info.ip + ":" + info.port; + //std::string in_out = info.incoming ? "INC " : "OUT "; + tools::msg_writer() + //<< std::setw(30) << std::left << in_out + << std::setw(30) << std::left << address + << std::setw(20) << info.peer_id + << std::setw(30) << std::to_string(info.recv_count) + "(" + std::to_string(info.recv_idle_time) + ")/" + std::to_string(info.send_count) + "(" + std::to_string(info.send_idle_time) + ")" + << std::setw(25) << info.state + << std::setw(20) << info.live_time + << std::setw(12) << info.avg_download + << std::setw(14) << info.current_download + << std::setw(10) << info.avg_upload + << std::setw(13) << info.current_upload + + << std::left << (info.localhost ? "[LOCALHOST]" : "") + << std::left << (info.local_ip ? "[LAN]" : ""); + //tools::msg_writer() << boost::format("%-25s peer_id: %-25s %s") % address % info.peer_id % in_out; + } return true; @@ -659,34 +687,134 @@ bool t_rpc_command_executor::print_status() bool t_rpc_command_executor::set_limit(int limit) { -/* epee::net_utils::connection_basic::set_rate_down_limit( limit ); epee::net_utils::connection_basic::set_rate_up_limit( limit ); std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; -*/ - return true; } bool t_rpc_command_executor::set_limit_up(int limit) { -/* epee::net_utils::connection_basic::set_rate_up_limit( limit ); std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; -*/ - return true; } bool t_rpc_command_executor::set_limit_down(int limit) { -/* epee::net_utils::connection_basic::set_rate_down_limit( limit ); std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; -*/ - return true; } +bool t_rpc_command_executor::fast_exit() +{ + cryptonote::COMMAND_RPC_FAST_EXIT::request req; + cryptonote::COMMAND_RPC_FAST_EXIT::response res; + + std::string fail_message = "Daemon did not stop"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/fast_exit", fail_message.c_str())) + { + return true; + } + } + + else + { + if (!m_rpc_server->on_fast_exit(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "Daemon stopped"; + return true; +} + +bool t_rpc_command_executor::out_peers(uint64_t limit) +{ + cryptonote::COMMAND_RPC_OUT_PEERS::request req; + cryptonote::COMMAND_RPC_OUT_PEERS::response res; + + epee::json_rpc::error error_resp; + + req.out_peers = limit; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "/out_peers", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_out_peers(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + return true; +} + +bool t_rpc_command_executor::start_save_graph() +{ + cryptonote::COMMAND_RPC_START_SAVE_GRAPH::request req; + cryptonote::COMMAND_RPC_START_SAVE_GRAPH::response res; + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/start_save_graph", fail_message.c_str())) + { + return true; + } + } + + else + { + if (!m_rpc_server->on_start_save_graph(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + return true; +} + +bool t_rpc_command_executor::stop_save_graph() +{ + cryptonote::COMMAND_RPC_STOP_SAVE_GRAPH::request req; + cryptonote::COMMAND_RPC_STOP_SAVE_GRAPH::response res; + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/stop_save_graph", fail_message.c_str())) + { + return true; + } + } + + else + { + if (!m_rpc_server->on_stop_save_graph(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + return true; +} + }// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index fe0181e6..43b8a9fe 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -105,7 +105,13 @@ public: bool set_limit_down(int limit); - + bool fast_exit(); + + bool out_peers(uint64_t limit); + + bool start_save_graph(); + + bool stop_save_graph(); }; } // namespace daemonize diff --git a/src/p2p/data_logger.cpp b/src/p2p/data_logger.cpp index 69e50141..bbb9fba3 100644 --- a/src/p2p/data_logger.cpp +++ b/src/p2p/data_logger.cpp @@ -43,6 +43,7 @@ namespace net_utils mFilesMap["sleep_up"] = data_logger::fileData("log/dr-monero/up_sleep_log.data"); mFilesMap["calc_time"] = data_logger::fileData("log/dr-monero/get_objects_calc_time.data"); mFilesMap["blockchain_processing_time"] = data_logger::fileData("log/dr-monero/blockchain_log.data"); + mFilesMap["block_processing"] = data_logger::fileData("log/dr-monero/block_proc.data"); mFilesMap["peers_limit"] = data_logger::fileData("log/dr-monero/peers_limit.info"); mFilesMap["download_limit"] = data_logger::fileData("log/dr-monero/limit_down.info"); @@ -109,6 +110,15 @@ namespace net_utils mFilesMap[filename].mDataToSave += data; // this holds a number that should be sum of all accumulated samples } } + + bool data_logger::is_dying() { + if (m_state == data_logger_state::state_dying) { + return true; + } + else { + return false; + } + } void data_logger::saveToFile() { _dbg2_c("dbg/data","saving to files"); @@ -125,10 +135,13 @@ namespace net_utils // the inner class: double data_logger::fileData::get_current_time() { - using namespace boost::chrono; - auto point = steady_clock::now(); + #if defined(__APPLE__) + auto point = std::chrono::system_clock::now(); + #else + auto point = std::chrono::steady_clock::now(); + #endif auto time_from_epoh = point.time_since_epoch(); - auto ms = duration_cast< milliseconds >( time_from_epoh ).count(); + auto ms = std::chrono::duration_cast< std::chrono::milliseconds >( time_from_epoh ).count(); double ms_f = ms; return ms_f / 1000.; } diff --git a/src/p2p/data_logger.hpp b/src/p2p/data_logger.hpp index 21591216..f38dacdc 100644 --- a/src/p2p/data_logger.hpp +++ b/src/p2p/data_logger.hpp @@ -40,6 +40,7 @@ enum class data_logger_state { state_before_init, state_during_init, state_ready void add_data(std::string filename, unsigned int data); ///< use this to append data here. Use it only the singleton. It locks itself. static std::atomic m_save_graph; ///< global setting flag, should we save all the data or not (can disable logging graphs data) + static bool is_dying(); private: static std::once_flag m_singleton; ///< to guarantee singleton creates the object exactly once diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index f94fedae..d956b37f 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -80,13 +80,16 @@ namespace nodetool public: typedef t_payload_net_handler payload_net_handler; - node_server( - t_payload_net_handler& payload_handler - ) - : m_payload_handler(payload_handler) - , m_allow_local_ip(false) - , m_hide_my_port(false) - {} + node_server(t_payload_net_handler& payload_handler) + :m_payload_handler(payload_handler), + m_allow_local_ip(false), + m_no_igd(false), + m_hide_my_port(false) + { + m_current_number_of_out_peers = 0; + m_save_graph = false; + is_closing = false; + } static void init_options(boost::program_options::options_description& desc); @@ -233,12 +236,12 @@ namespace nodetool public: config m_config; // TODO was private, add getters? std::atomic m_current_number_of_out_peers; + void set_save_graph(bool save_graph) { m_save_graph = save_graph; epee::net_utils::connection_basic::set_save_graph(save_graph); } - private: std::string m_config_folder; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index ef158d9a..c7413ec1 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -46,6 +46,7 @@ #include "net/local_ip.h" #include "crypto/crypto.h" #include "storages/levin_abstract_invoke2.h" +#include "data_logger.hpp" #include "daemon/command_line_args.h" // We have to look for miniupnpc headers in different places, dependent on if its compiled or external @@ -93,6 +94,8 @@ namespace nodetool const command_line::arg_descriptor arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", -1}; const command_line::arg_descriptor arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", -1}; const command_line::arg_descriptor arg_limit_rate = {"limit-rate", "set limit-rate [kB/s]", 128}; + + const command_line::arg_descriptor arg_save_graph = {"save-graph", "Save data for dr monero", false}; } //----------------------------------------------------------------------------------- @@ -114,7 +117,9 @@ namespace nodetool command_line::add_arg(desc, arg_tos_flag); command_line::add_arg(desc, arg_limit_rate_up); command_line::add_arg(desc, arg_limit_rate_down); - command_line::add_arg(desc, arg_limit_rate); } + command_line::add_arg(desc, arg_limit_rate); + command_line::add_arg(desc, arg_save_graph); + } //----------------------------------------------------------------------------------- template bool node_server::init_config() @@ -193,6 +198,11 @@ namespace nodetool m_command_line_peers.push_back(pe); } } + + if(command_line::has_arg(vm, arg_save_graph)) + { + set_save_graph(true); + } if (command_line::has_arg(vm,arg_p2p_add_exclusive_node)) { @@ -294,29 +304,6 @@ namespace nodetool std::vector> dns_results; dns_results.resize(m_seed_nodes_list.size()); - - // creating thread to log number of connections - mPeersLoggerThread.reset(new std::thread([&]() - { - _note("Thread monitor number of peers - start"); - while (!is_closing) - { // main loop of thread - //number_of_peers = m_net_server.get_config_object().get_connections_count(); - unsigned int number_of_peers = 0; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) - { - if (!cntxt.m_is_income) ++number_of_peers; - return true; - }); // lambda - - m_current_number_of_out_peers = number_of_peers; - epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers); - - std::this_thread::sleep_for(std::chrono::seconds(1)); - } // main loop of thread - _note("Thread monitor number of peers - done"); - })); // lambda - std::list dns_threads; uint64_t result_index = 0; @@ -483,6 +470,30 @@ namespace nodetool template bool node_server::run() { + // creating thread to log number of connections + mPeersLoggerThread.reset(new std::thread([&]() + { + _note("Thread monitor number of peers - start"); + while (!is_closing) + { // main loop of thread + //number_of_peers = m_net_server.get_config_object().get_connections_count(); + unsigned int number_of_peers = 0; + m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if (!cntxt.m_is_income) ++number_of_peers; + return true; + }); // lambda + + m_current_number_of_out_peers = number_of_peers; + if (epee::net_utils::data_logger::is_dying()) + break; + epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers); + + std::this_thread::sleep_for(std::chrono::seconds(1)); + } // main loop of thread + _note("Thread monitor number of peers - done"); + })); // lambda + //here you can set worker threads count int thrds_count = 10; @@ -516,7 +527,6 @@ namespace nodetool kill(); m_peerlist.deinit(); m_net_server.deinit_server(); - return store_config(); } //----------------------------------------------------------------------------------- diff --git a/src/p2p/network_throttle-detail.cpp b/src/p2p/network_throttle-detail.cpp index 7426e6dc..6fa27b62 100644 --- a/src/p2p/network_throttle-detail.cpp +++ b/src/p2p/network_throttle-detail.cpp @@ -330,10 +330,13 @@ void network_throttle::calculate_times(size_t packet_size, calculate_times_struc } double network_throttle::get_time_seconds() const { - using namespace std::chrono; - auto point = steady_clock::now(); + #if defined(__APPLE__) + auto point = std::chrono::system_clock::now(); + #else + auto point = std::chrono::steady_clock::now(); + #endif auto time_from_epoh = point.time_since_epoch(); - auto ms = duration_cast< milliseconds >( time_from_epoh ).count(); + auto ms = std::chrono::duration_cast< std::chrono::milliseconds >( time_from_epoh ).count(); double ms_f = ms; return ms_f / 1000.; } diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index d737ee6f..cb8a8426 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -45,6 +45,7 @@ bitmonero_add_library(rpc target_link_libraries(rpc LINK_PRIVATE cryptonote_core + cryptonote_protocol ${Boost_CHRONO_LIBRARY} ${Boost_REGEX_LIBRARY} ${Boost_SYSTEM_LIBRARY} diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index f29d0c60..71eb5b75 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -753,6 +753,45 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_fast_exit(const COMMAND_RPC_FAST_EXIT::request& req, COMMAND_RPC_FAST_EXIT::response& res) + { + cryptonote::core::set_fast_exit(); + m_p2p.deinit(); + m_core.deinit(); + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res) + { + // TODO + /*if (m_p2p.get_outgoing_connections_count() > req.out_peers) + { + m_p2p.m_config.m_net_config.connections_count = req.out_peers; + if (m_p2p.get_outgoing_connections_count() > req.out_peers) + { + int count = m_p2p.get_outgoing_connections_count() - req.out_peers; + m_p2p.delete_connections(count); + } + } + + else + m_p2p.m_config.m_net_config.connections_count = req.out_peers; + */ + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res) + { + m_p2p.set_save_graph(true); + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res) + { + m_p2p.set_save_graph(false); + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ const command_line::arg_descriptor core_rpc_server::arg_rpc_bind_ip = { "rpc-bind-ip" diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 6e603acb..cee8df25 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -87,6 +87,10 @@ namespace cryptonote MAP_URI_AUTO_JON2("/get_transaction_pool", on_get_transaction_pool, COMMAND_RPC_GET_TRANSACTION_POOL) MAP_URI_AUTO_JON2("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON) MAP_URI_AUTO_JON2("/getinfo", on_get_info, COMMAND_RPC_GET_INFO) + MAP_URI_AUTO_JON2("/fast_exit", on_fast_exit, COMMAND_RPC_FAST_EXIT) + MAP_URI_AUTO_JON2("/out_peers", on_out_peers, COMMAND_RPC_OUT_PEERS) + MAP_URI_AUTO_JON2("/start_save_graph", on_start_save_graph, COMMAND_RPC_START_SAVE_GRAPH) + MAP_URI_AUTO_JON2("/stop_save_graph", on_stop_save_graph, COMMAND_RPC_STOP_SAVE_GRAPH) BEGIN_JSON_RPC_MAP("/json_rpc") MAP_JON_RPC("getblockcount", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT) MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) @@ -116,6 +120,10 @@ namespace cryptonote bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res); bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res); bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res); + bool on_fast_exit(const COMMAND_RPC_FAST_EXIT::request& req, COMMAND_RPC_FAST_EXIT::response& res); + bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res); + bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res); + bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res); //json_rpc bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 5cb54752..e54dec2c 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -703,5 +703,79 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; }; + + struct COMMAND_RPC_FAST_EXIT + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_OUT_PEERS + { + struct request + { + uint64_t out_peers; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(out_peers) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_START_SAVE_GRAPH + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_STOP_SAVE_GRAPH + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; } diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 4ecf00c9..9ac80fa9 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -67,6 +67,8 @@ namespace po = boost::program_options; #define EXTENDED_LOGS_FILE "wallet_details.log" +unsigned int epee::g_test_dbg_lock_sleep = 0; + #define DEFAULT_MIX 3 namespace From f9dba47a17f60cf23f8549bd824c0d116d00c282 Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Wed, 1 Apr 2015 19:15:38 +0200 Subject: [PATCH 09/16] added windows_stream.* console colors --- contrib/otshell_utils/windows_stream.cpp | 64 ++++++++++++++++++++++++ contrib/otshell_utils/windows_stream.h | 20 ++++++++ 2 files changed, 84 insertions(+) create mode 100644 contrib/otshell_utils/windows_stream.cpp create mode 100644 contrib/otshell_utils/windows_stream.h diff --git a/contrib/otshell_utils/windows_stream.cpp b/contrib/otshell_utils/windows_stream.cpp new file mode 100644 index 00000000..59d8b12a --- /dev/null +++ b/contrib/otshell_utils/windows_stream.cpp @@ -0,0 +1,64 @@ +#if defined(_WIN32) +#include "windows_stream.h" +#include + +windows_stream::windows_stream(unsigned int pLevel) + : + mLevel(pLevel) +{ +} + +std::ostream& operator << (std::ostream &stream, windows_stream const& object) +{ + HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + + if (object.mLevel >= 100) + { + SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_RED | BACKGROUND_INTENSITY); + return stream; + } + if (object.mLevel >= 90) + { + SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_GREEN | BACKGROUND_RED | BACKGROUND_INTENSITY); + return stream; + } + if (object.mLevel >= 80) + { + SetConsoleTextAttribute(h_stdout, BACKGROUND_BLUE | BACKGROUND_RED | BACKGROUND_INTENSITY); + return stream; + } + if (object.mLevel >= 75) + { + SetConsoleTextAttribute(h_stdout, BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_INTENSITY); + return stream; + } + if (object.mLevel >= 70) + { + SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_INTENSITY); + return stream; + } + if (object.mLevel >= 50) + { + SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY); + return stream; + } + if (object.mLevel >= 40) + { + SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_RED | FOREGROUND_INTENSITY); + return stream; + } + if (object.mLevel >= 30) + { + SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_INTENSITY); + return stream; + } + if (object.mLevel >= 20) + { + SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE); + return stream; + } + + return stream; +} + +#endif diff --git a/contrib/otshell_utils/windows_stream.h b/contrib/otshell_utils/windows_stream.h new file mode 100644 index 00000000..859e7ee5 --- /dev/null +++ b/contrib/otshell_utils/windows_stream.h @@ -0,0 +1,20 @@ +#ifndef WINDOWS_STREAM_H +#define WINDOWS_STREAM_H + +#if defined(_WIN32) + +#include +#include + +class windows_stream +{ +public: + windows_stream(unsigned int pLevel); + friend std::ostream& operator<<(std::ostream &stream, windows_stream const& object); +private: + unsigned int mLevel = 0; +}; + +#endif // _WIN32 + +#endif // WINDOWS_STREAM_H From 1489310d5392a55de225ea55a0ebd90fc4dc7b1a Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Wed, 1 Apr 2015 19:22:25 +0200 Subject: [PATCH 10/16] doxygen related tool --- tools/doxygen-publish.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 tools/doxygen-publish.sh diff --git a/tools/doxygen-publish.sh b/tools/doxygen-publish.sh new file mode 100755 index 00000000..ad02e283 --- /dev/null +++ b/tools/doxygen-publish.sh @@ -0,0 +1,25 @@ +#!/bin/bash -e + +# maintainer (ask me any questions): rfree + +if [[ ! -r "Doxyfile" ]] ; then + echo "Error, can not read the Doxyfile - make sure to run this script from top of monero project, where the Doxyfile file is located" + exit 1 +fi + +wwwdir="$HOME/monero-www/" +if [[ ! -w "$wwwdir" ]] ; then + echo "Error, can not write into wwwdir=$wwwdir. It should be a directory readable/connected to your webserver, or a symlink to such directory" + exit 1 +fi + +if [[ ! -d "$wwwdir/doc" ]] ; then + echo "Creating subdirs" + mkdir "$wwwdir/doc" +fi + +echo "Generating:" +doxygen Doxyfile && echo "Backup previous version:" && rm -rf ~/monero-www-previous && mv "$wwwdir/doc" ~/monero-www-previous && cp -ar doc/ "$wwwdir/" && echo "Done, builded and copied to public - the doxygen docs" && echo "size:" && du -Dsh "$wwwdir/" && echo "files:" && find "$wwwdir/" | wc -l + + + From 2900b1e76cf89c46e5600a6d02a2d75fc2b0bd9a Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Wed, 1 Apr 2015 19:23:15 +0200 Subject: [PATCH 11/16] doxygen files --- cmake/Doxyfile.in | 1803 ++++++++++++++++++++++++++++++++++++ cmake/Doxygen.extra.css.in | 14 + 2 files changed, 1817 insertions(+) create mode 100644 cmake/Doxyfile.in create mode 100644 cmake/Doxygen.extra.css.in diff --git a/cmake/Doxyfile.in b/cmake/Doxyfile.in new file mode 100644 index 00000000..35a4911f --- /dev/null +++ b/cmake/Doxyfile.in @@ -0,0 +1,1803 @@ +# Doxyfile 1.8.1.2 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# http://www.gnu.org/software/libiconv for the list of possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or sequence of words) that should +# identify the project. Note that if you do not use Doxywizard you need +# to put quotes around the project name if it contains spaces. + +PROJECT_NAME = Monero + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = @VERSION_STRING@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer +# a quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = @CPACK_PACKAGE_DESCRIPTION_SUMMARY@ + +# With the PROJECT_LOGO tag one can specify an logo or icon that is +# included in the documentation. The maximum height of the logo should not +# exceed 55 pixels and the maximum width should not exceed 200 pixels. +# Doxygen will copy the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = ./docs + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = YES + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, +# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English +# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, +# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, +# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful if your file system +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# This tag can be used to specify a number of word-keyword mappings (TCL only). +# A mapping has the form "name=value". For example adding +# "class=itcl::class" will allow you to use the command class in the +# itcl::class meaning. + +TCL_SUBST = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for +# Java. For instance, namespaces will be presented as packages, qualified +# scopes will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources only. Doxygen will then generate output that is more tailored for +# Fortran. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for +# VHDL. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given extension. +# Doxygen has a built-in mapping, but you can override or extend it using this +# tag. The format is ext=language, where ext is a file extension, and language +# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, +# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make +# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C +# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions +# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If MARKDOWN_SUPPORT is enabled (the default) then doxygen pre-processes all +# comments according to the Markdown format, which allows for more readable +# documentation. See http://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you +# can mix doxygen, HTML, and XML commands with Markdown formatting. +# Disable only in case of backward compatibilities issues. + +MARKDOWN_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also makes the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = YES + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate getter +# and setter methods for a property. Setting this option to YES (the default) +# will make doxygen replace the get and set methods by a property in the +# documentation. This will only work if the methods are indeed getting or +# setting a simple type. If this is not the case, or you want to show the +# methods anyway, you should set this option to NO. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and +# unions are shown inside the group in which they are included (e.g. using +# @ingroup) instead of on a separate page (for HTML and Man pages) or +# section (for LaTeX and RTF). + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and +# unions with only public data fields will be shown inline in the documentation +# of the scope in which they are defined (i.e. file, namespace, or group +# documentation), provided this scope is documented. If set to NO (the default), +# structs, classes, and unions are shown on a separate page (for HTML and Man +# pages) or section (for LaTeX and RTF). + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum +# is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. + +TYPEDEF_HIDES_STRUCT = NO + +# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to +# determine which symbols to keep in memory and which to flush to disk. +# When the cache is full, less often used symbols will be written to disk. +# For small to medium size projects (<1000 input files) the default value is +# probably good enough. For larger projects a too small cache size can cause +# doxygen to be busy swapping symbols to and from disk most of the time +# causing a significant performance penalty. +# If the system has enough physical memory increasing the cache will improve the +# performance by keeping more symbols in memory. Note that the value works on +# a logarithmic scale so increasing the size by one will roughly double the +# memory usage. The cache size is given by this formula: +# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols. + +SYMBOL_CACHE_SIZE = 0 + +# Similar to the SYMBOL_CACHE_SIZE the size of the symbol lookup cache can be +# set using LOOKUP_CACHE_SIZE. This cache is used to resolve symbols given +# their name and scope. Since this can be an expensive process and often the +# same symbol appear multiple times in the code, doxygen keeps a cache of +# pre-resolved symbols. If the cache is too small doxygen will become slower. +# If the cache is too large, memory is wasted. The cache size is given by this +# formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols. + +LOOKUP_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = YES + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_PACKAGE tag is set to YES all members with package or internal +# scope will be included in the documentation. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base +# name of the file that contains the anonymous namespace. By default +# anonymous namespaces are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = NO + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen +# will list include files with double quotes in the documentation +# rather than with sharp brackets. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen +# will sort the (brief and detailed) documentation of class members so that +# constructors and destructors are listed first. If set to NO (the default) +# the constructors will appear in the respective orders defined by +# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. +# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO +# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the +# hierarchy of group names into alphabetical order. If set to NO (the default) +# the group names will appear in their defined order. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to +# do proper type resolution of all parameters of a function it will reject a +# match between the prototype and the implementation of a member function even +# if there is only one candidate or it is obvious which candidate to choose +# by doing a simple string match. By disabling STRICT_PROTO_MATCHING doxygen +# will still accept a match between prototype and implementation in such cases. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or macro consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and macros in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. +# This will remove the Files entry from the Quick Index and from the +# Folder Tree View (if specified). The default is YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the +# Namespaces page. This will remove the Namespaces entry from the Quick Index +# and from the Folder Tree View (if specified). The default is YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command , where is the value of +# the FILE_VERSION_FILTER tag, and is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. +# You can optionally specify a file name after the option, if omitted +# DoxygenLayout.xml will be used as the name of the layout file. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files +# containing the references data. This must be a list of .bib files. The +# .bib extension is automatically appended if omitted. Using this command +# requires the bibtex tool to be installed. See also +# http://en.wikipedia.org/wiki/BibTeX for more info. For LaTeX the style +# of the bibliography can be controlled using LATEX_BIB_STYLE. To use this +# feature you need bibtex and perl available in the search path. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# The WARN_NO_PARAMDOC option can be enabled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = ../README.md \ + ../contrib/ \ + ../external/ \ + ../include/ \ + ../tests/ \ + ../src/ + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is +# also the default input encoding. Doxygen uses libiconv (or the iconv built +# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for +# the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.d *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh +# *.hxx *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.dox *.py +# *.f90 *.f *.for *.vhd *.vhdl + +FILE_PATTERNS = *.cpp \ + *.h \ + *.hpp + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = *.pb.* \ + tinythread.* \ + fast_mutex.* \ + anyoption.* \ + stacktrace.* \ + */simpleini/* \ + ExportWrapper.* \ + WinsockWrapper.* \ + otapicli.* \ + test*.* \ + irrXML.* \ + */chaiscript/* \ + */zmq/* + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = tinythread + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command , where +# is the value of the INPUT_FILTER tag, and is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty or if +# non of the patterns match the file name, INPUT_FILTER is applied. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) +# and it is also possible to disable source filtering for a specific pattern +# using *.ext= (so without naming a filter). This option only has effect when +# FILTER_SOURCE_FILES is enabled. + +FILTER_SOURCE_PATTERNS = + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = YES + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = YES + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C, C++ and Fortran comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. Otherwise they will link to the documentation. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = YES + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. Note that when using a custom header you are responsible +# for the proper inclusion of any scripts and style sheets that doxygen +# needs, which is dependent on the configuration options used. +# It is advised to generate a default header using "doxygen -w html +# header.html footer.html stylesheet.css YourConfigFile" and then modify +# that header. Note that the header is subject to change so you typically +# have to redo this when upgrading to a newer version of doxygen or when +# changing the value of configuration settings such as GENERATE_TREEVIEW! + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# style sheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath$ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that +# the files will be copied as-is; there are no commands or markers available. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. +# Doxygen will adjust the colors in the style sheet and background images +# according to this color. Hue is specified as an angle on a colorwheel, +# see http://en.wikipedia.org/wiki/Hue for more information. +# For instance the value 0 represents red, 60 is yellow, 120 is green, +# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. +# The allowed range is 0 to 359. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of +# the colors in the HTML output. For a value of 0 the output will use +# grayscales only. A value of 255 will produce the most vivid colors. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to +# the luminance component of the colors in the HTML output. Values below +# 100 gradually make the output lighter, whereas values above 100 make +# the output darker. The value divided by 100 is the actual gamma applied, +# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, +# and 100 does not change the gamma. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting +# this to NO can help when comparing the output of multiple runs. + +HTML_TIMESTAMP = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. + +HTML_DYNAMIC_SECTIONS = NO + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of +# entries shown in the various tree structured indices initially; the user +# can expand and collapse entries dynamically later on. Doxygen will expand +# the tree to such a level that at most the specified number of entries are +# visible (unless a fully collapsed tree already exceeds this amount). +# So setting the number of entries 1 will produce a full collapsed tree by +# default. 0 is a special value representing an infinite number of entries +# and will result in a full expanded tree by default. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files +# will be generated that can be used as input for Apple's Xcode 3 +# integrated development environment, introduced with OSX 10.5 (Leopard). +# To create a documentation set, doxygen will generate a Makefile in the +# HTML output directory. Running make will produce the docset in that +# directory and running "make install" will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find +# it at startup. +# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. + +GENERATE_DOCSET = NO + +# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the +# feed. A documentation feed provides an umbrella under which multiple +# documentation sets from a single provider (such as a company or product suite) +# can be grouped. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that +# should uniquely identify the documentation set bundle. This should be a +# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen +# will append .docset to the name. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING +# is used to encode HtmlHelp index (hhk), content (hhc) and project file +# content. + +CHM_INDEX_ENCODING = + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated +# that can be used as input for Qt's qhelpgenerator to generate a +# Qt Compressed Help (.qch) of the generated HTML documentation. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can +# be used to specify the file name of the resulting .qch file. +# The path specified is relative to the HTML output folder. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#namespace + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#virtual-folders + +QHP_VIRTUAL_FOLDER = doc + +# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to +# add. For more information please see +# http://doc.trolltech.com/qthelpproject.html#custom-filters + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see +# +# Qt Help Project / Custom Filters. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's +# filter section matches. +# +# Qt Help Project / Filter Attributes. + +QHP_SECT_FILTER_ATTRS = + +# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can +# be used to specify the location of Qt's qhelpgenerator. +# If non-empty doxygen will try to run qhelpgenerator on the generated +# .qhp file. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files +# will be generated, which together with the HTML files, form an Eclipse help +# plugin. To install this plugin and make it available under the help contents +# menu in Eclipse, the contents of the directory containing the HTML and XML +# files needs to be copied into the plugins directory of eclipse. The name of +# the directory within the plugins directory should be the same as +# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before +# the help appears. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have +# this name. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) +# at top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. Since the tabs have the same information as the +# navigation tree you can set this option to NO if you already set +# GENERATE_TREEVIEW to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. +# If the tag value is set to YES, a side panel will be generated +# containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). +# Windows users are probably better off using the HTML help feature. +# Since the tree basically has the same information as the tab index you +# could consider to set DISABLE_INDEX to NO when enabling this option. + +GENERATE_TREEVIEW = YES + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values +# (range [0,1..20]) that doxygen will group on one line in the generated HTML +# documentation. Note that a value of 0 will completely suppress the enum +# values from appearing in the overview section. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open +# links to external symbols imported via tag files in a separate window. + +EXT_LINKS_IN_WINDOW = YES + +# Use this tag to change the font size of Latex formulas included +# as images in the HTML documentation. The default is 10. Note that +# when you change the font size after a successful doxygen run you need +# to manually remove any form_*.png images from the HTML output directory +# to force them to be regenerated. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are +# not supported properly for IE 6.0, but are supported on all modern browsers. +# Note that when changing this option you need to delete any form_*.png files +# in the HTML output before the changes have effect. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax +# (see http://www.mathjax.org) which uses client side Javascript for the +# rendering instead of using prerendered bitmaps. Use this if you do not +# have LaTeX installed or if you want to formulas look prettier in the HTML +# output. When enabled you may also need to install MathJax separately and +# configure the path to it using the MATHJAX_RELPATH option. + +USE_MATHJAX = NO + +# When MathJax is enabled you need to specify the location relative to the +# HTML output directory using the MATHJAX_RELPATH option. The destination +# directory should contain the MathJax.js script. For instance, if the mathjax +# directory is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to +# the MathJax Content Delivery Network so you can quickly see the result without +# installing MathJax. However, it is strongly recommended to install a local +# copy of MathJax from http://www.mathjax.org before deployment. + +MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest + +# The MATHJAX_EXTENSIONS tag can be used to specify one or MathJax extension +# names that should be enabled during MathJax rendering. + +MATHJAX_EXTENSIONS = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box +# for the HTML output. The underlying search engine uses javascript +# and DHTML and should work on any modern browser. Note that when using +# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets +# (GENERATE_DOCSET) there is already a search function so this one should +# typically be disabled. For large projects the javascript based search engine +# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. + +SEARCHENGINE = YES + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a PHP enabled web server instead of at the web client +# using Javascript. Doxygen will generate the search PHP script and index +# file to put on the web server. The advantage of the server +# based approach is that it scales better to large projects and allows +# full text search. The disadvantages are that it is more difficult to setup +# and does not have live searching capabilities. + +SERVER_BASED_SEARCH = NO + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. +# Note that when enabling USE_PDFLATEX this option is only used for +# generating bitmaps for formulas in the HTML output, but not in the +# Makefile that is written to the output directory. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4 + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for +# the generated latex document. The footer should contain everything after +# the last chapter. If it is left blank doxygen will generate a +# standard footer. Notice: only use this tag if you know what you are doing! + +LATEX_FOOTER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +# If LATEX_SOURCE_CODE is set to YES then doxygen will include +# source code with syntax highlighting in the LaTeX output. +# Note that which sources are shown also depends on other settings +# such as SOURCE_BROWSER. + +LATEX_SOURCE_CODE = NO + +# The LATEX_BIB_STYLE tag can be used to specify the style to use for the +# bibliography, e.g. plainnat, or ieeetr. The default style is "plain". See +# http://en.wikipedia.org/wiki/BibTeX for more info. + +LATEX_BIB_STYLE = plain + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load style sheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# pointed to by INCLUDE_PATH will be searched when a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = OT_CRYPTO_USING_OPENSSL \ + OT_CASH_USING_LUCRE + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition that +# overrules the definition found in the source code. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all references to function-like macros +# that are alone on a line, have an all uppercase name, and do not end with a +# semicolon, because these will confuse the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. For each +# tag file the location of the external documentation should be added. The +# format of a tag file without this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths +# or URLs. Note that each tag file must have a unique name (where the name does +# NOT include the path). If a tag file is not located in the directory in which +# doxygen is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option also works with HAVE_DOT disabled, but it is recommended to +# install and use dot, since it yields more powerful graphs. + +CLASS_DIAGRAMS = NO + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see +# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = YES + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is +# allowed to run in parallel. When set to 0 (the default) doxygen will +# base this on the number of processors available in the system. You can set it +# explicitly to a value larger than 0 to get control over the balance +# between CPU load and processing speed. + +DOT_NUM_THREADS = 0 + +# By default doxygen will use the Helvetica font for all dot files that +# doxygen generates. When you want a differently looking font you can specify +# the font name using DOT_FONTNAME. You need to make sure dot is able to find +# the font, which can be done by putting it in a standard location or by setting +# the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the +# directory containing the font. + +DOT_FONTNAME = Helvetica + +# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. +# The default size is 10pt. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the Helvetica font. +# If you specify a different font using DOT_FONTNAME you can use DOT_FONTPATH to +# set the path where dot can find it. + +DOT_FONTPATH = + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If the UML_LOOK tag is enabled, the fields and methods are shown inside +# the class node. If there are many fields or methods and many nodes the +# graph may become too big to be useful. The UML_LIMIT_NUM_FIELDS +# threshold limits the number of items for each type to make the size more +# managable. Set this to 0 for no limit. Note that the threshold may be +# exceeded by 50% before the limit is enforced. + +UML_LIMIT_NUM_FIELDS = 10 + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT options are set to YES then +# doxygen will generate a call dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable call graphs +# for selected functions only using the \callgraph command. + +CALL_GRAPH = YES + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then +# doxygen will generate a caller dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable caller +# graphs for selected functions only using the \callergraph command. + +CALLER_GRAPH = YES + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will generate a graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are svg, png, jpg, or gif. +# If left blank png will be used. If you choose svg you need to set +# HTML_FILE_EXTENSION to xhtml in order to make the SVG files +# visible in IE 9+ (other browsers do not have this requirement). + +DOT_IMAGE_FORMAT = png + +# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to +# enable generation of interactive SVG images that allow zooming and panning. +# Note that this requires a modern browser other than Internet Explorer. +# Tested and working are Firefox, Chrome, Safari, and Opera. For IE 9+ you +# need to set HTML_FILE_EXTENSION to xhtml in order to make the SVG files +# visible. Older versions of IE do not have SVG support. + +INTERACTIVE_SVG = NO + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the +# \mscfile command). + +MSCFILE_DIRS = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the +# number of direct children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not +# seem to support this out of the box. Warning: Depending on the platform used, +# enabling this option may lead to badly anti-aliased labels on the edges of +# a graph (i.e. they become hard to read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES diff --git a/cmake/Doxygen.extra.css.in b/cmake/Doxygen.extra.css.in new file mode 100644 index 00000000..022974fa --- /dev/null +++ b/cmake/Doxygen.extra.css.in @@ -0,0 +1,14 @@ +/* increase vertical space */ +#titlearea, #nav-path { + display: none; + height: 0px; +} + + +/* uncomment these lines for some extra vertical space */ + +/* +.tablist li { + line-height: 26px; +} +*/ From a3b2226394c9bce5ae7cea99439a79ab6f54bcda Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Wed, 1 Apr 2015 19:23:24 +0200 Subject: [PATCH 12/16] my changelog --- src/Changelog | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/Changelog diff --git a/src/Changelog b/src/Changelog new file mode 100644 index 00000000..807bbceb --- /dev/null +++ b/src/Changelog @@ -0,0 +1,71 @@ + +[F] - fixes a bug from existing official version +[B] - important bug discovered in official version +[T] - testes +[n] - not used now (only in history), replaced or not accepted to master + +- adding support for user-local installed compiler (e.g. non-root users on older systems to use new compiler) +instructions to use monero on Debian 7 (build clang + deps) +- [n] faster build: cmake with fast build, and with cotire (precompiled headers), and limited targets +- fixed cmake local compiler e.g. BOOST_IGNORE_SYSTEM_PATHS after merges with the other new CMake + +- faster rebuild: separate out not-template base for main network classes: connection_basic.cpp class connection_basic +- faster rebuild: separate out hooks: connection_basic::do_send_handler_write() do_send_handler_write_from_queue() +- created library/cmake for the new no-templates-only networking code - libp2p + +- added command exit without DB save (only for testers) +- added option --no-igd to not wait for IGD on start (faster testing) + +- imported logging/debug tools from otshell, with macros like _note() _info() _warn() +- logging: added proper thread locking; showing thread number {1} in the output +- logging: fixed colors for normal windows text console (currently they do not work in msys windows text console) +- logging: added channels to have separate files created + +- created network throttle class + throttle manager +- cmdline options for network throttle class (limit-rate-up limit-rate-down limit-rate) +- option and command to in fact limit (outgoing) peers --out-peers out_peers +- rpc commands for network throttle class (limit limit_up limit_down) +- setting ToS socket flag with option --tos-flag + +- connection type support, so that RPC connection is excluded from network limits + +- gather and save throughput statistics (speed vs limit) into data file +- statistics of details to tune implementation: sleep in network threads, number of peers +- optimized statistics: accumulate sum and limit disk writes + +- dr. monero show the collected statistics (in real time) +- dr. monero many windows opened at once from predefined list +- dr. monero auto scale; selectable average window +- dr. monero colors, show both samples and average values (transparent) +- dr. monero showing current configured network limit (goal) +- dr. monero show date/git commit/comments for better info in screenshots +- dr. monero optimize speed and memory usage to handle files from long tests + +- [F] found few UBs in code like wrong initialization order +- [B] found and partial debugged deadlocks on existing program (faster testing) + +- [B] found locking problem with connection close race + +- debug option --test-drop-download to test no-DB version without running out of RAM +- also option --test-drop-download-heigh to start drop only after certain height + +- created new from-scratch clean network engine model (a new program/library) +- connected the logging, the network throttles and all code to network model +- [T] run experiments in fully controlled and very fast environment + +- fine tuned parameters of throttles (sleep, 3 window sizes) +- code to estimated current network-transfer size of block based get_avg_block_size() +- [T] tested sleep vs request-size limiting of downloads (request less or more then 200 blocks) +- for now we use sleep method + +- documented throttle classes and few other classes +- adjusted comment headers, copyrights, added the @monero tag to doxygen denoting monero maintainer/contact for not-monero code +- doxygen scripts fixed, tested and generated actual website + +- [n] rebased most of above when change to branch development was decided +- [n] again rebase when we returned to developing against branch master +- finall version was tested by developer on Windows 64-bit, Debian, Ubuntu +- fixed clock compilation problems for freebsd and mac os x; Fixed related problem again on mac os x. + + + From 162c993262a656872479d20f1e60da2aa2685e0e Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Wed, 1 Apr 2015 19:34:59 +0200 Subject: [PATCH 13/16] Network 1.6: network limits, logging, +doxy -dr.monero and once more again merged all work to current official monero version --- src/Changelog | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Changelog b/src/Changelog index 807bbceb..84da8068 100644 --- a/src/Changelog +++ b/src/Changelog @@ -18,12 +18,15 @@ instructions to use monero on Debian 7 (build clang + deps) - imported logging/debug tools from otshell, with macros like _note() _info() _warn() - logging: added proper thread locking; showing thread number {1} in the output +- logging: also showing process number (PID) in short form (1,2,3,...) to debug forking daemon etc - logging: fixed colors for normal windows text console (currently they do not work in msys windows text console) - logging: added channels to have separate files created +- logging: option (compile-time) to debug the logging system itself +- logging: console colors work on windows/msys - created network throttle class + throttle manager - cmdline options for network throttle class (limit-rate-up limit-rate-down limit-rate) -- option and command to in fact limit (outgoing) peers --out-peers out_peers +- option and command to in fact limit (outgoing) peers --out-peers out_peers [currently not working! after merge] [TODO] - rpc commands for network throttle class (limit limit_up limit_down) - setting ToS socket flag with option --tos-flag @@ -33,6 +36,7 @@ instructions to use monero on Debian 7 (build clang + deps) - statistics of details to tune implementation: sleep in network threads, number of peers - optimized statistics: accumulate sum and limit disk writes +(dr. monero code is not published in this commit, but prepared) - dr. monero show the collected statistics (in real time) - dr. monero many windows opened at once from predefined list - dr. monero auto scale; selectable average window @@ -49,6 +53,8 @@ instructions to use monero on Debian 7 (build clang + deps) - debug option --test-drop-download to test no-DB version without running out of RAM - also option --test-drop-download-heigh to start drop only after certain height +- added again fast exit command (but it does not work too well yet. ONLY FOR DEVELOPERS!) + - created new from-scratch clean network engine model (a new program/library) - connected the logging, the network throttles and all code to network model - [T] run experiments in fully controlled and very fast environment From 44f423477a6ce0971ced31afee38e259b2b571d6 Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Wed, 1 Apr 2015 20:25:53 +0200 Subject: [PATCH 14/16] [fix] mac os x includes std::random... --- contrib/epee/include/net/levin_protocol_handler_async.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index dacbfd4d..a7fbffb4 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -34,6 +34,9 @@ #include "levin_base.h" #include "misc_language.h" +#include +#include + namespace epee { @@ -684,6 +687,7 @@ void async_protocol_handler_config::del_out_connections(si return; // close random out connections + // TODO or better just keep removing random elements (performance) unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); shuffle(out_connections.begin(), out_connections.end(), std::default_random_engine(seed)); while (count > 0 && out_connections.size() > 0) From 391c7f9612a9862f699165ab26390d5253807834 Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Thu, 2 Apr 2015 13:58:38 +0200 Subject: [PATCH 15/16] Utils: use const, document dbg. Less default debug --- contrib/otshell_utils/utils.cpp | 17 ++++++++- contrib/otshell_utils/utils.hpp | 64 +++++++++++++++++++-------------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/contrib/otshell_utils/utils.cpp b/contrib/otshell_utils/utils.cpp index 3ca84da8..dbdb8bdc 100644 --- a/contrib/otshell_utils/utils.cpp +++ b/contrib/otshell_utils/utils.cpp @@ -46,6 +46,21 @@ namespace nUtils { INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces +// ==================================================================== + +// Numerical values of the debug levels - see hpp +const int _debug_level_nr_dbg3=20; +const int _debug_level_nr_dbg2=30; +const int _debug_level_nr_dbg1=40; +const int _debug_level_nr_info=50; +const int _debug_level_nr_note=60; +const int _debug_level_nr_fact=75; +const int _debug_level_nr_mark=80; +const int _debug_level_nr_warn=90; +const int _debug_level_nr_erro=100; + +// ==================================================================== + myexception::myexception(const char * what) : std::runtime_error(what) { } @@ -277,7 +292,7 @@ cLogger::cLogger() : mStream(NULL), mStreamBrokenDebug(NULL), mIsBroken(true), // before constructor finishes -mLevel(70), +mLevel(_debug_level_nr_warn), mThread2Number_Biggest(0), // the CURRENT biggest value (no thread yet in map) mPid2Number_Biggest(0) { diff --git a/contrib/otshell_utils/utils.hpp b/contrib/otshell_utils/utils.hpp index 6fb0fc5f..297f13ac 100644 --- a/contrib/otshell_utils/utils.hpp +++ b/contrib/otshell_utils/utils.hpp @@ -132,26 +132,36 @@ std::atomic & gLoggerGuardDepth_Get(); // getter for the global singleton o } catch(...) { if (part<8) gCurrentLogger.write_stream(100,CHANNEL)<<"DEBUG-ERROR: problem in debug mechanism e.g. in locking." < & gLoggerGuardDepth_Get(); // getter for the global singleton o nOT::nUtils::gLoggerGuard.unlock(); #define _scope_debug_level(LEVEL,VAR) _scope_debug_level_c("",LEVEL,VAR) -#define _scope_dbg1(VAR) _scope_debug_level( 20,VAR) -#define _scope_dbg2(VAR) _scope_debug_level( 30,VAR) -#define _scope_dbg3(VAR) _scope_debug_level( 40,VAR) // details -#define _scope_info(VAR) _scope_debug_level( 50,VAR) // more boring info -#define _scope_note(VAR) _scope_debug_level( 70,VAR) // info -#define _scope_fact(VAR) _scope_debug_level( 75,VAR) // interesting event -#define _scope_mark(VAR) _scope_debug_level( 80,VAR) // marked action -#define _scope_warn(VAR) _scope_debug_level( 90,VAR) // some problem -#define _scope_erro(VAR) _scope_debug_level( 100,VAR) // error - report +#define _scope_dbg1(VAR) _scope_debug_level( _debug_level_nr_dbg3, VAR) +#define _scope_dbg2(VAR) _scope_debug_level( _debug_level_nr_dbg2, VAR) +#define _scope_dbg3(VAR) _scope_debug_level( _debug_level_nr_dbg1, VAR) +#define _scope_info(VAR) _scope_debug_level( _debug_level_nr_info, VAR) +#define _scope_note(VAR) _scope_debug_level( _debug_level_nr_note, VAR) +#define _scope_fact(VAR) _scope_debug_level( _debug_level_nr_fact, VAR) +#define _scope_mark(VAR) _scope_debug_level( _debug_level_nr_mark, VAR) +#define _scope_warn(VAR) _scope_debug_level( _debug_level_nr_warn, VAR) +#define _scope_erro(VAR) _scope_debug_level( _debug_level_nr_erro, VAR) /*** @brief do not use this namespace directly, it is implementation detail. From 618f20ce494ada0ac6ab087b33fc45e4968b3ac8 Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Thu, 2 Apr 2015 16:27:19 +0200 Subject: [PATCH 16/16] Network 1.7; Quieted the debug a bit. Really really finall version of this changes I hope. --- contrib/otshell_utils/utils.cpp | 10 +++++----- src/daemon/main.cpp | 6 +++++- src/p2p/data_logger.cpp | 10 +++++----- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/contrib/otshell_utils/utils.cpp b/contrib/otshell_utils/utils.cpp index dbdb8bdc..e1d7d9a1 100644 --- a/contrib/otshell_utils/utils.cpp +++ b/contrib/otshell_utils/utils.cpp @@ -322,9 +322,9 @@ void cLogger::SetStreamBroken(const std::string &msg) { _dbg_dbg("Stream is broken (msg: " << msg << ")"); if (!mIsBroken) { // if not already marked as broken _dbg_dbg("(It was not broken before)"); - std::cerr << OT_CODE_STAMP << "WARNING: due to debug stream problem ("<(channel , thefile ) ); // <- created the channel mapping } @@ -521,7 +521,7 @@ int cLogger::Thread2Number(const std::thread::id id) { if (found == mThread2Number.end()) { // new one mThread2Number_Biggest++; mThread2Number[id] = mThread2Number_Biggest; - _mark_c("dbg/main", "This is a new thread (used in debug), thread id="< lock(mMutex); // lock @@ -55,19 +55,19 @@ namespace net_utils // do NOT modify mFilesMap below this point, since there is no locking for this used (yet) - _note_c("dbg/data","Creating thread for data logger"); // create timer thread + _info_c("dbg/data","Creating thread for data logger"); // create timer thread m_thread_maybe_running=true; std::shared_ptr logger_thread(new std::thread([&]() { - _note_c("dbg/data","Inside thread for data logger"); + _info_c("dbg/data","Inside thread for data logger"); while (m_state == data_logger_state::state_during_init) { // wait for creation to be done (in other thread, in singleton) before actually running std::this_thread::sleep_for(std::chrono::seconds(1)); } - _note_c("dbg/data","Inside thread for data logger - going into main loop"); + _info_c("dbg/data","Inside thread for data logger - going into main loop"); while (m_state == data_logger_state::state_ready_to_use) { // run as long as we are not closing the single object std::this_thread::sleep_for(std::chrono::seconds(1)); saveToFile(); // save all the pending data } - _note_c("dbg/data","Inside thread for data logger - done the main loop"); + _info_c("dbg/data","Inside thread for data logger - done the main loop"); m_thread_maybe_running=false; })); logger_thread->detach();