kumi/danicoin
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #252

Browse source 
618f20c Network 1.7; Quieted the debug a bit. (rfree2monero)
391c7f9 Utils: use const, document dbg. Less default debug (rfree2monero)
44f4234 [fix] mac os x includes std::random... (rfree2monero)
162c993 Network 1.6: network limits, logging, +doxy (rfree2monero)
a3b2226 my changelog (rfree2monero)
2900b1e doxygen files (rfree2monero)
1489310 doxygen related tool (rfree2monero)
f9dba47 added windows_stream.* console colors (rfree2monero)
c511abf remerged; commands JSON. logging upgrade. doxygen (rfree2monero)
f79821a fix locking in count-peers thread (2) (rfree2monero)
0198ffb 2014 network limit 1.3 fix log/path/data +utils (rfree2monero)
ae2a506 2014 network limit 1.2 +utils +toc -doc -drmonero (rfree2monero)
0f06dca fixed size_t on windows (rfree2monero)
39fc63f removed not needed <netinet/in.h> (rfree2monero)
5ce4256 2014 network limit 1.1 +utils +toc -doc -drmonero (rfree2monero)
eabb519 2014 network limit 1.0a +utils +toc -doc -drmonero (rfree2monero)
This commit is contained in:
Riccardo Spagni 2015-04-02 17:43:38 +02:00
commit 6f0d93097e
No known key found for this signature in database
GPG key ID: 55432DF31CCD4FCD
79 changed files with 7166 additions and 268 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
.gitignore vendored
View file

@ -2,6 +2,7 @@
/doc
/build
/tags
log/
# vim swap files
*.swp
@ -19,5 +20,85 @@ cscope.out
cscope.in.out
cscope.po.out
external/miniupnpc/Makefile
miniupnpcstrings.h
version/
# Created by https://www.gitignore.io
### C++ ###
# Compiled Object files
*.slo
*.lo
*.o
*.obj
# Precompiled Headers
*.gch
*.pch
# Compiled Dynamic libraries
*.so
*.dylib
*.dll
# Fortran module files
*.mod
# Compiled Static libraries
*.lai
*.la
*.a
*.lib
# Executables
*.exe
*.out
*.app
### CMake ###
CMakeCache.txt
CMakeFiles
cmake_install.cmake
install_manifest.txt
*.cmake
### Linux ###
*~
# KDE directory preferences
.directory
### Eclipse ###
*.pydevproject
.metadata
.gradle
bin/
tmp/
*.tmp
*.bak
*.swp
*~.nib
local.properties
.settings/
.loadpath
# External tool builders
.externalToolBuilders/
# Locally stored "Eclipse launch configurations"
*.launch
# CDT-specific
.cproject
# PDT-specific
.buildpath
# sbteclipse plugin
.target
# TeXlipse plugin
.texlipse

49
CMakeLists.txt
View file

@ -51,7 +51,7 @@ list(INSERT CMAKE_MODULE_PATH 0
if (NOT DEFINED ENV{DEVELOPER_LOCAL_TOOLS})
message(STATUS "Could not find DEVELOPER_LOCAL_TOOLS in env (not required)")
set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT OFF)
elseif (ENV{DEVELOPER_LOCAL_TOOLS} EQUAL 1)
elseif ("$ENV{DEVELOPER_LOCAL_TOOLS}" EQUAL 1)
message(STATUS "Found: env DEVELOPER_LOCAL_TOOLS = 1")
set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT ON)
else()
@ -62,9 +62,24 @@ endif()
message(STATUS "BOOST_IGNORE_SYSTEM_PATHS defaults to ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}")
option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost installation" ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT})
if (NOT DEFINED ENV{DEVELOPER_LIBUNBOUND_OLD})
message(STATUS "Could not find DEVELOPER_LIBUNBOUND_OLD in env (not required)")
elseif ("$ENV{DEVELOPER_LIBUNBOUND_OLD}" EQUAL 1)
message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD = 1, will use the work around")
add_definitions(-DDEVELOPER_LIBUNBOUND_OLD)
elseif ("$ENV{DEVELOPER_LIBUNBOUND_OLD}" EQUAL 0)
message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD = 0")
else()
message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD with bad value. Will NOT use the work around")
endif()
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
enable_testing()
option(BUILD_DOCUMENTATION "Build the Doxygen documentation." ON)
# Check if we're on FreeBSD so we can exclude the local miniupnpc (it should be installed from ports instead)
# CMAKE_SYSTEM_NAME checks are commonly known, but specifically taken from libsdl's CMakeLists
if(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD.*")
@ -155,9 +170,9 @@ else()
else()
set(ARCH_FLAG "-march=${ARCH}")
endif()
set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-error=sign-compare -Wno-error=strict-aliasing -Wno-error=type-limits -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized")
set(WARNINGS "-Wall -pedantic -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-error=sign-compare -Wno-error=strict-aliasing -Wno-error=type-limits -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized")
if(NOT MINGW)
set(WARNINGS "${WARNINGS} -Werror")
# set(WARNINGS "${WARNINGS} -Werror") # to allow pedantic but not stop compilation
endif()
if(CMAKE_C_COMPILER_ID STREQUAL "Clang")
set(WARNINGS "${WARNINGS} -Wno-error=mismatched-tags -Wno-error=null-conversion -Wno-overloaded-shift-op-parentheses -Wno-error=shift-count-overflow -Wno-error=tautological-constant-out-of-range-compare -Wno-error=unused-private-field -Wno-error=unneeded-internal-declaration")
@ -234,7 +249,7 @@ else()
endif()
endif()
if (BOOST_IGNORE_SYSTEM_PATHS)
if (${BOOST_IGNORE_SYSTEM_PATHS} STREQUAL "ON")
set(Boost_NO_SYSTEM_PATHS TRUE)
endif()
@ -270,8 +285,34 @@ endif()
include(version.cmake)
add_subdirectory(contrib)
add_subdirectory(src)
if(BUILD_TESTS)
add_subdirectory(tests)
endif()
if(BUILD_DOCUMENTATION)
set(DOC_GRAPHS "YES" CACHE STRING "Create dependency graphs (needs graphviz)")
set(DOC_FULLGRAPHS "NO" CACHE STRING "Create call/callee graphs (large)")
find_program(DOT_PATH dot)
if (DOT_PATH STREQUAL "DOT_PATH-NOTFOUND")
message("Doxygen: graphviz not found - graphs disabled")
set(DOC_GRAPHS "NO")
endif()
find_package(Doxygen)
if(DOXYGEN_FOUND)
configure_file("cmake/Doxyfile.in" "Doxyfile" @ONLY)
configure_file("cmake/Doxygen.extra.css.in" "Doxygen.extra.css" @ONLY)
add_custom_target(doc
${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Generating API documentation with Doxygen.." VERBATIM)
endif()
endif()

1803
cmake/Doxyfile.in Normal file
View file

File diff suppressed because it is too large Load diff

14
cmake/Doxygen.extra.css.in Normal file
View file

@ -0,0 +1,14 @@
/* increase vertical space */
#titlearea, #nav-path {
display: none;
height: 0px;
}
/* uncomment these lines for some extra vertical space */
/*
.tablist li {
line-height: 26px;
}
*/

3
contrib/CMakeLists.txt Normal file
View file

@ -0,0 +1,3 @@
add_subdirectory(otshell_utils)

80
contrib/epee/include/net/abstract_tcp_server2.h
View file

@ -1,3 +1,9 @@
/**
@file
@author from CrypoNote (see copyright below; Andrey N. Sabelnikov)
@monero rfree
@brief the connection templated-class for one peer connection
*/
// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
// All rights reserved.
//
@ -36,6 +42,8 @@
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <map>
#include <memory>
#include <boost/asio.hpp>
#include <boost/array.hpp>
@ -46,7 +54,9 @@
#include <boost/thread/thread.hpp>
#include "net_utils_base.h"
#include "syncobj.h"
#include "../../../../src/p2p/connection_basic.hpp"
#include "../../../../contrib/otshell_utils/utils.hpp"
#include "../../../../src/p2p/network_throttle-detail.hpp"
#define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000
@ -62,6 +72,12 @@ namespace net_utils
virtual ~i_connection_filter(){}
};
enum t_server_role { // type of the server, e.g. so that we will know how to limit it
NET = 0, // default (not used? used for misc connections maybe?) TODO
RPC = 1, // the rpc commands
P2P = 2 // to other p2p node
};
/************************************************************************/
/* */
/************************************************************************/
@ -70,13 +86,18 @@ namespace net_utils
class connection
: public boost::enable_shared_from_this<connection<t_protocol_handler> >,
private boost::noncopyable,
public i_service_endpoint
public i_service_endpoint,
public connection_basic
{
public:
typedef typename t_protocol_handler::connection_context t_connection_context;
/// Construct a connection with the given io_service.
explicit connection( boost::asio::io_service& io_service,
typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter * &pfilter);
typename t_protocol_handler::config_type& config,
std::atomic<long> &ref_sock_count, // the ++/-- counter
std::atomic<long> &sock_number, // the only increasing ++ number generator
i_connection_filter * &pfilter);
virtual ~connection();
/// Get the socket associated with the connection.
@ -90,7 +111,8 @@ namespace net_utils
void call_back_starter();
private:
//----------------- i_service_endpoint ---------------------
virtual bool do_send(const void* ptr, size_t cb);
virtual bool do_send(const void* ptr, size_t cb); ///< (see do_send from i_service_endpoint)
virtual bool do_send_chunk(const void* ptr, size_t cb); ///< will send (or queue) a part of data
virtual bool close();
virtual bool call_run_once_service_io();
virtual bool request_callback();
@ -107,29 +129,31 @@ namespace net_utils
/// Handle completion of a write operation.
void handle_write(const boost::system::error_code& e, size_t cb);
/// Strand to ensure the connection's handlers are not called concurrently.
boost::asio::io_service::strand strand_;
/// Socket for the connection.
boost::asio::ip::tcp::socket socket_;
/// Buffer for incoming data.
boost::array<char, 8192> buffer_;
//boost::array<char, 1024> buffer_;
t_connection_context context;
volatile uint32_t m_want_close_connection;
std::atomic<bool> m_was_shutdown;
critical_section m_send_que_lock;
std::list<std::string> m_send_que;
volatile uint32_t& m_ref_sockets_count;
i_connection_filter* &m_pfilter;
volatile bool m_is_multithreaded;
// TODO what do they mean about wait on destructor?? --rfree :
//this should be the last one, because it could be wait on destructor, while other activities possible on other threads
t_protocol_handler m_protocol_handler;
//typename t_protocol_handler::config_type m_dummy_config;
std::list<boost::shared_ptr<connection<t_protocol_handler> > > m_self_refs; // add_ref/release support
critical_section m_self_refs_lock;
critical_section m_chunking_lock; // held while we add small chunks of the big do_send() to small do_send_chunk()
t_server_role m_connection_type;
// for calculate speed (last 60 sec)
network_throttle m_throttle_speed_in;
network_throttle m_throttle_speed_out;
std::mutex m_throttle_speed_in_mutex;
std::mutex m_throttle_speed_out_mutex;
public:
void setRPcStation();
};
@ -146,9 +170,11 @@ namespace net_utils
/// Construct the server to listen on the specified TCP address and port, and
/// serve up files from the given directory.
boosted_tcp_server();
explicit boosted_tcp_server(boost::asio::io_service& external_io_service);
explicit boosted_tcp_server(boost::asio::io_service& external_io_service, t_server_role s_type);
~boosted_tcp_server();
std::map<std::string, t_server_role> server_type_map;
void create_server_type_map();
bool init_server(uint32_t port, const std::string address = "0.0.0.0");
bool init_server(const std::string port, const std::string& address = "0.0.0.0");
@ -254,22 +280,26 @@ namespace net_utils
/// Acceptor used to listen for incoming connections.
boost::asio::ip::tcp::acceptor acceptor_;
/// The next connection to be accepted.
connection_ptr new_connection_;
std::atomic<bool> m_stop_signal_sent;
uint32_t m_port;
volatile uint32_t m_sockets_count;
std::atomic<long> m_sock_count;
std::atomic<long> m_sock_number;
std::string m_address;
std::string m_thread_name_prefix;
std::string m_thread_name_prefix; //TODO: change to enum server_type, now used
size_t m_threads_count;
i_connection_filter* m_pfilter;
std::vector<boost::shared_ptr<boost::thread> > m_threads;
boost::thread::id m_main_thread_id;
critical_section m_threads_lock;
volatile uint32_t m_thread_index;
};
}
}
volatile uint32_t m_thread_index; // TODO change to std::atomic
t_server_role type;
void detach_threads();
/// The next connection to be accepted
connection_ptr new_connection_;
}; // class <>boosted_tcp_server
} // namespace
} // namespace
#include "abstract_tcp_server2.inl"

394
contrib/epee/include/net/abstract_tcp_server2.inl
View file

@ -1,3 +1,9 @@
/**
@file
@author from CrypoNote (see copyright below; Andrey N. Sabelnikov)
@monero rfree
@brief the connection templated-class for one peer connection
*/
// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
// All rights reserved.
//
@ -26,7 +32,7 @@
#include "net_utils_base.h"
//#include "net_utils_base.h"
#include <boost/lambda/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/lambda/lambda.hpp>
@ -34,9 +40,21 @@
#include <boost/chrono.hpp>
#include <boost/utility/value_init.hpp>
#include <boost/asio/deadline_timer.hpp>
#include <boost/date_time/posix_time/posix_time.hpp> // TODO
#include <boost/thread/thread.hpp> // TODO
#include "misc_language.h"
#include "pragma_comp_defs.h"
#include <sstream>
#include <iomanip>
#include <algorithm>
#include "../../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal()
#include "../../../../contrib/otshell_utils/utils.hpp"
#include "../../../../src/p2p/data_logger.hpp"
using namespace nOT::nUtils; // TODO
PRAGMA_WARNING_PUSH
namespace epee
{
@ -49,16 +67,20 @@ PRAGMA_WARNING_DISABLE_VS(4355)
template<class t_protocol_handler>
connection<t_protocol_handler>::connection( boost::asio::io_service& io_service,
typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter* &pfilter)
: strand_(io_service),
socket_(io_service),
m_want_close_connection(0),
m_was_shutdown(0),
m_ref_sockets_count(sock_count),
typename t_protocol_handler::config_type& config,
std::atomic<long> &ref_sock_count, // the ++/-- counter
std::atomic<long> &sock_number, // the only increasing ++ number generator
i_connection_filter* &pfilter
)
:
connection_basic(io_service, ref_sock_count, sock_number),
m_protocol_handler(this, config, context),
m_pfilter( pfilter ),
m_protocol_handler(this, config, context)
m_connection_type(NET),
m_throttle_speed_in("speed_in", "throttle_speed_in"),
m_throttle_speed_out("speed_out", "throttle_speed_out")
{
boost::interprocess::ipcdetail::atomic_inc32(&m_ref_sockets_count);
_info_c("net/sleepRPC", "connection constructor set m_connection_type="<<m_connection_type);
}
PRAGMA_WARNING_DISABLE_VS(4355)
//---------------------------------------------------------------------------------
@ -67,12 +89,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
{
if(!m_was_shutdown)
{
LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown.");
_dbg3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown.");
shutdown();
}
LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed");
boost::interprocess::ipcdetail::atomic_dec32(&m_ref_sockets_count);
_dbg3("[sock " << socket_.native_handle() << "] Socket destroyed");
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
@ -118,13 +139,13 @@ PRAGMA_WARNING_DISABLE_VS(4355)
long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong());
context.set_details(boost::uuids::random_generator()(), ip_, remote_ep.port(), is_income);
LOG_PRINT_L3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) <<
_dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) <<
" to " << local_ep.address().to_string() << ':' << local_ep.port() <<
", total sockets objects " << m_ref_sockets_count);
", total sockets objects " << m_ref_sock_count);
if(m_pfilter && !m_pfilter->is_remote_ip_allowed(context.m_remote_ip))
{
LOG_PRINT_L2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection");
_dbg2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection");
close();
return false;
}
@ -137,6 +158,16 @@ PRAGMA_WARNING_DISABLE_VS(4355)
boost::asio::placeholders::error,
boost::asio::placeholders::bytes_transferred)));
//set ToS flag
int tos = get_tos_flag();
boost::asio::detail::socket_option::integer< IPPROTO_IP, IP_TOS >
optionTos( tos );
socket_.set_option( optionTos );
//_dbg1("Set ToS flag to " << tos);
boost::asio::ip::tcp::no_delay noDelayOption(false);
socket_.set_option(noDelayOption);
return true;
CATCH_ENTRY_L0("connection<t_protocol_handler>::start()", false);
@ -146,7 +177,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
bool connection<t_protocol_handler>::request_callback()
{
TRY_ENTRY();
LOG_PRINT_L2("[" << print_connection_context_short(context) << "] request_callback");
_dbg2("[" << print_connection_context_short(context) << "] request_callback");
// Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
auto self = safe_shared_from_this();
if(!self)
@ -167,8 +198,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
bool connection<t_protocol_handler>::add_ref()
{
TRY_ENTRY();
LOG_PRINT_L4("[sock " << socket_.native_handle() << "] add_ref");
//_dbg3("[sock " << socket_.native_handle() << "] add_ref, m_peer_number=" << mI->m_peer_number);
CRITICAL_REGION_LOCAL(m_self_refs_lock);
//_dbg3("[sock " << socket_.native_handle() << "] add_ref 2, m_peer_number=" << mI->m_peer_number);
// Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
auto self = safe_shared_from_this();
@ -201,7 +233,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
void connection<t_protocol_handler>::call_back_starter()
{
TRY_ENTRY();
LOG_PRINT_L2("[" << print_connection_context_short(context) << "] fired_callback");
_dbg2("[" << print_connection_context_short(context) << "] fired_callback");
m_protocol_handler.handle_qued_callback();
CATCH_ENTRY_L0("connection<t_protocol_handler>::call_back_starter()", void());
}
@ -211,17 +243,44 @@ PRAGMA_WARNING_DISABLE_VS(4355)
std::size_t bytes_transferred)
{
TRY_ENTRY();
LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async read calledback.");
//_info("[sock " << socket_.native_handle() << "] Async read calledback.");
if (!e)
{
LOG_PRINT("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred, LOG_LEVEL_4);
{
CRITICAL_REGION_LOCAL(m_throttle_speed_in_mutex);
m_throttle_speed_in.handle_trafic_exact(bytes_transferred);
context.m_current_speed_down = m_throttle_speed_in.get_current_speed();
}
{
CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_in );
epee::net_utils::network_throttle_manager::network_throttle_manager::get_global_throttle_in().handle_trafic_exact(bytes_transferred * 1024);
}
double delay=0; // will be calculated
do
{
{ //_scope_dbg1("CRITICAL_REGION_LOCAL");
CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in );
delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred ); // decission from global
}
delay *= 0.5;
if (delay > 0) {
long int ms = (long int)(delay * 100);
epee::net_utils::data_logger::get_instance().add_data("sleep_down", ms);
std::this_thread::sleep_for(std::chrono::milliseconds(ms));
}
} while(delay > 0);
//_info("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred);
logger_handle_net_read(bytes_transferred);
context.m_last_recv = time(NULL);
context.m_recv_cnt += bytes_transferred;
bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred);
if(!recv_res)
{
LOG_PRINT("[sock " << socket_.native_handle() << "] protocol_want_close", LOG_LEVEL_4);
//_info("[sock " << socket_.native_handle() << "] protocol_want_close");
//some error in protocol, protocol handler ask to close connection
boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1);
@ -239,14 +298,14 @@ PRAGMA_WARNING_DISABLE_VS(4355)
boost::bind(&connection<t_protocol_handler>::handle_read, connection<t_protocol_handler>::shared_from_this(),
boost::asio::placeholders::error,
boost::asio::placeholders::bytes_transferred)));
LOG_PRINT_L4("[sock " << socket_.native_handle() << "]Async read requested.");
//_info("[sock " << socket_.native_handle() << "]Async read requested.");
}
}else
{
LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value());
_dbg3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value());
if(e.value() != 2)
{
LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value());
_dbg3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value());
shutdown();
}
}
@ -284,7 +343,90 @@ PRAGMA_WARNING_DISABLE_VS(4355)
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb)
bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb) {
TRY_ENTRY();
// Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
auto self = safe_shared_from_this();
if (!self) return false;
if (m_was_shutdown) return false;
// TODO avoid copy
const double factor = 32; // TODO config
typedef long long signed int t_safe; // my t_size to avoid any overunderflow in arithmetic
const t_safe chunksize_good = (t_safe)( 1024 * std::max(1.0,factor) );
const t_safe chunksize_max = chunksize_good * 2 ;
const bool allow_split = (m_connection_type == RPC) ? false : true; // TODO config
ASRT(! (chunksize_max<0) ); // make sure it is unsigned before removin sign with cast:
long long unsigned int chunksize_max_unsigned = static_cast<long long unsigned int>( chunksize_max ) ;
if (allow_split && (cb > chunksize_max_unsigned)) {
{ // LOCK: chunking
epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical ***
_dbg3_c("net/out/size", "do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr);
t_safe all = cb; // all bytes to send
t_safe pos = 0; // current sending position
// 01234567890
// ^^^^ (pos=0, len=4) ; pos:=pos+len, pos=4
// ^^^^ (pos=4, len=4) ; pos:=pos+len, pos=8
// ^^^ (pos=8, len=4) ;
// const size_t bufsize = chunksize_good; // TODO safecast
// char* buf = new char[ bufsize ];
bool all_ok = true;
while (pos < all) {
t_safe lenall = all-pos; // length from here to end
t_safe len = std::min( chunksize_good , lenall); // take a smaller part
ASRT(len<=chunksize_good);
// pos=8; len=4; all=10; len=3;
ASRT(! (len<0) ); // check before we cast away sign:
unsigned long long int len_unsigned = static_cast<long long int>( len );
ASRT(len>0); // (redundand)
ASRT(len_unsigned < std::numeric_limits<size_t>::max()); // yeap we want strong < then max size, to be sure
void *chunk_start = ((char*)ptr) + pos;
_fact_c("net/out/size","chunk_start="<<chunk_start<<" ptr="<<ptr<<" pos="<<pos);
ASRT(chunk_start >= ptr); // not wrapped around address?
//std::memcpy( (void*)buf, chunk_start, len);
_dbg3_c("net/out/size", "part of " << lenall << ": pos="<<pos << " len="<<len);
bool ok = do_send_chunk(chunk_start, len); // <====== ***
all_ok = all_ok && ok;
if (!all_ok) {
_dbg1_c("net/out/size", "do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr);
_dbg1("do_send() SEND was aborted in middle of big package - this is mostly harmless "
<< " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << cb);
return false; // partial failure in sending
}
pos = pos+len; ASRT(pos >0);
// (in catch block, or uniq pointer) delete buf;
} // each chunk
_dbg3_c("net/out/size", "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr);
_dbg3 ( "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr);
_info_c("net/sleepRPC", "do_send() m_connection_type = " << m_connection_type);
return all_ok; // done - e.g. queued - all the chunks of current do_send call
} // LOCK: chunking
} // a big block (to be chunked) - all chunks
else { // small block
return do_send_chunk(ptr,cb); // just send as 1 big chunk
}
CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false);
} // do_send()
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
bool connection<t_protocol_handler>::do_send_chunk(const void* ptr, size_t cb)
{
TRY_ENTRY();
// Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
@ -293,50 +435,86 @@ PRAGMA_WARNING_DISABLE_VS(4355)
return false;
if(m_was_shutdown)
return false;
{
CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex);
m_throttle_speed_out.handle_trafic_exact(cb);
context.m_current_speed_up = m_throttle_speed_out.get_current_speed();
}
LOG_PRINT("[sock " << socket_.native_handle() << "] SEND " << cb, LOG_LEVEL_4);
//_info("[sock " << socket_.native_handle() << "] SEND " << cb);
context.m_last_send = time(NULL);
context.m_send_cnt += cb;
//some data should be wrote to stream
//request complete
epee::critical_region_t<decltype(m_send_que_lock)> send_guard(m_send_que_lock);
if(m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT)
sleep_before_packet(cb, 1, 1);
epee::critical_region_t<decltype(m_send_que_lock)> send_guard(m_send_que_lock); // *** critical ***
long int retry=0;
const long int retry_limit = 5*4;
while (m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT)
{
retry++;
/* if ( ::cryptonote::core::get_is_stopping() ) { // TODO re-add fast stop
_fact("ABORT queue wait due to stopping");
return false; // aborted
}*/
long int ms = 250 + (rand()%50);
_info_c("net/sleep", "Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<cb); // XXX debug sleep
boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) );
_dbg1("sleep for queue: " << ms);
if (retry > retry_limit) {
send_guard.unlock();
LOG_PRINT_L2("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
_erro("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
// _dbg1_c("net/sleep", "send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
close();
return false;
}
}
m_send_que.resize(m_send_que.size()+1);
m_send_que.back().assign((const char*)ptr, cb);
if(m_send_que.size() > 1)
{
//active operation should be in progress, nothing to do, just wait last operation callback
}else
{
//no active operation
{ // active operation should be in progress, nothing to do, just wait last operation callback
auto size_now = cb;
_info_c("net/out/size", "do_send() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size());
//do_send_handler_delayed( ptr , size_now ); // (((H))) // empty function
LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size());
}
else
{ // no active operation
if(m_send_que.size()!=1)
{
LOG_ERROR("Looks like no active operations, but send que size != 1!!");
_erro("Looks like no active operations, but send que size != 1!!");
return false;
}
boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()),
auto size_now = m_send_que.front().size();
_dbg1_c("net/out/size", "do_send() NOW SENSD: packet="<<size_now<<" B");
do_send_handler_write( ptr , size_now ); // (((H)))
ASRT( size_now == m_send_que.front().size() );
boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now ) ,
//strand_.wrap(
boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2)
//)
);
LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size());
//_dbg3("(chunk): " << size_now);
//logger_handle_net_write(size_now);
//_info("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size());
}
//do_send_handler_stop( ptr , cb ); // empty function
return true;
CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false);
}
} // do_send_chunk
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
bool connection<t_protocol_handler>::shutdown()
@ -353,7 +531,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
bool connection<t_protocol_handler>::close()
{
TRY_ENTRY();
LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Que Shutdown called.");
//_info("[sock " << socket_.native_handle() << "] Que Shutdown called.");
size_t send_que_size = 0;
CRITICAL_REGION_BEGIN(m_send_que_lock);
send_que_size = m_send_que.size();
@ -376,16 +554,17 @@ PRAGMA_WARNING_DISABLE_VS(4355)
if (e)
{
LOG_PRINT_L1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value());
_dbg1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value());
shutdown();
return;
}
logger_handle_net_write(cb);
sleep_before_packet(cb, 1, 1);
bool do_shutdown = false;
CRITICAL_REGION_BEGIN(m_send_que_lock);
if(m_send_que.empty())
{
LOG_ERROR("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!");
_erro("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!");
return;
}
@ -399,10 +578,16 @@ PRAGMA_WARNING_DISABLE_VS(4355)
}else
{
//have more data to send
boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()),
auto size_now = m_send_que.front().size();
_dbg1_c("net/out/size", "handle_write() NOW SENDS: packet="<<size_now<<" B" <<", from queue size="<<m_send_que.size());
do_send_handler_write_from_queue(e, m_send_que.front().size() , m_send_que.size()); // (((H)))
ASRT( size_now == m_send_que.front().size() );
boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now) ,
// strand_.wrap(
boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2));
//);
boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2)
// )
);
//_dbg3("(normal)" << size_now);
}
CRITICAL_REGION_END();
@ -412,6 +597,13 @@ PRAGMA_WARNING_DISABLE_VS(4355)
}
CATCH_ENTRY_L0("connection<t_protocol_handler>::handle_write", void());
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
void connection<t_protocol_handler>::setRPcStation()
{
m_connection_type = RPC;
_fact_c("net/sleepRPC", "set m_connection_type = RPC ");
}
/************************************************************************/
/* */
/************************************************************************/
@ -420,19 +612,27 @@ PRAGMA_WARNING_DISABLE_VS(4355)
m_io_service_local_instance(new boost::asio::io_service()),
io_service_(*m_io_service_local_instance.get()),
acceptor_(io_service_),
new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)),
m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0)
m_stop_signal_sent(false), m_port(0),
m_sock_count(0), m_sock_number(0), m_threads_count(0),
m_pfilter(NULL), m_thread_index(0),
new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter))
{
create_server_type_map();
m_thread_name_prefix = "NET";
type = NET;
}
template<class t_protocol_handler>
boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service):
boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service, t_server_role s_type):
io_service_(extarnal_io_service),
acceptor_(io_service_),
new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)),
m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0)
m_stop_signal_sent(false), m_port(0),
m_sock_count(0), m_sock_number(0), m_threads_count(0),
m_pfilter(NULL), m_thread_index(0),
type(NET),
new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter))
{
create_server_type_map();
m_thread_name_prefix = "NET";
}
//---------------------------------------------------------------------------------
@ -444,6 +644,14 @@ PRAGMA_WARNING_DISABLE_VS(4355)
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
void boosted_tcp_server<t_protocol_handler>::create_server_type_map()
{
server_type_map["NET"] = t_server_role::NET;
server_type_map["RPC"] = t_server_role::RPC;
server_type_map["P2P"] = t_server_role::P2P;
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
bool boosted_tcp_server<t_protocol_handler>::init_server(uint32_t port, const std::string address)
{
TRY_ENTRY();
@ -491,6 +699,7 @@ POP_WARNINGS
std::string thread_name = std::string("[") + m_thread_name_prefix;
thread_name += boost::to_string(local_thr_index) + "]";
log_space::log_singletone::set_thread_log_prefix(thread_name);
// _fact("Thread name: " << m_thread_name_prefix);
while(!m_stop_signal_sent)
{
try
@ -499,14 +708,14 @@ POP_WARNINGS
}
catch(const std::exception& ex)
{
LOG_ERROR("Exception at server worker thread, what=" << ex.what());
_erro("Exception at server worker thread, what=" << ex.what());
}
catch(...)
{
LOG_ERROR("Exception at server worker thread, unknown execption");
_erro("Exception at server worker thread, unknown execption");
}
}
LOG_PRINT_L4("Worker thread finished");
//_info("Worker thread finished");
return true;
CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::worker_thread", false);
}
@ -515,6 +724,9 @@ POP_WARNINGS
void boosted_tcp_server<t_protocol_handler>::set_threads_prefix(const std::string& prefix_name)
{
m_thread_name_prefix = prefix_name;
type = server_type_map[m_thread_name_prefix];
_note("Set server type to: " << type);
_note("Set server type to: " << m_thread_name_prefix);
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
@ -539,32 +751,38 @@ POP_WARNINGS
{
boost::shared_ptr<boost::thread> thread(new boost::thread(
attrs, boost::bind(&boosted_tcp_server<t_protocol_handler>::worker_thread, this)));
_note("Run server thread name: " << m_thread_name_prefix);
m_threads.push_back(thread);
}
CRITICAL_REGION_END();
// Wait for all threads in the pool to exit.
if(wait)
if (wait) // && ! ::cryptonote::core::get_is_stopping()) // TODO fast_exit
{
for (std::size_t i = 0; i < m_threads.size(); ++i)
_fact("JOINING all threads");
for (std::size_t i = 0; i < m_threads.size(); ++i) {
m_threads[i]->join();
}
_fact("JOINING all threads - almost");
m_threads.clear();
_fact("JOINING all threads - DONE");
}else
{
}
else {
_dbg1("Reiniting OK.");
return true;
}
if(wait && !m_stop_signal_sent)
{
//some problems with the listening socket ?..
LOG_PRINT_L0("Net service stopped without stop request, restarting...");
_dbg1("Net service stopped without stop request, restarting...");
if(!this->init_server(m_port, m_address))
{
LOG_PRINT_L0("Reiniting service failed, exit.");
_dbg1("Reiniting service failed, exit.");
return false;
}else
{
LOG_PRINT_L0("Reiniting OK.");
_dbg1("Reiniting OK.");
}
}
}
@ -597,7 +815,7 @@ POP_WARNINGS
{
if(m_threads[i]->joinable() && !m_threads[i]->try_join_for(ms))
{
LOG_PRINT_L0("Interrupting thread " << m_threads[i]->native_handle());
_dbg1("Interrupting thread " << m_threads[i]->native_handle());
m_threads[i]->interrupt();
}
}
@ -608,6 +826,10 @@ POP_WARNINGS
template<class t_protocol_handler>
void boosted_tcp_server<t_protocol_handler>::send_stop_signal()
{
if (::cryptonote::core::get_fast_exit() == true)
{
detach_threads();
}
m_stop_signal_sent = true;
TRY_ENTRY();
io_service_.stop();
@ -626,19 +848,22 @@ POP_WARNINGS
TRY_ENTRY();
if (!e)
{
if (type == RPC) {
new_connection_->setRPcStation();
_note("New server for RPC connections");
}
connection_ptr conn(std::move(new_connection_));
new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter));
new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter));
acceptor_.async_accept(new_connection_->socket(),
boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this,
boost::asio::placeholders::error));
bool r = conn->start(true, 1 < m_threads_count);
if (!r)
LOG_ERROR("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count);
_erro("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count);
}else
{
LOG_ERROR("Some problems at accept: " << e.message() << ", connections_count = " << m_sockets_count);
_erro("Some problems at accept: " << e.message() << ", connections_count = " << m_sock_count);
}
CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::handle_accept", void());
}
@ -648,7 +873,7 @@ POP_WARNINGS
{
TRY_ENTRY();
connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter) );
connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter) );
boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket();
//////////////////////////////////////////////////////////////////////////
@ -658,7 +883,7 @@ POP_WARNINGS
boost::asio::ip::tcp::resolver::iterator end;
if(iterator == end)
{
LOG_ERROR("Failed to resolve " << adr);
_erro("Failed to resolve " << adr);
return false;
}
//////////////////////////////////////////////////////////////////////////
@ -704,7 +929,7 @@ POP_WARNINGS
{
//timeout
sock_.close();
LOG_PRINT_L3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")");
_dbg3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")");
return false;
}
}
@ -712,21 +937,21 @@ POP_WARNINGS
if (ec || !sock_.is_open())
{
LOG_PRINT("Some problems at connect, message: " << ec.message(), LOG_LEVEL_3);
_dbg3("Some problems at connect, message: " << ec.message());
return false;
}
LOG_PRINT_L3("Connected success to " << adr << ':' << port);
_dbg3("Connected success to " << adr << ':' << port);
bool r = new_connection_l->start(false, 1 < m_threads_count);
if (r)
{
new_connection_l->get_context(conn_context);
//new_connection_l.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter));
//new_connection_l.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_pfilter));
}
else
{
LOG_ERROR("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count);
_erro("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count);
}
return r;
@ -738,7 +963,7 @@ POP_WARNINGS
bool boosted_tcp_server<t_protocol_handler>::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, t_callback cb, const std::string& bind_ip)
{
TRY_ENTRY();
connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter) );
connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter) );
boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket();
//////////////////////////////////////////////////////////////////////////
@ -748,7 +973,7 @@ POP_WARNINGS
boost::asio::ip::tcp::resolver::iterator end;
if(iterator == end)
{
LOG_ERROR("Failed to resolve " << adr);
_erro("Failed to resolve " << adr);
return false;
}
//////////////////////////////////////////////////////////////////////////
@ -768,7 +993,7 @@ POP_WARNINGS
{
if(error != boost::asio::error::operation_aborted)
{
LOG_PRINT_L3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")");
_dbg3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")");
new_connection_l->socket().close();
}
});
@ -785,7 +1010,7 @@ POP_WARNINGS
cb(conn_context, boost::asio::error::operation_aborted);//this mean that deadline timer already queued callback with cancel operation, rare situation
}else
{
LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port <<
_dbg3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port <<
" from " << lep.address().to_string() << ':' << lep.port());
bool r = new_connection_l->start(false, 1 < m_threads_count);
if (r)
@ -795,13 +1020,13 @@ POP_WARNINGS
}
else
{
LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port);
_dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port);
cb(conn_context, boost::asio::error::fault);
}
}
}else
{
LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port <<
_dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port <<
" from " << lep.address().to_string() << ':' << lep.port() << ": " << ec_.message() << ':' << ec_.value());
cb(conn_context, ec_);
}
@ -809,6 +1034,15 @@ POP_WARNINGS
return true;
CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::connect_async", false);
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
void boosted_tcp_server<t_protocol_handler>::detach_threads()
{
for (auto thread : m_threads)
thread->detach();
}
}
} // namespace
} // namespace
PRAGMA_WARNING_POP

32
contrib/epee/include/net/levin_protocol_handler_async.h
View file

@ -34,6 +34,9 @@
#include "levin_base.h"
#include "misc_language.h"
#include <random>
#include <chrono>
namespace epee
{
@ -81,6 +84,7 @@ public:
async_protocol_handler_config():m_pcommands_handler(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE)
{}
void del_out_connections(size_t count);
};
@ -669,6 +673,34 @@ void async_protocol_handler_config<t_connection_context>::del_connection(async_p
}
//------------------------------------------------------------------------------------------
template<class t_connection_context>
void async_protocol_handler_config<t_connection_context>::del_out_connections(size_t count)
{
std::vector <boost::uuids::uuid> out_connections;
CRITICAL_REGION_BEGIN(m_connects_lock);
for (auto& c: m_connects)
{
if (!c.second->m_connection_context.m_is_income)
out_connections.push_back(c.first);
}
if (out_connections.size() == 0)
return;
// close random out connections
// TODO or better just keep removing random elements (performance)
unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();
shuffle(out_connections.begin(), out_connections.end(), std::default_random_engine(seed));
while (count > 0 && out_connections.size() > 0)
{
close(*out_connections.begin());
del_connection(m_connects.at(*out_connections.begin()));
--count;
}
CRITICAL_REGION_END();
}
//------------------------------------------------------------------------------------------
template<class t_connection_context>
void async_protocol_handler_config<t_connection_context>::add_connection(async_protocol_handler<t_connection_context>* pconn)
{
CRITICAL_REGION_BEGIN(m_connects_lock);

10
contrib/epee/include/net/net_utils_base.h
View file

@ -55,6 +55,8 @@ namespace net_utils
time_t m_last_send;
uint64_t m_recv_cnt;
uint64_t m_send_cnt;
double m_current_speed_down;
double m_current_speed_up;
connection_context_base(boost::uuids::uuid connection_id,
long remote_ip, int remote_port, bool is_income,
@ -68,7 +70,9 @@ namespace net_utils
m_last_recv(last_recv),
m_last_send(last_send),
m_recv_cnt(recv_cnt),
m_send_cnt(send_cnt)
m_send_cnt(send_cnt),
m_current_speed_down(0),
m_current_speed_up(0)
{}
connection_context_base(): m_connection_id(),
@ -79,7 +83,9 @@ namespace net_utils
m_last_recv(0),
m_last_send(0),
m_recv_cnt(0),
m_send_cnt(0)
m_send_cnt(0),
m_current_speed_down(0),
m_current_speed_up(0)
{}
connection_context_base& operator=(const connection_context_base& a)

4
contrib/epee/include/storages/levin_abstract_invoke2.h
View file

@ -185,7 +185,7 @@ namespace epee
}
return res;
};
}
template<class t_owner, class t_in_type, class t_context, class callback_t>
int buff_to_t_adapter(t_owner* powner, int command, const std::string& in_buff, callback_t cb, t_context& context)
@ -199,7 +199,7 @@ namespace epee
boost::value_initialized<t_in_type> in_struct;
static_cast<t_in_type&>(in_struct).load(strg);
return cb(command, in_struct, context);
};
}
#define CHAIN_LEVIN_INVOKE_MAP2(context_type) \
int invoke(int command, const std::string& in_buff, std::string& buff_out, context_type& context) \

12
contrib/epee/include/syncobj.h
View file

@ -35,10 +35,14 @@
#include <boost/thread/locks.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/recursive_mutex.hpp>
#include <thread>
#include <chrono>
namespace epee
{
extern unsigned int g_test_dbg_lock_sleep;
struct simple_event
{
simple_event() : m_rised(false)
@ -215,10 +219,10 @@ namespace epee
#define SHARED_CRITICAL_REGION_BEGIN(x) { shared_guard critical_region_var(x)
#define EXCLUSIVE_CRITICAL_REGION_BEGIN(x) { exclusive_guard critical_region_var(x)
#define CRITICAL_REGION_LOCAL(x) epee::critical_region_t<decltype(x)> critical_region_var(x)
#define CRITICAL_REGION_BEGIN(x) { epee::critical_region_t<decltype(x)> critical_region_var(x)
#define CRITICAL_REGION_LOCAL1(x) epee::critical_region_t<decltype(x)> critical_region_var1(x)
#define CRITICAL_REGION_BEGIN1(x) { epee::critical_region_t<decltype(x)> critical_region_var1(x)
#define CRITICAL_REGION_LOCAL(x) {std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep));} epee::critical_region_t<decltype(x)> critical_region_var(x)
#define CRITICAL_REGION_BEGIN(x) { std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep)); epee::critical_region_t<decltype(x)> critical_region_var(x)
#define CRITICAL_REGION_LOCAL1(x) {std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep));} epee::critical_region_t<decltype(x)> critical_region_var1(x)
#define CRITICAL_REGION_BEGIN1(x) { std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep)); epee::critical_region_t<decltype(x)> critical_region_var1(x)
#define CRITICAL_REGION_END() }

14
contrib/otshell_utils/CMakeLists.txt Normal file
View file

@ -0,0 +1,14 @@
cmake_minimum_required (VERSION 2.6)
project (otshell CXX)
# Add executable
file(GLOB otshell_utils_sources # All files in directory:
"*.h"
"*.hpp"
"*.cpp"
)
add_library (otshell_utils STATIC ${otshell_utils_sources})
set_target_properties (otshell_utils PROPERTIES OUTPUT_NAME "otshell_utils")
#target_link_libraries (upnpc-static ${LDLIBS}) # to add used libs

21
contrib/otshell_utils/LICENCE.txt Normal file
View file

@ -0,0 +1,21 @@
This are some files also from OpenTransactions / otshell project,
developed thanks to the awesome OpenTransaction project, organization and developers :)
Parts of code here was also developed thanks to the excellent Monero project,
thanks to Monero project, organization and developers :)
[Some] files/code here (in external/otshell_utils) are under licence defined in
src/doc/LICENCE-otshell.txt ;
Others are from monero, with licence in src/doc/LICENCE-monero.txt ;
For me (rfree) the licence seem compatbile so no problem, personally (as author of many parts of the code,
possibly not all) I do not worry who uses it how; I'am not a lawyer.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Please share :-) This licence can be used e.g. for parts of code that are usable in both open-source FOSS project
Monero and Open Transactions, to share and develop both faster.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

116
contrib/otshell_utils/ccolor.cpp Normal file
View file

@ -0,0 +1,116 @@
#include "ccolor.hpp"
#include <cstdarg>
// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal
// from http://wiznet.gr/src/ccolor.zip
// edited by rfree - as part of https://github.com/rfree/Open-Transactions/
using namespace std;
#ifdef _MSC_VER
#define snprintf c99_snprintf
inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) {
int count = -1;
if (size != 0)
count = _vsnprintf_s(str, size, _TRUNCATE, format, ap);
if (count == -1)
count = _vscprintf(format, ap);
return count;
}
inline int c99_snprintf(char* str, size_t size, const char* format, ...) {
int count;
va_list ap;
va_start(ap, format);
count = c99_vsnprintf(str, size, format, ap);
va_end(ap);
return count;
}
#endif // _MSC_VER
#define CC_CONSOLE_COLOR_DEFAULT "\033[0m"
#define CC_FORECOLOR(C) "\033[" #C "m"
#define CC_BACKCOLOR(C) "\033[" #C "m"
#define CC_ATTR(A) "\033[" #A "m"
namespace zkr
{
enum Color
{
Black,
Red,
Green,
Yellow,
Blue,
Magenta,
Cyan,
White,
Default = 9
};
enum Attributes
{
Reset,
Bright,
Dim,
Underline,
Blink,
Reverse,
Hidden
};
char * cc::color(int attr, int fg, int bg)
{
static const int size = 20;
static char command[size];
/* Command is the control command to the terminal */
snprintf(command, size, "%c[%d;%d;%dm", 0x1B, attr, fg + 30, bg + 40);
return command;
}
const char *cc::console = CC_CONSOLE_COLOR_DEFAULT;
const char *cc::underline = CC_ATTR(4);
const char *cc::bold = CC_ATTR(1);
const char *cc::fore::black = CC_FORECOLOR(30);
const char *cc::fore::blue = CC_FORECOLOR(34);
const char *cc::fore::red = CC_FORECOLOR(31);
const char *cc::fore::magenta = CC_FORECOLOR(35);
const char *cc::fore::green = CC_FORECOLOR(92);
const char *cc::fore::cyan = CC_FORECOLOR(36);
const char *cc::fore::yellow = CC_FORECOLOR(33);
const char *cc::fore::white = CC_FORECOLOR(37);
const char *cc::fore::console = CC_FORECOLOR(39);
const char *cc::fore::lightblack = CC_FORECOLOR(90);
const char *cc::fore::lightblue = CC_FORECOLOR(94);
const char *cc::fore::lightred = CC_FORECOLOR(91);
const char *cc::fore::lightmagenta = CC_FORECOLOR(95);
const char *cc::fore::lightgreen = CC_FORECOLOR(92);
const char *cc::fore::lightcyan = CC_FORECOLOR(96);
const char *cc::fore::lightyellow = CC_FORECOLOR(93);
const char *cc::fore::lightwhite = CC_FORECOLOR(97);
const char *cc::back::black = CC_BACKCOLOR(40);
const char *cc::back::blue = CC_BACKCOLOR(44);
const char *cc::back::red = CC_BACKCOLOR(41);
const char *cc::back::magenta = CC_BACKCOLOR(45);
const char *cc::back::green = CC_BACKCOLOR(42);
const char *cc::back::cyan = CC_BACKCOLOR(46);
const char *cc::back::yellow = CC_BACKCOLOR(43);
const char *cc::back::white = CC_BACKCOLOR(47);
const char *cc::back::console = CC_BACKCOLOR(49);
const char *cc::back::lightblack = CC_BACKCOLOR(100);
const char *cc::back::lightblue = CC_BACKCOLOR(104);
const char *cc::back::lightred = CC_BACKCOLOR(101);
const char *cc::back::lightmagenta = CC_BACKCOLOR(105);
const char *cc::back::lightgreen = CC_BACKCOLOR(102);
const char *cc::back::lightcyan = CC_BACKCOLOR(106);
const char *cc::back::lightyellow = CC_BACKCOLOR(103);
const char *cc::back::lightwhite = CC_BACKCOLOR(107);
}

73
contrib/otshell_utils/ccolor.hpp Normal file
View file

@ -0,0 +1,73 @@
// ccolor.hpp
// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal
// from http://wiznet.gr/src/ccolor.zip
// edited by rfree - as part of https://github.com/rfree/Open-Transactions/
#ifndef INCLUDE_OT_ccolor
#define INCLUDE_OT_ccolor
#include <iostream>
#include <stdio.h>
namespace zkr
{
class cc
{
public:
class fore
{
public:
static const char *black;
static const char *blue;
static const char *red;
static const char *magenta;
static const char *green;
static const char *cyan;
static const char *yellow;
static const char *white;
static const char *console;
static const char *lightblack;
static const char *lightblue;
static const char *lightred;
static const char *lightmagenta;
static const char *lightgreen;
static const char *lightcyan;
static const char *lightyellow;
static const char *lightwhite;
};
class back
{
public:
static const char *black;
static const char *blue;
static const char *red;
static const char *magenta;
static const char *green;
static const char *cyan;
static const char *yellow;
static const char *white;
static const char *console;
static const char *lightblack;
static const char *lightblue;
static const char *lightred;
static const char *lightmagenta;
static const char *lightgreen;
static const char *lightcyan;
static const char *lightyellow;
static const char *lightwhite;
};
static char *color(int attr, int fg, int bg);
static const char *console;
static const char *underline;
static const char *bold;
};
}
#endif

52
contrib/otshell_utils/lib_common1.hpp Normal file
View file

@ -0,0 +1,52 @@
/* See other files here for the LICENCE that applies here. */
#ifndef INCLUDE_OT_NEWCLI_COMMON1
#define INCLUDE_OT_NEWCLI_COMMON1
#include <string>
#include <cstring>
#include <vector>
#include <map>
#include <list>
#include <algorithm>
#include <iostream>
#include <fstream>
#include <sstream>
#include <set>
#include <iterator>
#include <stdexcept>
#include <functional>
#include <memory>
#include <thread>
#include <atomic>
#include <mutex>
// list of thigs from libraries that we pull into namespace nOT::nNewcli
// we might still need to copy/paste it in few places to make IDEs pick it up correctly
#define INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 \
using std::string; \
using std::vector; \
using std::vector; \
using std::list; \
using std::set; \
using std::map; \
using std::ostream; \
using std::istream; \
using std::cin; \
using std::cerr; \
using std::cout; \
using std::cerr; \
using std::endl; \
using std::function; \
using std::unique_ptr; \
using std::shared_ptr; \
using std::weak_ptr; \
using std::enable_shared_from_this; \
using std::mutex; \
using std::lock_guard; \
#endif

69
contrib/otshell_utils/runoptions.cpp Normal file
View file

@ -0,0 +1,69 @@
/* See other files here for the LICENCE that applies here. */
/* See header file .hpp for info */
#include "runoptions.hpp"
#include "lib_common1.hpp"
namespace nOT {
INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces
// (no debug - this is the default)
// +nodebug (no debug)
// +debug ...... --asdf
// +debug +debugcerr .... --asfs
// +debug +debugfile .... --asfs
cRunOptions::cRunOptions()
: mRunMode(eRunModeCurrent), mDebug(false), mDebugSendToFile(false), mDebugSendToCerr(false)
,mDoRunDebugshow(false)
{ }
vector<string> cRunOptions::ExecuteRunoptionsAndRemoveThem(const vector<string> & args) {
vector<string> arg_clear; // will store only the arguments that are not removed
for (auto arg : args) {
bool thisIsRunoption=false;
if (arg.size()>0) {
if (arg.at(0) == '+') thisIsRunoption=true;
}
if (thisIsRunoption) Exec(arg); // ***
if (! thisIsRunoption) arg_clear.push_back(arg);
}
Normalize();
return arg_clear;
}
void cRunOptions::Exec(const string & runoption) { // eg: Exec("+debug");
if (runoption == "+nodebug") { mDebug=false; }
else if (runoption == "+debug") { mDebug=true; }
else if (runoption == "+debugcerr") { mDebug=true; mDebugSendToCerr=true; }
else if (runoption == "+debugfile") { mDebug=true; mDebugSendToFile=true; }
else if (runoption == "+demo") { mRunMode=eRunModeDemo; }
else if (runoption == "+normal") { mRunMode=eRunModeNormal; }
else if (runoption == "+current") { mRunMode=eRunModeCurrent; }
else if (runoption == "+debugshow") { mDebug=true; mDebugSendToCerr=true; mDoRunDebugshow=true; }
else {
cerr << "Unknown runoption in Exec: '" << runoption << "'" << endl;
throw std::runtime_error("Unknown runoption");
}
// cerr<<"debug="<<mDebug<<endl;
}
void cRunOptions::Normalize() {
if (mDebug) {
if (!( mDebugSendToFile || mDebugSendToCerr )) mDebugSendToCerr=true; // if debug is on then send to something, e.g. to cerr
}
}
cRunOptions gRunOptions; // (extern)
} // namespace OT

58
contrib/otshell_utils/runoptions.hpp Normal file
View file

@ -0,0 +1,58 @@
/* See other files here for the LICENCE that applies here. */
/*
Template for new files, replace word "template" and later delete this line here.
*/
#ifndef INCLUDE_OT_NEWCLI_runoptions_hpp
#define INCLUDE_OT_NEWCLI_runoptions_hpp
#include "lib_common1.hpp"
namespace nOT {
INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces
/** Global options to run this program main() Eg used for developer's special options like +setdemo +setdebug.
This is NOT for all the other options that are parsed and executed by program. */
class cRunOptions {
public:
enum tRunMode { ///< Type of run mode - is this normal, or demonstration etc.
eRunModeCurrent=1, ///< currently developed version
eRunModeDemo, ///< best currently available Demo of something nice
eRunModeNormal, ///< do the normal things that the program should do
};
private:
tRunMode mRunMode; ///< selected run mode
bool mDebug; // turn debug on, Eg: +debug without it probably nothing will be written to debug (maybe just error etc)
bool mDebugSendToFile; // send to file, Eg: for +debugfile ; also turns on debug
bool mDebugSendToCerr; // send to cerr, Eg: for +debugcerr ; also turns on debug
// if debug is set but not any other DebugSend* then we will default to sending to debugcerr
bool mDoRunDebugshow;
public:
tRunMode getTRunMode() const { return mRunMode; }
bool getDebug() const { return mDebug; }
bool getDebugSendToFile() const { return mDebugSendToFile; }
bool getDebugSendToCerr() const { return mDebugSendToCerr; }
bool getDoRunDebugshow() const { return mDoRunDebugshow; }
cRunOptions();
vector<string> ExecuteRunoptionsAndRemoveThem(const vector<string> & args);
void Exec(const string & runoption); // eg: Exec("+debug");
void Normalize();
};
extern cRunOptions gRunOptions;
} // namespace nOT
#endif

804
contrib/otshell_utils/utils.cpp Normal file
View file

@ -0,0 +1,804 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug
/* See other files here for the LICENCE that applies here. */
/* See header file .hpp for info */
#include <algorithm>
#include <functional>
#include <cctype>
#include <locale>
#include <fstream>
#include <iostream>
#include <iomanip>
#include "utils.hpp"
#include "ccolor.hpp"
#include "lib_common1.hpp"
#include "runoptions.hpp"
#if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined (WIN64)
#define OS_TYPE_WINDOWS
#elif defined(__unix__) || defined(__posix) || defined(__linux) || defined(__darwin) || defined(__APPLE__) || defined(__clang__)
#define OS_TYPE_POSIX
#else
#warning "Compiler/OS platform is not recognized. Just assuming it will work as POSIX then"
#define OS_TYPE_POSIX
#endif
#if defined(OS_TYPE_WINDOWS)
#include <windows.h>
#elif defined(OS_TYPE_POSIX)
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#else
#error "Compiler/OS platform detection failed - not supported"
#endif
namespace nOT {
namespace nUtils {
INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces
// ====================================================================
// Numerical values of the debug levels - see hpp
const int _debug_level_nr_dbg3=20;
const int _debug_level_nr_dbg2=30;
const int _debug_level_nr_dbg1=40;
const int _debug_level_nr_info=50;
const int _debug_level_nr_note=60;
const int _debug_level_nr_fact=75;
const int _debug_level_nr_mark=80;
const int _debug_level_nr_warn=90;
const int _debug_level_nr_erro=100;
// ====================================================================
myexception::myexception(const char * what)
: std::runtime_error(what)
{ }
myexception::myexception(const std::string &what)
: std::runtime_error(what)
{ }
void myexception::Report() const {
_erro("Error: " << what());
}
//myexception::~myexception() { }
// ====================================================================
// text trimming
// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
std::string & ltrim(std::string &s) {
s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun<int, int>(std::isspace))));
return s;
}
std::string & rtrim(std::string &s) {
s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end());
return s;
}
std::string & trim(std::string &s) {
return ltrim(rtrim(s));
}
std::string get_current_time() {
std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
time_t time_now = std::chrono::system_clock::to_time_t(now);
std::chrono::high_resolution_clock::duration duration = now.time_since_epoch();
int64_t micro = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
// std::localtime() - This function may not be thread-safe.
#ifdef OS_TYPE_WINDOWS
struct tm * tm_pointer = std::localtime( &time_now ); // thread-safe on mingw-w64 (thread local variable) and on MSVC btw
// http://stackoverflow.com/questions/18551409/localtime-r-support-on-mingw
// tm_pointer points to thread-local data, memory is owned/managed by the system/library
#else
// linux, freebsd, have this
struct tm tm_object; // automatic storage duration http://en.cppreference.com/w/cpp/language/storage_duration
struct tm * tm_pointer = & tm_object; // just point to our data
auto x = localtime_r( &time_now , tm_pointer ); // modifies our own (this thread) data in tm_object, this is safe http://linux.die.net/man/3/localtime_r
if (x != tm_pointer) return "(internal error in get_current_time)"; // redundant check in case of broken implementation of localtime_r
#endif
// tm_pointer now points to proper time data, and that memory is automatically managed
if (!tm_pointer) return "(internal error in get_current_time - NULL)"; // redundant check in case of broken implementation of used library methods
std::stringstream stream;
stream << std::setfill('0')
<< std::setw(2) << tm_pointer->tm_year+1900
<< '-' << std::setw(2) << tm_pointer->tm_mon+1
<< '-' << std::setw(2) << tm_pointer->tm_mday
<< ' ' << std::setw(2) << tm_pointer->tm_hour
<< ':' << std::setw(2) << tm_pointer->tm_min
<< ':' << std::setw(2) << tm_pointer->tm_sec
<< '.' << std::setw(6) << (micro%1000000); // 6 because microseconds
return stream.str();
}
cNullstream g_nullstream; // extern a stream that does nothing (eats/discards data)
std::recursive_mutex gLoggerGuard; // extern
std::atomic<int> gLoggerGuardDepth; // extern
std::atomic<int> & gLoggerGuardDepth_Get() {
// TODO std::once would be nicer here
static bool once=0;
if (!once) { // initialize it once
once=1;
gLoggerGuardDepth=0;
}
return gLoggerGuardDepth; // global, atomic counter
}
// ====================================================================
namespace nDetail {
const char* DbgShortenCodeFileName(const char *s) {
const char *p = s;
const char *a = s;
bool inc=1;
while (*p) {
++p;
if (inc && ('\0' != * p)) { a=p; inc=false; } // point to the current character (if valid) becasue previous one was slash
if ((*p)=='/') { a=p; inc=true; } // point at current slash (but set inc to try to point to next character)
}
return a;
}
}
// a workaround for MSVC compiler; e.g. see https://bugs.webkit.org/show_bug.cgi?format=multiple&id=125795
#ifndef _MSC_VER
template<typename T, typename ...Args>
std::unique_ptr<T> make_unique( Args&& ...args )
{
return std::unique_ptr<T>( new T( std::forward<Args>(args)... ) );
}
#else
using std::make_unique;
#endif
// ====================================================================
char cFilesystemUtils::GetDirSeparatorSys() {
// TODO nicer os detection?
#if defined(OS_TYPE_POSIX)
return '/';
#elif defined(OS_TYPE_WINDOWS)
return '\\';
#else
#error "Do not know how to compile this for your platform."
#endif
}
char cFilesystemUtils::GetDirSeparatorInter() {
return '/';
}
string cFilesystemUtils::FileInternalToSystem(const std::string &name) {
string ret;
ret.resize(name.size());
std::replace_copy(name.begin(), name.end(), ret.begin(),
GetDirSeparatorInter() , GetDirSeparatorSys());
return ret;
}
string cFilesystemUtils::FileSystemToInternal(const std::string &name) {
string ret;
ret.reserve(name.size());
std::replace_copy(name.begin(), name.end(), ret.begin(),
GetDirSeparatorSys() , GetDirSeparatorInter());
return ret;
}
bool cFilesystemUtils::CreateDirTree(const std::string & dir, bool only_below) {
const bool dbg=false;
//struct stat st;
const char dirchS = cFilesystemUtils::GetDirSeparatorSys();
const char dirchI = cFilesystemUtils::GetDirSeparatorInter();
std::istringstream iss(dir);
string partI; // current par is in internal format (though it should not matter since it doesn't contain any slashes). eg "bar"
string sofarS=""; // sofarS - the so far created dir part is in SYSTEM format. eg "foo/bar"
if (dir.size()<1) return false; // illegal name
// dir[0] is valid from here
if ( only_below && ((dir[0]==dirchS) || (dir[0]==dirchI))) return false; // no jumping to top (on any os)
while (getline(iss,partI,dirchI)) { // get new component eg "bar" into part
if (dbg) cout << '['<<partI<<']' << endl;
sofarS += partI;
if (partI.size()<1) return false; // bad format?
if ((only_below) && (partI=="..")) return false; // trying to go up
if (dbg) cout << "test ["<<sofarS<<"]"<<endl;
// TODO nicer os detection?
#if defined(OS_TYPE_POSIX)
struct stat st;
bool exists = stat(sofarS.c_str() ,&st) == 0; // *
if (exists) {
if (! S_ISDIR(st.st_mode)) {
// std::cerr << "This exists, but as a file: [" << sofar << "]" << (size_t)st.st_ino << endl;
return false; // exists but is a file nor dir
}
}
#elif defined(OS_TYPE_WINDOWS)
DWORD dwAttrib = GetFileAttributesA(sofarS.c_str());
bool exists = (dwAttrib != INVALID_FILE_ATTRIBUTES && (dwAttrib & FILE_ATTRIBUTE_DIRECTORY));
#else
#error "Do not know how to compile this for your platform."
#endif
if (!exists) {
if (dbg) cout << "mkdir ["<<sofarS<<"]"<<endl;
#if defined(OS_TYPE_POSIX)
bool ok = 0== mkdir(sofarS.c_str(), 0700); // ***
#elif defined(OS_TYPE_WINDOWS)
bool ok = (bool) CreateDirectoryA(sofarS.c_str(), NULL); // TODO use -W() after conversion to unicode UTF16
#else
#error "Do not know how to compile this for your platform."
#endif
if (!ok) return false;
}
sofarS += dirchS;
}
return true;
}
// ====================================================================
namespace nDetail {
struct channel_use_info { ///< feedback information about using (e.g. opening) given debug channel - used internally by logging system
/// TODO not yet used in code
/// e.g. used to write into channel net/in/all that given message was a first logged message from never-before-logged thread or PID etc
bool m_was_interesting; ///< anything interesting happened when using the channel?
std::vector<std::string> m_extra_msg; ///< any additional messages about this channel use
};
cDebugScopeGuard::cDebugScopeGuard() : mLevel(-1) {
}
cDebugScopeGuard::~cDebugScopeGuard() {
if (mLevel != -1) {
gCurrentLogger.write_stream(mLevel,mChan) << mMsg << " ... end" << gCurrentLogger.endline() << std::flush;
}
}
void cDebugScopeGuard::Assign(const string &chan, const int level, const string &msg) {
mChan=chan;
mLevel=level;
mMsg=msg;
}
} // namespace nDetail
// ====================================================================
cLogger::cLogger() :
mStream(NULL),
mStreamBrokenDebug(NULL),
mIsBroken(true), // before constructor finishes
mLevel(_debug_level_nr_warn),
mThread2Number_Biggest(0), // the CURRENT biggest value (no thread yet in map)
mPid2Number_Biggest(0)
{
mStream = & std::cout;
mStreamBrokenDebug = & std::cerr; // the backup stream
*mStreamBrokenDebug << "Creating the logger system" << endl;
mIsBroken=false; // ok, constr. succeeded, so string is not broken now
// this is here, because it could be using logging itself to log creation of first thread/PID etc
Thread2Number( std::this_thread::get_id() ); // convert current id to short number, useful to reserve a number so that main thread is usually called 1
Pid2Number( getpid() ); // add this proces ID as first one
}
cLogger::~cLogger() {
for (auto pair : mChannels) {
std::ofstream *ptr = pair.second;
delete ptr;
pair.second=NULL;
}
}
void cLogger::SetStreamBroken() {
SetStreamBroken("(no additional details about this problem)");
}
void cLogger::SetStreamBroken(const std::string &msg) {
_dbg_dbg("Stream is broken (msg: " << msg << ")");
if (!mIsBroken) { // if not already marked as broken
_dbg_dbg("(It was not broken before)");
std::cerr << OT_CODE_STAMP << "WARNING: due to a problem in the debug/logging system itself ("<<msg<<") - we are switching back to fallback stream (e.g. cerr)" << std::endl;
if (mStreamBrokenDebug == nullptr) {
std::cerr << OT_CODE_STAMP << " ERROR: in addition, while reporting this problem, mStreamBrokenDebug stream is NULL: " << mStreamBrokenDebug << std::endl;
} else {
(*mStreamBrokenDebug) << OT_CODE_STAMP << "WARNING: due to debug stream problem ("<<msg<<") - switching back to fallback stream (e.g. cerr)" << std::endl;
}
mIsBroken = true;
}
}
std::ostream & cLogger::write_stream(int level) {
return write_stream(level,"");
}
std::ostream & cLogger::write_stream(int level, const std::string & channel ) {
_dbg_dbg("level="<<level<<" channel="<<channel);
if (level >= mLevel) {
if (mStream) { // TODO now disabling mStream also disables writting to any channel
_dbg_dbg("Selecting output...");
ostream & output = SelectOutput(level,channel);
_dbg_dbg("Selecting output... done, output=" << (void*)(&output));
#if defined(OS_TYPE_WINDOWS)
output << windows_stream(level);
#endif
output << icon(level) << ' ';
std::thread::id this_id = std::this_thread::get_id();
output << "{" << Thread2Number(this_id) << "}";
auto nicePid = Pid2Number(getpid());
if (nicePid>0) output << " {p" << nicePid << "}";
output << ' ';
return output; // <--- return
} else _dbg_dbg("Not writting: No mStream");
} else _dbg_dbg("Not writting: Too low level level="<<level<<" not >= mLevel="<<mLevel);
return g_nullstream;
}
std::string cLogger::GetLogBaseDir() const {
return "log";
}
void cLogger::OpenNewChannel(const std::string & channel) noexcept {
try {
_dbg_dbg("Openning channel for channel="<<channel);
OpenNewChannel_(channel);
}
catch (const std::exception &except) {
SetStreamBroken(OT_CODE_STAMP + " Got exception when opening debug channel: " + ToStr(except.what()));
}
catch (...) {
SetStreamBroken(OT_CODE_STAMP + " Got not-standard exception when opening debug channel.");
}
}
void cLogger::OpenNewChannel_(const std::string & channel) { // channel=="net/sleep"
_dbg_dbg("Openning channel for channel="<<channel);
size_t last_split = channel.find_last_of(cFilesystemUtils::GetDirSeparatorInter());
string fname_system; // the full file name in system format
if (last_split==string::npos) { // The channel name has no directory, eg channel=="test"
string dir = GetLogBaseDir();
string basefile = channel + ".log";
string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile;
fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <-
}
else { // there is a directory eg channel=="net/sleep"
// net/sleep
// ^----- last_split
string dir = GetLogBaseDir() + cFilesystemUtils::GetDirSeparatorInter() + channel.substr(0, last_split);
string basefile = channel.substr(last_split+1) + ".log";
string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile;
fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <-
bool dirok = cFilesystemUtils::CreateDirTree(dir);
if (!dirok) { string err = "In logger failed to open directory (" + dir +") for channel (" + channel +")"; throw std::runtime_error(err); }
}
_dbg_dbg("Openning fname_system="<<fname_system);
std::ofstream * thefile = new std::ofstream( fname_system.c_str() ); // file system
*thefile << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl;
// cerr << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl;
_dbg_dbg( "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" );
mChannels.insert( std::pair<string,std::ofstream*>(channel , thefile ) ); // <- created the channel mapping
}
std::ostream & cLogger::SelectOutput(int level, const std::string & channel) noexcept {
try {
if (mIsBroken) {
_dbg_dbg("The stream is broken mIsBroken="<<mIsBroken<<" so will return backup stream");
return *mStreamBrokenDebug;
}
if (channel=="") {
_dbg_dbg("No channel given (channel="<<channel<<") so will return main stream");
return *mStream;
}
auto obj = mChannels.find(channel);
if (obj == mChannels.end()) { // not found - need to make new channel
_dbg_dbg("No stream openened for channel="<<channel<<" so will create it now");
OpenNewChannel(channel); // <- create channel
obj = mChannels.find(channel); // find again
if (obj == mChannels.end()) { // still not found! something is wrong
SetStreamBroken( OT_CODE_STAMP + " WARNING: can not get stream for channel="+ToStr(channel)+" level="+ToStr(channel) );
return *mStreamBrokenDebug;
}
}
auto the_stream_ptr = obj->second;
_dbg_dbg("Found the stream file for channel="<<channel<<" as the_stream_ptr="<<the_stream_ptr);
ASRT(the_stream_ptr);
return *the_stream_ptr; // <--- RETURN
}
catch (std::exception &except) {
SetStreamBroken( OT_CODE_STAMP + " Got exception: " + ToStr(except.what()) );
_dbg_dbg("Exception! Returning broken stream");
return *mStreamBrokenDebug;
}
catch (...) {
SetStreamBroken( OT_CODE_STAMP + " Got not-standard exception.");
_dbg_dbg("Exception! Returning broken stream");
return *mStreamBrokenDebug;
}
// dead code
}
void cLogger::setOutStreamFile(const string &fname) { // switch to using this file
_mark("WILL SWITCH DEBUG NOW to file: " << fname);
mOutfile = make_unique<std::ofstream>(fname);
mStream = & (*mOutfile);
_mark("Started new debug, to file: " << fname);
}
void cLogger::setOutStreamFromGlobalOptions() {
if ( gRunOptions.getDebug() ) {
if ( gRunOptions.getDebugSendToFile() ) {
mOutfile = make_unique<std::ofstream> ("debuglog.txt");
mStream = & (*mOutfile);
}
else if ( gRunOptions.getDebugSendToCerr() ) {
mStream = & std::cerr;
}
else {
mStream = & g_nullstream;
}
}
else {
mStream = & g_nullstream;
}
}
void cLogger::setDebugLevel(int level) {
bool note_before = (mLevel > level); // report the level change before or after the change? (on higher level)
if (note_before) _note("Setting debug level to "<<level);
mLevel = level;
if (!note_before) _note("Setting debug level to "<<level);
}
std::string cLogger::icon(int level) const {
// TODO replan to avoid needles converting back and forth char*, string etc
using namespace zkr;
#if defined(OS_TYPE_POSIX)
if (level >= 100) return cc::back::lightred + ToStr(cc::fore::lightyellow) + ToStr("ERROR ") + ToStr(cc::fore::lightyellow) + " " ;
if (level >= 90) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("Warn ") + ToStr(cc::fore::red)+ " " ;
if (level >= 80) return cc::back::lightmagenta + ToStr(cc::fore::black) + ToStr("MARK "); //+ zkr::cc::console + ToStr(cc::fore::lightmagenta)+ " ";
if (level >= 75) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("FACT ") + zkr::cc::console + ToStr(cc::fore::lightyellow)+ " ";
if (level >= 70) return cc::fore::green + ToStr("Note ");
if (level >= 50) return cc::fore::cyan + ToStr("info ");
if (level >= 40) return cc::fore::lightwhite + ToStr("dbg ");
if (level >= 30) return cc::fore::lightblue + ToStr("dbg ");
if (level >= 20) return cc::fore::blue + ToStr("dbg ");
#elif defined(OS_TYPE_WINDOWS)
if (level >= 100) return ToStr("ERROR ");
if (level >= 90) return ToStr("Warn ");
if (level >= 80) return ToStr("MARK ");
if (level >= 75) return ToStr("FACT ");
if (level >= 70) return ToStr("Note ");
if (level >= 50) return ToStr("info ");
if (level >= 40) return ToStr("dbg ");
if (level >= 30) return ToStr("dbg ");
if (level >= 20) return ToStr("dbg ");
#endif
return " ";
}
std::string cLogger::endline() const {
#if defined(OS_TYPE_POSIX)
return ToStr("") + zkr::cc::console + ToStr("\n"); // TODO replan to avoid needles converting back and forth char*, string etc
#elif defined(OS_TYPE_WINDOWS)
return ToStr("\n");
#endif
}
int cLogger::Thread2Number(const std::thread::id id) {
auto found = mThread2Number.find( id );
if (found == mThread2Number.end()) { // new one
mThread2Number_Biggest++;
mThread2Number[id] = mThread2Number_Biggest;
_info_c("dbg/main", "This is a new thread (used in debug), thread id="<<id); // can cause some recursion
return mThread2Number_Biggest;
} else {
return mThread2Number[id];
}
}
int cLogger::Pid2Number(const t_anypid id) {
auto found = mPid2Number.find( id );
if (found == mPid2Number.end()) { // new one
mPid2Number_Biggest++;
mPid2Number[id] = mPid2Number_Biggest;
_info_c("dbg/main", "This is a new process (used in debug), process pid="<<id); // can cause some recursion
return mPid2Number_Biggest;
} else {
return mPid2Number[id];
}
}
// ====================================================================
// object gCurrentLogger is defined later - in global namespace below
// ====================================================================
// vector debug
void DisplayStringEndl(std::ostream & out, const std::string text) {
out << text;
out << std::endl;
}
std::string SpaceFromEscape(const std::string &s) {
std::ostringstream newStr;
for(size_t i = 0; i < s.length();i++) {
if(s[i] == '\\' && s[i+1] ==32)
newStr<<"";
else
newStr<<s[i];
}
return newStr.str();
}
std::string EscapeFromSpace(const std::string &s) {
std::ostringstream newStr;
for(size_t i = 0; i < s.length();i++) {
if(s[i] == 32)
newStr << "\\" << " ";
else
newStr << s[i];
}
return newStr.str();
}
std::string EscapeString(const std::string &s) {
std::ostringstream newStr;
for(size_t i = 0; i < s.length();i++) {
if(s[i] >=32 && s[i] <= 126)
newStr<<s[i];
else
newStr<<"\\"<< (int) s[i];
}
return newStr.str();
}
bool CheckIfBegins(const std::string & beggining, const std::string & all) {
if (all.compare(0, beggining.length(), beggining) == 0) {
return 1;
}
else {
return 0;
}
}
bool CheckIfEnds (std::string const & ending, std::string const & all){
if (all.length() >= ending.length()) {
return (0 == all.compare (all.length() - ending.length(), ending.length(), ending));
} else {
return false;
}
}
vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib) {
vector<string> ret;
for ( auto rec : possib) { // check of possibilities
if (CheckIfBegins(sofar,rec)) {
rec = EscapeFromSpace(rec);
ret.push_back(rec); // this record matches
}
}
return ret;
}
char GetLastChar(const std::string & str) { // TODO unicode?
auto s = str.length();
if (s==0) throw std::runtime_error("Getting last character of empty string (" + ToStr(s) + ")" + OT_CODE_STAMP);
return str.at( s - 1);
}
std::string GetLastCharIf(const std::string & str) { // TODO unicode?
auto s = str.length();
if (s==0) return ""; // empty string signalizes ther is nothing to be returned
return std::string( 1 , str.at( s - 1) );
}
// ====================================================================
// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no.
// Use it like this: ASRT( x>y ); with the semicolon at end, a clever trick forces this syntax :)
void Assert(bool result, const std::string &stamp, const std::string &condition) {
if (!result) {
_erro("Assert failed at "+stamp+": ASSERT( " << condition << ")");
throw std::runtime_error("Assert failed at "+stamp+": ASSERT( " + condition + ")");
}
}
// ====================================================================
// advanced string
const std::string GetMultiline(string endLine) {
std::string result(""); // Taken from OT_CLI_ReadUntilEOF
while (true) {
std::string input_line("");
if (std::getline(std::cin, input_line, '\n'))
{
input_line += "\n";
if (input_line[0] == '~')
break;
result += input_line;
}
if (std::cin.eof() )
{
std::cin.clear();
break;
}
if (std::cin.fail() )
{
std::cin.clear();
break;
}
if (std::cin.bad())
{
std::cin.clear();
break;
}
}
return result;
}
vector<string> SplitString(const string & str){
std::istringstream iss(str);
vector<string> vec { std::istream_iterator<string>{iss}, std::istream_iterator<string>{} };
return vec;
}
bool checkPrefix(const string & str, char prefix) {
if (str.at(0) == prefix)
return true;
return false;
}
// ====================================================================
// operation on files
#ifdef __unix
void cEnvUtils::GetTmpTextFile() {
// TODO make this name configurable (depending on project)
char filename[] = "/tmp/otshellutils_text.XXXXXX";
fd = mkstemp(filename);
if (fd == -1) {
_erro("Can't create the file: " << filename);
return;
}
mFilename = filename;
}
void cEnvUtils::CloseFile() {
close(fd);
unlink( mFilename.c_str() );
}
void cEnvUtils::OpenEditor() {
char* editor = std::getenv("OT_EDITOR"); //TODO Read editor from configuration file
if (editor == NULL)
editor = std::getenv("VISUAL");
if (editor == NULL)
editor = std::getenv("EDITOR");
string command;
if (editor != NULL)
command = ToStr(editor) + " " + mFilename;
else
command = "/usr/bin/editor " + mFilename;
_dbg3("Opening editor with command: " << command);
if ( system( command.c_str() ) == -1 )
_erro("Cannot execute system command: " << command);
}
const string cEnvUtils::ReadFromTmpFile() {
std::ifstream ifs(mFilename);
string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
return msg;
}
const string cEnvUtils::Compose() {
GetTmpTextFile();
OpenEditor();
string input = ReadFromTmpFile();
CloseFile();
return input;
}
#endif
const string cEnvUtils::ReadFromFile(const string path) {
std::ifstream ifs(path);
string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
return msg;
}
void hintingToTxt(std::fstream & file, string command, vector<string> &commands) {
if(file.good()) {
file<<command<<"~"<<endl;
for (auto a: commands) {
file <<a<< " ";
file.flush();
}
file<<endl;
}
}
string stringToColor(const string &hash) {
// Generete vector with all possible light colors
vector <string> lightColors;
using namespace zkr;
lightColors.push_back(cc::fore::lightblue);
lightColors.push_back(cc::fore::lightred);
lightColors.push_back(cc::fore::lightmagenta);
lightColors.push_back(cc::fore::lightgreen);
lightColors.push_back(cc::fore::lightcyan);
lightColors.push_back(cc::fore::lightyellow);
lightColors.push_back(cc::fore::lightwhite);
int sum=0;
for (auto ch : hash) sum+=ch;
auto color = sum%(lightColors.size()-1);
return lightColors.at( color );
}
// ====================================================================
// algorthms
} // namespace nUtil
} // namespace OT
// global namespace
const extern int _dbg_ignore = 0; // see description in .hpp
std::string GetObjectName() {
//static std::string * name=nullptr;
//if (!name) name = new std::string("(global)");
return "";
}
// ====================================================================
nOT::nUtils::cLogger gCurrentLogger;

532
contrib/otshell_utils/utils.hpp Normal file
View file

@ -0,0 +1,532 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug
/* See other files here for the LICENCE that applies here. */
#include "ccolor.hpp"
#ifndef INCLUDE_OT_NEWCLI_UTILS
#define INCLUDE_OT_NEWCLI_UTILS
#include "lib_common1.hpp"
#ifdef __unix
#include <unistd.h>
#endif
#if defined(_WIN32)
#include"windows_stream.h"
#endif
#ifndef CFG_WITH_TERMCOLORS
//#error "You requested to turn off terminal colors (CFG_WITH_TERMCOLORS), however currently they are hardcoded (this option to turn them off is not yet implemented)."
#endif
///Macros related to automatic deduction of class name etc;
#define MAKE_CLASS_NAME(NAME) private: static std::string GetObjectName() { return #NAME; }
#define MAKE_STRUCT_NAME(NAME) private: static std::string GetObjectName() { return #NAME; } public:
// define this to debug the debug system itself:
// #define opt_debug_debug
#ifdef opt_debug_debug
#define _dbg_dbg(X) do { std::cerr<<"_dbg_dbg: " << OT_CODE_STAMP << " {thread=" << std::this_thread::get_id()<<"} " \
<< " {pid="<<getpid()<<"} " << ": " << X << std::endl; } while(0)
#else
#define _dbg_dbg(X) do { } while(0)
#endif
namespace nOT {
namespace nUtils {
/// @brief general based for my runtime errors
class myexception : public std::runtime_error {
public:
myexception(const char * what);
myexception(const std::string &what);
//virtual ~myexception();
virtual void Report() const;
};
/// @macro Use this macro INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 as a shortcut for various using std::string etc.
INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces
// ======================================================================================
/// text trimming functions (they do mutate the passes string); they trim based on std::isspace. also return it's reference again
/// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
std::string & trim(std::string &s); ///< trim text http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
std::string & ltrim(std::string &s); ///< left trim
std::string & rtrim(std::string &s); ///< right trim
// ======================================================================================
std::string get_current_time();
// string conversions
template <class T>
std::string ToStr(const T & obj) {
std::ostringstream oss;
oss << obj;
return oss.str();
}
struct cNullstream : std::ostream {
cNullstream() : std::ios(0), std::ostream(0) {}
};
extern cNullstream g_nullstream; // a stream that does nothing (eats/discards data)
// ========== debug ==========
// _dbg_ignore is moved to global namespace (on purpose)
// TODO make _dbg_ignore thread-safe everywhere
extern std::recursive_mutex gLoggerGuard; // the mutex guarding logging/debugging code e.g. protecting streams, files, etc
std::atomic<int> & gLoggerGuardDepth_Get(); // getter for the global singleton of counter (it guarantees initializing it to 0). This counter shows the current recursion (re-entrant) level of debug macros.
// TODO more debug of the debug system:
// detect lock() error e.g. recursive limit
// detect stream e.g. operator<< error
#define _debug_level(LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \
_dbg_dbg("WRITE DEBUG: LEVEL="<<LEVEL<<" VAR: " << VAR ); \
auto level=LEVEL; short int part=0; \
try { \
std::lock_guard<std::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \
part=1; \
try { \
++nOT::nUtils::gLoggerGuardDepth_Get(); \
/* int counter = nOT::nUtils::gLoggerGuardDepth_Get(); if (counter!=1) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: recursion, counter="<<counter<<gCurrentLogger.endline(); */ \
gCurrentLogger.write_stream(LEVEL,"") << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \
part=9; \
} catch(...) { \
gCurrentLogger.write_stream(std::max(level,90),"") << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << "(ERROR IN DEBUG)" << gCurrentLogger.endline(); \
--nOT::nUtils::gLoggerGuardDepth_Get(); throw ; \
} \
--nOT::nUtils::gLoggerGuardDepth_Get(); \
} catch(...) { if (part<8) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: problem in debug mechanism e.g. in locking." <<gCurrentLogger.endline(); throw ; } \
} } while(0)
// info for code below: oss object is normal stack variable, using it does not need lock protection
#define _debug_level_c(CHANNEL,LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \
_dbg_dbg("WRITE DEBUG: LEVEL="<<LEVEL<<" CHANNEL="<<CHANNEL<<" VAR: " << VAR ); \
auto level=LEVEL; short int part=0; \
try { \
std::lock_guard<std::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \
part=1; \
try { \
++nOT::nUtils::gLoggerGuardDepth_Get(); \
std::ostringstream oss; \
oss << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \
std::string as_string = oss.str(); \
_dbg_dbg("START will write to log LEVEL="<<LEVEL<<" to CHANNEL="<<CHANNEL<<" as_string="<<as_string); \
/* int counter = nOT::nUtils::gLoggerGuardDepth_Get(); if (counter!=1) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: recursion, counter="<<counter<<gCurrentLogger.endline(); */ \
gCurrentLogger.write_stream(LEVEL,"" ) << as_string << gCurrentLogger.endline() << std::flush; \
gCurrentLogger.write_stream(LEVEL,CHANNEL) << as_string << gCurrentLogger.endline() << std::flush; \
_dbg_dbg("DONE will write to log LEVEL="<<LEVEL<<" to CHANNEL="<<CHANNEL<<" as_string="<<as_string); \
part=9; \
} catch(...) { \
gCurrentLogger.write_stream(std::max(level,90),CHANNEL) << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << "(ERROR IN DEBUG)" << gCurrentLogger.endline(); \
--nOT::nUtils::gLoggerGuardDepth_Get(); throw ; \
} \
--nOT::nUtils::gLoggerGuardDepth_Get(); \
} catch(...) { if (part<8) gCurrentLogger.write_stream(100,CHANNEL)<<"DEBUG-ERROR: problem in debug mechanism e.g. in locking." <<gCurrentLogger.endline(); throw ; } \
} } while(0)
// Numerical values of the debug levels - are defined here as const ints. Full name (with namespace) given for clarity.
extern const int _debug_level_nr_dbg3;
extern const int _debug_level_nr_dbg2;
extern const int _debug_level_nr_dbg1;
extern const int _debug_level_nr_info;
extern const int _debug_level_nr_note;
extern const int _debug_level_nr_fact;
extern const int _debug_level_nr_mark;
extern const int _debug_level_nr_warn;
extern const int _debug_level_nr_erro;
#define _dbg3(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg3,VAR) // details - most detailed
#define _dbg2(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg2,VAR) // details - a bit more important
#define _dbg1(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg1,VAR) // details - more important
#define _info(VAR) _debug_level( nOT::nUtils::_debug_level_nr_info,VAR) // information
#define _note(VAR) _debug_level( nOT::nUtils::_debug_level_nr_note,VAR) // more interesting information
#define _fact(VAR) _debug_level( nOT::nUtils::_debug_level_nr_fact,VAR) // interesting events that could be interesting even for user, for logical/business things
#define _mark(VAR) _debug_level( nOT::nUtils::_debug_level_nr_mark,VAR) // marked actions
#define _warn(VAR) _debug_level( nOT::nUtils::_debug_level_nr_warn,VAR) // some problems
#define _erro(VAR) _debug_level( nOT::nUtils::_debug_level_nr_erro,VAR) // errors
#define _dbg3_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg3, VAR) // details - most detailed
#define _dbg2_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg2, VAR) // details - a bit more important
#define _dbg1_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg1, VAR) // details - more important
#define _info_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_info, VAR) // information
#define _note_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_note, VAR) // more interesting information
#define _fact_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_fact, VAR) // interesting events that could be interesting even for user, for logical/business things
#define _mark_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_mark, VAR) // marked actions
#define _warn_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_warn, VAR) // some problems
#define _erro_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_erro, VAR) // errors
// lock // because of VAR
#define _scope_debug_level_c(CHANNEL,LEVEL,VAR) \
std::ostringstream debug_detail_oss; \
nOT::nUtils::gLoggerGuard.lock(); \
debug_detail_oss << OT_CODE_STAMP << ' ' << VAR ; \
nOT::nUtils::nDetail::cDebugScopeGuard debugScopeGuard; \
if (_dbg_ignore<LEVEL) debugScopeGuard.Assign(CHANNEL,LEVEL, debug_detail_oss.str()); \
if (_dbg_ignore<LEVEL) _debug_level_c(CHANNEL,LEVEL,debug_detail_oss.str() + " ... begin"); \
nOT::nUtils::gLoggerGuard.unlock();
#define _scope_debug_level(LEVEL,VAR) _scope_debug_level_c("",LEVEL,VAR)
#define _scope_dbg1(VAR) _scope_debug_level( _debug_level_nr_dbg3, VAR)
#define _scope_dbg2(VAR) _scope_debug_level( _debug_level_nr_dbg2, VAR)
#define _scope_dbg3(VAR) _scope_debug_level( _debug_level_nr_dbg1, VAR)
#define _scope_info(VAR) _scope_debug_level( _debug_level_nr_info, VAR)
#define _scope_note(VAR) _scope_debug_level( _debug_level_nr_note, VAR)
#define _scope_fact(VAR) _scope_debug_level( _debug_level_nr_fact, VAR)
#define _scope_mark(VAR) _scope_debug_level( _debug_level_nr_mark, VAR)
#define _scope_warn(VAR) _scope_debug_level( _debug_level_nr_warn, VAR)
#define _scope_erro(VAR) _scope_debug_level( _debug_level_nr_erro, VAR)
/***
@brief do not use this namespace directly, it is implementation detail.
*/
namespace nDetail {
/***
@brief a Debug scope-guard, to log a debug message when current scope is left. Do NOT use this directly,
only use it via the macros like _scope_dbg1 etc.
*/
class cDebugScopeGuard {
protected:
string mMsg;
int mLevel;
string mChan;
public:
cDebugScopeGuard();
~cDebugScopeGuard();
void Assign(const string &chan, const int level, const string &msg);
};
const char* DbgShortenCodeFileName(const char *s); ///< Returns a pointer to some part of the string that was given, skipping directory names, for log/debug
} // namespace nDetail
// ========== logger ==========
namespace nDetail {
struct channel_use_info;
} // namespace nDetail
/***
@brief Class to write debug into. Used it by calling the debug macros _dbg1(...) _info(...) _erro(...) etc, NOT directly!
@author rfree (maintainer)
@thread this class is NOT thread safe and must used only by one thread at once (use it via ot_debug_macros like _info macro they do proper locking)
*/
class cLogger {
public:
cLogger();
~cLogger();
std::ostream & write_stream(int level); ///< starts a new message on given level (e.g. writes out the icon/tag) and returns stream to output to
std::ostream & write_stream(int level, const std::string & channel); ///< the same but with name of the debug channel
void setOutStreamFromGlobalOptions(); // set debug level, file etc - according to global Options
void setOutStreamFile(const std::string &fname); // switch to using this file
void setDebugLevel(int level); // change the debug level e.g. to mute debug from now
std::string icon(int level) const; ///< returns "icon" for given debug level. It is text, might include color controll characters
std::string endline() const; ///< returns string to be written at end of message
protected:
typedef long int t_anypid; // a portable representation of PID. long int should cover all platforms
void SetStreamBroken(); ///< call in case of internal error in logger (e.g. can not open a file)
void SetStreamBroken(const std::string &msg); ///< same but with error message
unique_ptr<std::ofstream> mOutfile;
std::ostream * mStream; ///< pointing only! can point to our own mOutfile, or maye to global null stream
std::ostream * mStreamBrokenDebug; ///< pointing only! this is a pointer to some stream that should be used when normal debugging is broken eg std::cerr
bool mIsBroken; ///< is the debugging system broken (this should be set when internal problems occur and should cause fallback to std::cerr)
std::map< std::string , std::ofstream * > mChannels; // the ofstream objects are owned by this class
int mLevel; ///< current debug level
std::ostream & SelectOutput(int level, const std::string & channel) noexcept; ///< returns a proper stream for this level and channel (always usable string)
void OpenNewChannel(const std::string & channel) noexcept; ///< tries to prepare this channel. does NOT guarantee to created mChannels[] entry!
void OpenNewChannel_(const std::string & channel); ///< internal function, will throw in case of problems
std::string GetLogBaseDir() const;
std::map< std::thread::id , int > mThread2Number; ///< change long thread IDs into a short nice number to show
int mThread2Number_Biggest; ///< current biggest value held there (biggest key) - works as growing-only counter basically
int Thread2Number(const std::thread::id id); ///< convert the system's thread id into a nice short our id; make one if new thread
std::map< t_anypid , int > mPid2Number; ///< change long proces PID into a short nice number to show
int mPid2Number_Biggest; ///< current biggest value held there (biggest key) - works as growing-only counter basically
int Pid2Number(const t_anypid id); ///< convert the system's PID id into a nice short our id; make one if new thread
};
// ====================================================================
// vector debug
template <class T>
std::string vectorToStr(const T & v) {
std::ostringstream oss;
for(auto rec: v) {
oss << rec <<",";
}
return oss.str();
}
template <class T>
void DisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
std::copy( v.begin(), v.end(), std::ostream_iterator<T>(out, delim.c_str()) );
}
template <class T>
void EndlDisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
out << std::endl;
DisplayVector(out,v,delim);
}
template <class T>
void DisplayVectorEndl(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
DisplayVector(out,v,delim);
out << std::endl;
}
template <class T>
void DbgDisplayVector(const std::vector<T> &v, const std::string &delim=" ") {
std::cerr << "[";
std::copy( v.begin(), v.end(), std::ostream_iterator<T>(std::cerr, delim.c_str()) );
std::cerr << "]";
}
string stringToColor(const string &hash);
template <class T, class T2>
void DisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") {
auto *no_color = zkr::cc::fore::console;
for(auto var : m) {
out << stringToColor(var.first) << var.first << delim << var.second << no_color << endl;
}
}
template <class T, class T2>
void EndlDisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") {
out << endl;
for(auto var : m) {
out << var.first << delim << var.second << endl;
}
}
template <class T, class T2>
void DbgDisplayMap(const std::map<T, T2> &m, const std::string &delim=" ") {
for(auto var : m) {
std::cerr << var.first << delim << var.second << endl;
}
}
template <class T>
void DbgDisplayVectorEndl(const std::vector<T> &v, const std::string &delim=" ") {
DbgDisplayVector(v,delim);
std::cerr << std::endl;
}
void DisplayStringEndl(std::ostream & out, const std::string text);
bool CheckIfBegins(const std::string & beggining, const std::string & all);
bool CheckIfEnds (std::string const & ending, std::string const & all);
std::string SpaceFromEscape(const std::string &s);
std::string EscapeFromSpace(const std::string &s);
vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib);
char GetLastChar(const std::string & str);
std::string GetLastCharIf(const std::string & str); // TODO unicode?
std::string EscapeString(const std::string &s);
template <class T>
std::string DbgVector(const std::vector<T> &v, const std::string &delim="|") {
std::ostringstream oss;
oss << "[";
bool first=true;
for(auto vElement : v) { if (!first) oss<<delim; first=false; oss <<vElement ; }
oss << "]";
//std::copy( v.begin(), v.end(), std::ostream_iterator<T>(oss, delim.c_str()) );
return oss.str();
}
template <class T>
std::ostream & operator<<(std::ostream & os, const map< T, vector<T> > & obj){
os << "[";
for(auto const & elem : obj) {
os << " [" << elem.first << "=" << DbgVector(elem.second) << "] ";
}
os << "]";
return os;
}
template <class T, class T2>
std::string DbgMap(const map<T, T2> & map) {
std::ostringstream oss;
oss << map;
return oss.str();
}
// ====================================================================
// assert
// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no.
// Use it like this: ASRT( x>y ); with the semicolon at end, a clever trick forces this syntax :)
#define ASRT(x) do { if (!(x)) nOT::nUtils::Assert(false, OT_CODE_STAMP, #x); } while(0)
void Assert(bool result, const std::string &stamp, const std::string &condition);
// ====================================================================
// advanced string
const std::string GetMultiline(string endLine = "~");
vector<string> SplitString(const string & str);
bool checkPrefix(const string & str, char prefix = '^');
// ====================================================================
// nUse utils
enum class eSubjectType {Account, Asset, User, Server, Unknown};
string SubjectType2String(const eSubjectType & type);
eSubjectType String2SubjectType(const string & type);
// ====================================================================
// operation on files
/// @brief tools related to filesystem
/// @author rfree (maintainer)
class cFilesystemUtils { // if we do not want to use boost in given project (or we could optionally write boost here later)
public:
static bool CreateDirTree(const std::string & dir, bool only_below=false);
static char GetDirSeparatorSys(); /// < eg '/' or '\'
static char GetDirSeparatorInter(); /// < internal is '/'
static string FileInternalToSystem(const std::string &name); ///< converts from internal file name string to system file name string
static string FileSystemToInternal(const std::string &name); ///< converts from system file name string to internal file name string
};
/// @brief utils to e.g. edit a file from console
/// @author rfree (maintainer)
class cEnvUtils {
int fd;
string mFilename;
void GetTmpTextFile();
void CloseFile();
void OpenEditor();
const string ReadFromTmpFile();
public:
const string Compose();
const string ReadFromFile(const string path);
};
void hintingToTxt(std::fstream & file, string command, vector<string> &commands);
void generateQuestions (std::fstream & file, string command);
void generateAnswers (std::fstream & file, string command, vector<string> &completions);
// ====================================================================
namespace nOper { // nOT::nUtils::nOper
// cool shortcut operators, like vector + vecotr operator working same as string (appending)
// isolated to namespace because it's unorthodox ide to implement this
using namespace std;
// TODO use && and move?
template <class T>
vector<T> operator+(const vector<T> &a, const vector<T> &b) {
vector<T> ret = a;
ret.insert( ret.end() , b.begin(), b.end() );
return ret;
}
template <class T>
vector<T> operator+(const T &a, const vector<T> &b) {
vector<T> ret(1,a);
ret.insert( ret.end() , b.begin(), b.end() );
return ret;
}
template <class T>
vector<T> operator+(const vector<T> &a, const T &b) {
vector<T> b_vector(1,a);
return a + b_vector;
}
template <class T>
vector<T>& operator+=(vector<T> &a, const vector<T> &b) {
a.insert( a.end() , b.begin(), b.end() );
return a;
}
// map
template <class TK,class TV>
map<TK,TV> operator+(const map<TK,TV> &a, const map<TK,TV> &b) {
map<TK,TV> ret = a;
for (const auto & elem : b) {
ret.insert(elem);
}
return ret;
}
} // nOT::nUtils::nOper
// ====================================================================
// ====================================================================
// Algorithms
// ====================================================================
// ====================================================================
/**
@brief Special type that on creation will be initialized to have value INIT given as template argument.
Might be usefull e.g. to express in the declaration of class what will be the default value of member variable
See also http://www.boost.org/doc/libs/1_56_0/libs/utility/value_init.htm
Probably not needed when using boost in your project.
*/
template <class T, T INIT>
class value_init {
private:
T data;
public:
value_init();
T& operator=(const T& v) { data=v; return *this; }
operator T const &() const { return data; }
operator T&() { return data; }
};
template <class T, T INIT>
value_init<T, INIT>::value_init() : data(INIT) { }
} // namespace nUtils
} // namespace nOT
// global namespace
extern nOT::nUtils::cLogger gCurrentLogger; ///< The current main logger. Usually do not use it directly, instead use macros like _dbg1 etc
std::string GetObjectName(); ///< Method to return name of current object; To use in debug; Can be shadowed in your classes. (Might be not used currently)
const extern int _dbg_ignore; ///< the global _dbg_ignore, but local code (blocks, classes etc) you could shadow it in your code blocks,
// to override debug compile-time setting for given block/class, e.g. to disable debug in one of your methods or increase it there.
// Or to make it runtime by providing a class normal member and editing it in runtime
#define OT_CODE_STAMP ( nOT::nUtils::ToStr("[") + nOT::nUtils::nDetail::DbgShortenCodeFileName(__FILE__) + nOT::nUtils::ToStr("+") + nOT::nUtils::ToStr(__LINE__) + nOT::nUtils::ToStr(" ") + (GetObjectName()) + nOT::nUtils::ToStr("::") + nOT::nUtils::ToStr(__FUNCTION__) + nOT::nUtils::ToStr("]"))
#endif

64
contrib/otshell_utils/windows_stream.cpp Normal file
View file

@ -0,0 +1,64 @@
#if defined(_WIN32)
#include "windows_stream.h"
#include <windows.h>
windows_stream::windows_stream(unsigned int pLevel)
:
mLevel(pLevel)
{
}
std::ostream& operator << (std::ostream &stream, windows_stream const& object)
{
HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
if (object.mLevel >= 100)
{
SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_RED | BACKGROUND_INTENSITY);
return stream;
}
if (object.mLevel >= 90)
{
SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_GREEN | BACKGROUND_RED | BACKGROUND_INTENSITY);
return stream;
}
if (object.mLevel >= 80)
{
SetConsoleTextAttribute(h_stdout, BACKGROUND_BLUE | BACKGROUND_RED | BACKGROUND_INTENSITY);
return stream;
}
if (object.mLevel >= 75)
{
SetConsoleTextAttribute(h_stdout, BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_INTENSITY);
return stream;
}
if (object.mLevel >= 70)
{
SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_INTENSITY);
return stream;
}
if (object.mLevel >= 50)
{
SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY);
return stream;
}
if (object.mLevel >= 40)
{
SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_RED | FOREGROUND_INTENSITY);
return stream;
}
if (object.mLevel >= 30)
{
SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_INTENSITY);
return stream;
}
if (object.mLevel >= 20)
{
SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE);
return stream;
}
return stream;
}
#endif

20
contrib/otshell_utils/windows_stream.h Normal file
View file

@ -0,0 +1,20 @@
#ifndef WINDOWS_STREAM_H
#define WINDOWS_STREAM_H
#if defined(_WIN32)
#include <string>
#include <iostream>
class windows_stream
{
public:
windows_stream(unsigned int pLevel);
friend std::ostream& operator<<(std::ostream &stream, windows_stream const& object);
private:
unsigned int mLevel = 0;
};
#endif // _WIN32
#endif // WINDOWS_STREAM_H

2
src/CMakeLists.txt
View file

@ -92,6 +92,8 @@ add_subdirectory(cryptonote_core)
add_subdirectory(mnemonics)
add_subdirectory(rpc)
add_subdirectory(wallet)
add_subdirectory(p2p)
add_subdirectory(cryptonote_protocol)
add_subdirectory(connectivity_tool)
add_subdirectory(miner)

77
src/Changelog Normal file
View file

@ -0,0 +1,77 @@
[F] - fixes a bug from existing official version
[B] - important bug discovered in official version
[T] - testes
[n] - not used now (only in history), replaced or not accepted to master
- adding support for user-local installed compiler (e.g. non-root users on older systems to use new compiler)
instructions to use monero on Debian 7 (build clang + deps)
- [n] faster build: cmake with fast build, and with cotire (precompiled headers), and limited targets
- fixed cmake local compiler e.g. BOOST_IGNORE_SYSTEM_PATHS after merges with the other new CMake
- faster rebuild: separate out not-template base for main network classes: connection_basic.cpp class connection_basic
- faster rebuild: separate out hooks: connection_basic::do_send_handler_write() do_send_handler_write_from_queue()
- created library/cmake for the new no-templates-only networking code - libp2p
- added command exit without DB save (only for testers)
- added option --no-igd to not wait for IGD on start (faster testing)
- imported logging/debug tools from otshell, with macros like _note() _info() _warn()
- logging: added proper thread locking; showing thread number {1} in the output
- logging: also showing process number (PID) in short form (1,2,3,...) to debug forking daemon etc
- logging: fixed colors for normal windows text console (currently they do not work in msys windows text console)
- logging: added channels to have separate files created
- logging: option (compile-time) to debug the logging system itself
- logging: console colors work on windows/msys
- created network throttle class + throttle manager
- cmdline options for network throttle class (limit-rate-up limit-rate-down limit-rate)
- option and command to in fact limit (outgoing) peers --out-peers out_peers [currently not working! after merge] [TODO]
- rpc commands for network throttle class (limit limit_up limit_down)
- setting ToS socket flag with option --tos-flag
- connection type support, so that RPC connection is excluded from network limits
- gather and save throughput statistics (speed vs limit) into data file
- statistics of details to tune implementation: sleep in network threads, number of peers
- optimized statistics: accumulate sum and limit disk writes
(dr. monero code is not published in this commit, but prepared)
- dr. monero show the collected statistics (in real time)
- dr. monero many windows opened at once from predefined list
- dr. monero auto scale; selectable average window
- dr. monero colors, show both samples and average values (transparent)
- dr. monero showing current configured network limit (goal)
- dr. monero show date/git commit/comments for better info in screenshots
- dr. monero optimize speed and memory usage to handle files from long tests
- [F] found few UBs in code like wrong initialization order
- [B] found and partial debugged deadlocks on existing program (faster testing)
- [B] found locking problem with connection close race
- debug option --test-drop-download to test no-DB version without running out of RAM
- also option --test-drop-download-heigh to start drop only after certain height
- added again fast exit command (but it does not work too well yet. ONLY FOR DEVELOPERS!)
- created new from-scratch clean network engine model (a new program/library)
- connected the logging, the network throttles and all code to network model
- [T] run experiments in fully controlled and very fast environment
- fine tuned parameters of throttles (sleep, 3 window sizes)
- code to estimated current network-transfer size of block based get_avg_block_size()
- [T] tested sleep vs request-size limiting of downloads (request less or more then 200 blocks)
- for now we use sleep method
- documented throttle classes and few other classes
- adjusted comment headers, copyrights, added the @monero tag to doxygen denoting monero maintainer/contact for not-monero code
- doxygen scripts fixed, tested and generated actual website
- [n] rebased most of above when change to branch development was decided
- [n] again rebase when we returned to developing against branch master
- finall version was tested by developer on Windows 64-bit, Debian, Ubuntu
- fixed clock compilation problems for freebsd and mac os x; Fixed related problem again on mac os x.

3
src/common/command_line.cpp
View file

@ -48,4 +48,7 @@ namespace command_line
const arg_descriptor<bool> arg_version = {"version", "Output version information"};
const arg_descriptor<std::string> arg_data_dir = {"data-dir", "Specify data directory"};
const arg_descriptor<std::string> arg_testnet_data_dir = {"testnet-data-dir", "Specify testnet data directory"};
const arg_descriptor<bool> arg_test_drop_download = {"test-drop-download", "For net tests: in download, discard ALL blocks instead checking/saving them (very fast)"};
const arg_descriptor<uint64_t> arg_test_drop_download_height = {"test-drop-download-height", "Like test-drop-download but disards only after around certain height", 0};
const arg_descriptor<int> arg_test_dbg_lock_sleep = {"test-dbg-lock-sleep", "Sleep time in ms, defaults to 0 (off), used to debug before/after locking mutex. Values 100 to 1000 are good for tests."};
}

3
src/common/command_line.h
View file

@ -204,4 +204,7 @@ namespace command_line
extern const arg_descriptor<bool> arg_version;
extern const arg_descriptor<std::string> arg_data_dir;
extern const arg_descriptor<std::string> arg_testnet_data_dir;
extern const arg_descriptor<bool> arg_test_drop_download;
extern const arg_descriptor<uint64_t> arg_test_drop_download_height;
extern const arg_descriptor<int> arg_test_dbg_lock_sleep;
}

17
src/common/dns_utils.cpp
View file

@ -166,7 +166,24 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData())
ub_ctx_resolvconf(m_data->m_ub_context, NULL);
ub_ctx_hosts(m_data->m_ub_context, NULL);
#ifdef DEVELOPER_LIBUNBOUND_OLD
#warning "Using the work around for old libunbound"
{ // work around for bug https://www.nlnetlabs.nl/bugs-script/show_bug.cgi?id=515 needed for it to compile on e.g. Debian 7
char * ds_copy = NULL; // this will be the writable copy of string that bugged version of libunbound requires
try {
char * ds_copy = strdup( ::get_builtin_ds() );
ub_ctx_add_ta(m_data->m_ub_context, ds_copy);
} catch(...) { // probably not needed but to work correctly in every case...
if (ds_copy) { free(ds_copy); ds_copy=NULL; } // for the strdup
throw ;
}
if (ds_copy) { free(ds_copy); ds_copy=NULL; } // for the strdup
}
#else
// normal version for fixed libunbound
ub_ctx_add_ta(m_data->m_ub_context, ::get_builtin_ds() );
#endif
}
DNSResolver::~DNSResolver()

2
src/connectivity_tool/conn_tool.cpp
View file

@ -49,6 +49,8 @@ namespace po = boost::program_options;
using namespace cryptonote;
using namespace nodetool;
unsigned int epee::g_test_dbg_lock_sleep = 0;
namespace
{
const command_line::arg_descriptor<std::string, true> arg_ip = {"ip", "set ip"};

1
src/cryptonote_core/CMakeLists.txt
View file

@ -70,6 +70,7 @@ target_link_libraries(cryptonote_core
LINK_PUBLIC
common
crypto
otshell_utils
${Boost_DATE_TIME_LIBRARY}
${Boost_PROGRAM_OPTIONS_LIBRARY}
${Boost_SERIALIZATION_LIBRARY}

29
src/cryptonote_core/blockchain_storage.cpp
View file

@ -49,6 +49,8 @@
#include "crypto/hash.h"
#include "cryptonote_core/checkpoints_create.h"
//#include "serialization/json_archive.h"
#include "../../contrib/otshell_utils/utils.hpp"
#include "../../src/p2p/data_logger.hpp"
using namespace cryptonote;
@ -1154,6 +1156,31 @@ uint64_t blockchain_storage::block_difficulty(size_t i)
return m_blocks[i].cumulative_difficulty - m_blocks[i-1].cumulative_difficulty;
}
//------------------------------------------------------------------
double blockchain_storage::get_avg_block_size( size_t count)
{
if (count > get_current_blockchain_height()) return 500;
double average = 0;
_dbg1_c("net/blksize", "HEIGHT: " << get_current_blockchain_height());
_dbg1_c("net/blksize", "BLOCK ID BY HEIGHT: " << get_block_id_by_height(get_current_blockchain_height()) );
_dbg1_c("net/blksize", "BLOCK TAIL ID: " << get_tail_id() );
std::vector<size_t> size_vector;
get_backward_blocks_sizes(get_current_blockchain_height() - count, size_vector, count);
std::vector<size_t>::iterator it;
it = size_vector.begin();
while (it != size_vector.end()) {
average += *it;
_dbg2_c("net/blksize", "VECTOR ELEMENT: " << (*it) );
it++;
}
average = average / count;
_dbg1_c("net/blksize", "VECTOR SIZE: " << size_vector.size() << " average=" << average);
return average;
}
//------------------------------------------------------------------
void blockchain_storage::print_blockchain(uint64_t start_index, uint64_t end_index)
{
std::stringstream ss;
@ -1745,6 +1772,8 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt
<< "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size
<< ", " << block_processing_time << "("<< target_calculating_time << "/" << longhash_calculating_time << ")ms");
epee::net_utils::data_logger::get_instance().add_data("blockchain_processing_time", block_processing_time);
bvc.m_added_to_main_chain = true;
/*if(!m_orphanes_reorganize_in_work)
review_orphaned_blocks_with_new_block_id(id, true);*/

1
src/cryptonote_core/blockchain_storage.h
View file

@ -134,6 +134,7 @@ namespace cryptonote
uint64_t get_current_comulative_blocksize_limit();
bool is_storing_blockchain(){return m_is_blockchain_storing;}
uint64_t block_difficulty(size_t i);
double get_avg_block_size( size_t count);
template<class t_ids_container, class t_blocks_container, class t_missed_container>
bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs)

46
src/cryptonote_core/cryptonote_core.cpp
View file

@ -145,6 +145,11 @@ namespace cryptonote
set_enforce_dns_checkpoints(command_line::get_arg(vm, daemon_args::arg_dns_checkpoints));
test_drop_download_height(command_line::get_arg(vm, command_line::arg_test_drop_download_height));
if (command_line::get_arg(vm, command_line::arg_test_drop_download) == true)
test_drop_download();
return true;
}
//-----------------------------------------------------------------------------------------------
@ -218,9 +223,48 @@ namespace cryptonote
{
m_miner.stop();
m_mempool.deinit();
if (!m_fast_exit)
{
m_blockchain_storage.deinit();
}
return true;
}
//-----------------------------------------------------------------------------------------------
void core::set_fast_exit()
{
m_fast_exit = true;
}
//-----------------------------------------------------------------------------------------------
bool core::get_fast_exit()
{
return m_fast_exit;
}
//-----------------------------------------------------------------------------------------------
void core::test_drop_download()
{
m_test_drop_download = false;
}
//-----------------------------------------------------------------------------------------------
void core::test_drop_download_height(uint64_t height)
{
m_test_drop_download_height = height;
}
//-----------------------------------------------------------------------------------------------
bool core::get_test_drop_download()
{
return m_test_drop_download;
}
//-----------------------------------------------------------------------------------------------
bool core::get_test_drop_download_height()
{
if (m_test_drop_download_height == 0)
return true;
if (get_blockchain_storage().get_current_blockchain_height() <= m_test_drop_download_height)
return true;
return false;
}
//-----------------------------------------------------------------------------------------------
bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block)
{
@ -624,4 +668,6 @@ namespace cryptonote
{
raise(SIGTERM);
}
std::atomic<bool> core::m_fast_exit(false);
}

10
src/cryptonote_core/cryptonote_core.h
View file

@ -75,6 +75,12 @@ namespace cryptonote
bool init(const boost::program_options::variables_map& vm);
bool set_genesis_block(const block& b);
bool deinit();
static void set_fast_exit();
static bool get_fast_exit();
void test_drop_download();
void test_drop_download_height(uint64_t height);
bool get_test_drop_download();
bool get_test_drop_download_height();
uint64_t get_current_blockchain_height();
bool get_blockchain_top(uint64_t& heeight, crypto::hash& top_id);
bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs);
@ -148,7 +154,9 @@ namespace cryptonote
bool on_update_blocktemplate_interval();
bool check_tx_inputs_keyimages_diff(const transaction& tx);
void graceful_exit();
static std::atomic<bool> m_fast_exit;
bool m_test_drop_download = true;
uint64_t m_test_drop_download_height = 0;
tx_memory_pool m_mempool;
blockchain_storage m_blockchain_storage;

46
src/cryptonote_protocol/CMakeLists.txt Normal file
View file

@ -0,0 +1,46 @@
# Copyright (c) 2014, The Monero Project
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are
# permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
# of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cmake_minimum_required (VERSION 2.6)
project (bitmonero CXX)
file(GLOB CRYPTONOTE_PROTOCOL *)
source_group(cryptonote_protocol FILES ${CRYPTONOTE_PROTOCOL})
#add_library(p2p ${P2P})
#bitmonero_private_headers(p2p ${CRYPTONOTE_PROTOCOL})
bitmonero_add_library(cryptonote_protocol ${CRYPTONOTE_PROTOCOL})
#target_link_libraries(p2p)
# LINK_PRIVATE
# ${Boost_CHRONO_LIBRARY}
# ${Boost_REGEX_LIBRARY}
# ${Boost_SYSTEM_LIBRARY}
# ${Boost_THREAD_LIBRARY}
# ${EXTRA_LIBRARIES})
add_dependencies(cryptonote_protocol
version)

14
src/cryptonote_protocol/cryptonote_protocol_defs.h
View file

@ -46,6 +46,8 @@ namespace cryptonote
struct connection_info
{
bool incoming;
bool localhost;
bool local_ip;
std::string ip;
std::string port;
@ -62,8 +64,16 @@ namespace cryptonote
uint64_t live_time;
uint64_t avg_download;
uint64_t current_download;
uint64_t avg_upload;
uint64_t current_upload;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(incoming)
KV_SERIALIZE(localhost)
KV_SERIALIZE(local_ip)
KV_SERIALIZE(ip)
KV_SERIALIZE(port)
KV_SERIALIZE(peer_id)
@ -73,6 +83,10 @@ namespace cryptonote
KV_SERIALIZE(send_idle_time)
KV_SERIALIZE(state)
KV_SERIALIZE(live_time)
KV_SERIALIZE(avg_download)
KV_SERIALIZE(current_download)
KV_SERIALIZE(avg_upload)
KV_SERIALIZE(current_upload)
END_KV_SERIALIZE_MAP()
};

176
src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp Normal file
View file

@ -0,0 +1,176 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief This is the place to implement our handlers for protocol network actions, e.g. for ratelimit for download-requests
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <boost/asio.hpp>
#include <string>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/interprocess/detail/atomic.hpp>
#include <boost/thread/thread.hpp>
#include <memory>
#include "syncobj.h"
#include "../../contrib/epee/include/net/net_utils_base.h"
#include "../../contrib/epee/include/misc_log_ex.h"
#include <boost/lambda/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/lambda/lambda.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/chrono.hpp>
#include <boost/utility/value_init.hpp>
#include <boost/asio/deadline_timer.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/thread/thread.hpp>
#include "misc_language.h"
#include "pragma_comp_defs.h"
#include <sstream>
#include <iomanip>
#include <algorithm>
#include <boost/asio/basic_socket.hpp>
#include <boost/asio/ip/unicast.hpp>
#include "../../src/cryptonote_protocol/cryptonote_protocol_handler.h"
#include "../../src/p2p/network_throttle.hpp"
#include "../../contrib/otshell_utils/utils.hpp"
using namespace nOT::nUtils;
#include "../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal()
// ################################################################################################
// ################################################################################################
// the "header part". Not separeted out for .hpp because point of this modification is
// to rebuild just 1 translation unit while working on this code.
// (But maybe common parts will be separated out later though - if needed)
// ################################################################################################
// ################################################################################################
namespace cryptonote {
class cryptonote_protocol_handler_base_pimpl { // placeholer if needed
public:
};
} // namespace
// ################################################################################################
// ################################################################################################
// ################################################################################################
// ################################################################################################
namespace cryptonote {
double cryptonote_protocol_handler_base::estimate_one_block_size() noexcept { // for estimating size of blocks to downloa
const double size_min = 500; // XXX 500
//const int history_len = 20; // how many blocks to average over
double avg=0;
try {
avg = get_avg_block_size(/*history_len*/);
} catch (...) { }
avg = std::max( size_min , avg);
return avg;
}
cryptonote_protocol_handler_base::cryptonote_protocol_handler_base() {
}
cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() {
}
void cryptonote_protocol_handler_base::handler_request_blocks_history(std::list<crypto::hash>& ids) {
using namespace epee::net_utils;
LOG_PRINT_L0("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size());
LOG_PRINT_RED("RATE LIMIT NOT IMPLEMENTED HERE YET (download at unlimited speed?)" , LOG_LEVEL_0);
_note_c("net/req2", "### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size());
// TODO
}
void cryptonote_protocol_handler_base::handler_response_blocks_now(size_t packet_size) { _scope_dbg1("");
using namespace epee::net_utils;
double delay=0; // will be calculated
_dbg1("Packet size: " << packet_size);
do
{ // rate limiting
//XXX
/*if (::cryptonote::core::get_is_stopping()) {
_dbg1("We are stopping - so abort sleep");
return;
}*/
/*if (m_was_shutdown) {
_dbg2_c("net/netuse/sleep","m_was_shutdown - so abort sleep");
return;
}*/
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global
}
delay *= 0.50;
//delay = 0; // XXX
if (delay > 0) {
//delay += rand2*0.1;
long int ms = (long int)(delay * 1000);
_info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<packet_size); // XXX debug sleep
_dbg1_c("net/sleep/", "sleep in sleep_before_packet");
_dbg2("Sleep for " << ms);
boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); // TODO randomize sleeps
}
} while(delay > 0);
// XXX LATER XXX
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
network_throttle_manager::get_global_throttle_out().handle_trafic_tcp( packet_size ); // increase counter - global
//epee::critical_region_t<decltype(m_throttle_global_lock)> guard(m_throttle_global_lock); // *** critical ***
//m_throttle_global.m_out.handle_trafic_tcp( packet_size ); // increase counter - global
}
}
} // namespace

39
src/cryptonote_protocol/cryptonote_protocol_handler.h
View file

@ -1,3 +1,7 @@
/// @file
/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote)
/// @brief This is the orginal cryptonote protocol network-events handler, modified by us
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
@ -41,15 +45,36 @@
#include "cryptonote_core/connection_context.h"
#include "cryptonote_core/cryptonote_stat_info.h"
#include "cryptonote_core/verification_context.h"
// #include <netinet/in.h>
#include <boost/circular_buffer.hpp>
PUSH_WARNINGS
DISABLE_VS_WARNINGS(4355)
#define LOCALHOST_INT 2130706433
namespace cryptonote
{
class cryptonote_protocol_handler_base_pimpl;
class cryptonote_protocol_handler_base {
private:
std::unique_ptr<cryptonote_protocol_handler_base_pimpl> mI;
public:
cryptonote_protocol_handler_base();
virtual ~cryptonote_protocol_handler_base();
void handler_request_blocks_history(std::list<crypto::hash>& ids); // before asking for list of objects, we can change the list still
void handler_response_blocks_now(size_t packet_size);
virtual double get_avg_block_size() = 0;
virtual double estimate_one_block_size() noexcept; // for estimating size of blocks to download
virtual std::ofstream& get_logreq() const =0;
};
template<class t_core>
class t_cryptonote_protocol_handler: public i_cryptonote_protocol
class t_cryptonote_protocol_handler: public i_cryptonote_protocol, cryptonote_protocol_handler_base
{
public:
typedef cryptonote_connection_context connection_context;
@ -106,6 +131,12 @@ namespace cryptonote
nodetool::i_p2p_endpoint<connection_context>* m_p2p;
std::atomic<uint32_t> m_syncronized_connections_count;
std::atomic<bool> m_synchronized;
bool m_one_request = true;
// static std::ofstream m_logreq;
std::mutex m_buffer_mutex;
double get_avg_block_size();
boost::circular_buffer<size_t> m_avg_buffer = boost::circular_buffer<size_t>(10);
template<class t_parametr>
bool post_notify(typename t_parametr::request& arg, cryptonote_connection_context& context)
@ -113,6 +144,7 @@ namespace cryptonote
LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parametr).name() << " -->");
std::string blob;
epee::serialization::store_t_to_binary(arg, blob);
//handler_response_blocks_now(blob.size()); // XXX
return m_p2p->invoke_notify_to_peer(t_parametr::ID, blob, context);
}
@ -124,8 +156,11 @@ namespace cryptonote
epee::serialization::store_t_to_binary(arg, arg_buff);
return m_p2p->relay_notify_to_all(t_parametr::ID, arg_buff, exlude_context);
}
virtual std::ofstream& get_logreq() const ;
};
}
} // namespace
#include "cryptonote_protocol_handler.inl"

228
src/cryptonote_protocol/cryptonote_protocol_handler.inl
View file

@ -1,3 +1,7 @@
/// @file
/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote)
/// @brief This is the orginal cryptonote protocol network-events handler, modified by us
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
@ -28,14 +32,28 @@
//
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
// (may contain code and/or modifications by other developers)
// developer rfree: this code is caller of our new network code, and is modded; e.g. for rate limiting
#include <boost/interprocess/detail/atomic.hpp>
#include <list>
#include "cryptonote_core/cryptonote_format_utils.h"
#include "profile_tools.h"
#include "../../contrib/otshell_utils/utils.hpp"
#include "../../src/p2p/network_throttle-detail.hpp"
#include "../../src/p2p/data_logger.hpp"
using namespace nOT::nUtils;
namespace cryptonote
{
// static
// template<class t_core> std::ofstream t_cryptonote_protocol_handler<t_core>::m_logreq("logreq.txt"); // static
//-----------------------------------------------------------------------------------------------------------------------
template<class t_core>
t_cryptonote_protocol_handler<t_core>::t_cryptonote_protocol_handler(t_core& rcore, nodetool::i_p2p_endpoint<connection_context>* p_net_layout):m_core(rcore),
@ -99,23 +117,65 @@ namespace cryptonote
void t_cryptonote_protocol_handler<t_core>::log_connections()
{
std::stringstream ss;
ss.precision(1);
ss << std::setw(25) << std::left << "Remote Host"
double down_sum = 0.0;
double down_curr_sum = 0.0;
double up_sum = 0.0;
double up_curr_sum = 0.0;
ss << std::setw(30) << std::left << "Remote Host"
<< std::setw(20) << "Peer id"
<< std::setw(25) << "Recv/Sent (inactive,sec)"
<< std::setw(30) << "Recv/Sent (inactive,sec)"
<< std::setw(25) << "State"
<< std::setw(20) << "Livetime(seconds)" << ENDL;
<< std::setw(20) << "Livetime(sec)"
<< std::setw(12) << "Down (kB/s)"
<< std::setw(14) << "Down(now)"
<< std::setw(10) << "Up (kB/s)"
<< std::setw(13) << "Up(now)"
<< ENDL;
uint32_t ip;
m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id)
{
ss << std::setw(25) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") +
bool local_ip = false;
ip = ntohl(cntxt.m_remote_ip);
// TODO: local ip in calss A, B
if (ip > 3232235520 && ip < 3232301055) // 192.168.x.x
local_ip = true;
auto connection_time = time(NULL) - cntxt.m_started;
ss << std::setw(30) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") +
epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port)
<< std::setw(20) << std::hex << peer_id
<< std::setw(25) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")"
<< std::setw(30) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")"
<< std::setw(25) << get_protocol_state_string(cntxt.m_state)
<< std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) << ENDL;
<< std::setw(20) << std::to_string(time(NULL) - cntxt.m_started)
<< std::setw(12) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_recv_cnt / connection_time / 1024)
<< std::setw(14) << std::fixed << cntxt.m_current_speed_down / 1024
<< std::setw(10) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_send_cnt / connection_time / 1024)
<< std::setw(13) << std::fixed << cntxt.m_current_speed_up / 1024
<< (local_ip ? "[LAN]" : "")
<< std::left << (ip == LOCALHOST_INT ? "[LOCALHOST]" : "") // 127.0.0.1
<< ENDL;
if (connection_time > 1)
{
down_sum += (cntxt.m_recv_cnt / connection_time / 1024);
up_sum += (cntxt.m_send_cnt / connection_time / 1024);
}
down_curr_sum += (cntxt.m_current_speed_down / 1024);
up_curr_sum += (cntxt.m_current_speed_up / 1024);
return true;
});
ss << ENDL
<< std::setw(125) << " "
<< std::setw(12) << down_sum
<< std::setw(14) << down_curr_sum
<< std::setw(10) << up_sum
<< std::setw(13) << up_curr_sum
<< ENDL;
LOG_PRINT_L0("Connections: " << ENDL << ss.str());
}
//------------------------------------------------------------------------------------------------------------------------
@ -150,6 +210,42 @@ namespace cryptonote
cnx.live_time = timestamp - cntxt.m_started;
uint32_t ip;
ip = ntohl(cntxt.m_remote_ip);
if (ip == LOCALHOST_INT)
{
cnx.localhost = true;
}
else
{
cnx.localhost = false;
}
if (ip > 3232235520 && ip < 3232301055) // 192.168.x.x
{
cnx.local_ip = true;
}
else
{
cnx.local_ip = false;
}
auto connection_time = time(NULL) - cntxt.m_started;
if (connection_time == 0)
{
cnx.avg_download = 0;
cnx.avg_upload = 0;
}
else
{
cnx.avg_download = cntxt.m_recv_cnt / connection_time / 1024;
cnx.avg_upload = cntxt.m_send_cnt / connection_time / 1024;
}
cnx.current_download = cntxt.m_current_speed_down / 1024;
cnx.current_upload = cntxt.m_current_speed_up / 1024;
connections.push_back(cnx);
return true;
@ -234,11 +330,11 @@ namespace cryptonote
block_verification_context bvc = boost::value_initialized<block_verification_context>();
m_core.pause_mine();
m_core.handle_incoming_block(arg.b.block, bvc);
m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block
m_core.resume_mine();
if(bvc.m_verifivation_failed)
{
LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection");
LOG_PRINT_CCONTEXT_L0("Block verification failed, dropping connection");
m_p2p->drop_connection(context);
return 1;
}
@ -304,13 +400,70 @@ namespace cryptonote
LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size()
<< ", rsp.m_current_blockchain_height=" << rsp.current_blockchain_height << ", missed_ids.size()=" << rsp.missed_ids.size());
post_notify<NOTIFY_RESPONSE_GET_OBJECTS>(rsp, context);
//handler_response_blocks_now(sizeof(rsp)); // XXX
//handler_response_blocks_now(200);
return 1;
}
//------------------------------------------------------------------------------------------------------------------------
template<class t_core>
double t_cryptonote_protocol_handler<t_core>::get_avg_block_size() {
// return m_core.get_blockchain_storage().get_avg_block_size(count); // this does not count too well the actuall network-size of data we need to download
CRITICAL_REGION_LOCAL(m_buffer_mutex);
double avg = 0;
if (m_avg_buffer.size() == 0) {
_warn("m_avg_buffer.size() == 0");
return 500;
}
const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option
long int dbg_repeat=0;
do {
for (auto element : m_avg_buffer) avg += element;
} while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one
return avg / m_avg_buffer.size();
}
template<class t_core>
int t_cryptonote_protocol_handler<t_core>::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context)
{
LOG_PRINT_CCONTEXT_L2("NOTIFY_RESPONSE_GET_OBJECTS");
// calculate size of request - mainly for logging/debug
size_t size = 0;
for (auto element : arg.txs) size += element.size();
for (auto element : arg.blocks) {
size += element.block.size();
for (auto tx : element.txs)
size += tx.size();
}
for (auto element : arg.missed_ids)
size += sizeof(element.data);
size += sizeof(arg.current_blockchain_height);
{
CRITICAL_REGION_LOCAL(m_buffer_mutex);
m_avg_buffer.push_back(size);
const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option
long int dbg_repeat=0;
do {
m_avg_buffer.push_back(666); // a test value
m_avg_buffer.erase_end(1);
} while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one
}
/*using namespace boost::chrono;
auto point = steady_clock::now();
auto time_from_epoh = point.time_since_epoch();
auto sec = duration_cast< seconds >( time_from_epoh ).count();*/
//epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
if(context.m_last_response_height > arg.current_blockchain_height)
{
LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height
@ -373,11 +526,17 @@ namespace cryptonote
return 1;
}
{
m_core.pause_mine();
epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler(
boost::bind(&t_core::resume_mine, &m_core));
LOG_PRINT_CCONTEXT_YELLOW( "Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() , LOG_LEVEL_0);
if (m_core.get_test_drop_download() && m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing
BOOST_FOREACH(const block_complete_entry& block_entry, arg.blocks)
{
// process transactions
@ -397,10 +556,11 @@ namespace cryptonote
TIME_MEASURE_FINISH(transactions_process_time);
// process block
TIME_MEASURE_START(block_process_time);
block_verification_context bvc = boost::value_initialized<block_verification_context>();
m_core.handle_incoming_block(block_entry.block, bvc, false);
m_core.handle_incoming_block(block_entry.block, bvc, false); // <--- process block
if(bvc.m_verifivation_failed)
{
@ -417,9 +577,16 @@ namespace cryptonote
TIME_MEASURE_FINISH(block_process_time);
LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms");
}
}
epee::net_utils::data_logger::get_instance().add_data("calc_time", block_process_time + transactions_process_time);
epee::net_utils::data_logger::get_instance().add_data("block_processing", 1);
} // each download block
} // if not DISCARD BLOCK
}
request_missing_objects(context, true);
return 1;
}
@ -448,6 +615,15 @@ namespace cryptonote
template<class t_core>
bool t_cryptonote_protocol_handler<t_core>::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks)
{
//if (!m_one_request == false)
//return true;
m_one_request = false;
// save request size to log (dr monero)
/*using namespace boost::chrono;
auto point = steady_clock::now();
auto time_from_epoh = point.time_since_epoch();
auto sec = duration_cast< seconds >( time_from_epoh ).count();*/
if(context.m_needed_objects.size())
{
//we know objects that we need, request this objects
@ -455,6 +631,8 @@ namespace cryptonote
size_t count = 0;
auto it = context.m_needed_objects.begin();
size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
_note_c("net/req-calc" , "Setting count_limit: " << count_limit);
while(it != context.m_needed_objects.end() && count < BLOCKS_SYNCHRONIZING_DEFAULT_COUNT)
{
if( !(check_having_blocks && m_core.have_block(*it)))
@ -465,14 +643,24 @@ namespace cryptonote
}
context.m_needed_objects.erase(it++);
}
LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size());
LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size()
<< "requested blocks count=" << count << " / " << count_limit);
//epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context);
}else if(context.m_last_response_height < context.m_remote_blockchain_height-1)
{//we have to fetch more objects ids, request blockchain entry
NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>();
m_core.get_short_chain_history(r.block_ids);
LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
//std::string blob; // for calculate size of request
//epee::serialization::store_t_to_binary(r, blob);
//epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
LOG_PRINT_CCONTEXT_L0("r = " << 200);
LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
}else
{
@ -575,4 +763,18 @@ namespace cryptonote
{
return relay_post_notify<NOTIFY_NEW_TRANSACTIONS>(arg, exclude_context);
}
/// @deprecated
template<class t_core> std::ofstream& t_cryptonote_protocol_handler<t_core>::get_logreq() const {
static std::ofstream * logreq=NULL;
if (!logreq) {
LOG_PRINT_RED("LOG OPENED",LOG_LEVEL_0);
logreq = new std::ofstream("logreq.txt"); // leak mem (singleton)
*logreq << "Opened log" << std::endl;
}
LOG_PRINT_YELLOW("LOG USED",LOG_LEVEL_0);
(*logreq) << "log used" << std::endl;
return *logreq;
}
} // namespace

3
src/daemon/CMakeLists.txt
View file

@ -77,6 +77,9 @@ target_link_libraries(daemon
cryptonote_core
crypto
common
otshell_utils
p2p
cryptonote_protocol
daemonizer
${Boost_CHRONO_LIBRARY}
${Boost_FILESYSTEM_LIBRARY}

37
src/daemon/command_parser_executor.cpp
View file

@ -296,4 +296,41 @@ bool t_command_parser_executor::set_limit_down(const std::vector<std::string>& a
return m_executor.set_limit_down(limit);
}
bool t_command_parser_executor::fast_exit(const std::vector<std::string>& args)
{
if (!args.empty()) return false;
return m_executor.fast_exit();
}
bool t_command_parser_executor::out_peers(const std::vector<std::string>& args)
{
if (args.empty()) return false;
unsigned int limit;
try {
limit = std::stoi(args[0]);
}
catch(std::invalid_argument& ex) {
_erro("stoi exception");
return false;
}
return m_executor.out_peers(limit);
}
bool t_command_parser_executor::start_save_graph(const std::vector<std::string>& args)
{
if (!args.empty()) return false;
return m_executor.start_save_graph();
}
bool t_command_parser_executor::stop_save_graph(const std::vector<std::string>& args)
{
if (!args.empty()) return false;
return m_executor.stop_save_graph();
}
} // namespace daemonize

7
src/daemon/command_parser_executor.h
View file

@ -93,6 +93,13 @@ public:
bool set_limit_down(const std::vector<std::string>& args);
bool fast_exit(const std::vector<std::string>& args);
bool out_peers(const std::vector<std::string>& args);
bool start_save_graph(const std::vector<std::string>& args);
bool stop_save_graph(const std::vector<std::string>& args);
};
} // namespace daemonize

24
src/daemon/command_server.cpp
View file

@ -150,15 +150,35 @@ t_command_server::t_command_server(
, "limit <kB/s> - Set download and upload limit"
);
m_command_lookup.set_handler(
"limit-up"
"limit_up"
, std::bind(&t_command_parser_executor::set_limit_up, &m_parser, p::_1)
, "limit <kB/s> - Set upload limit"
);
m_command_lookup.set_handler(
"limit-down"
"limit_down"
, std::bind(&t_command_parser_executor::set_limit_down, &m_parser, p::_1)
, "limit <kB/s> - Set download limit"
);
m_command_lookup.set_handler(
"fast_exit"
, std::bind(&t_command_parser_executor::fast_exit, &m_parser, p::_1)
, "Exit"
);
m_command_lookup.set_handler(
"out_peers"
, std::bind(&t_command_parser_executor::out_peers, &m_parser, p::_1)
, "Set max limit of out peers"
);
m_command_lookup.set_handler(
"start_save_graph"
, std::bind(&t_command_parser_executor::start_save_graph, &m_parser, p::_1)
, "Start save data for dr monero"
);
m_command_lookup.set_handler(
"stop_save_graph"
, std::bind(&t_command_parser_executor::stop_save_graph, &m_parser, p::_1)
, "Stop save data for dr monero"
);
}
bool t_command_server::process_command_str(const std::string& cmd)

8
src/daemon/daemon.cpp
View file

@ -1,4 +1,4 @@
// Copyright (c) 2014, The Monero Project
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
@ -38,10 +38,16 @@
#include "daemon/command_server.h"
#include "misc_log_ex.h"
#include "version.h"
#include "../../contrib/epee/include/syncobj.h"
using namespace epee;
#include <boost/program_options.hpp>
#include <functional>
#include <memory>
unsigned int epee::g_test_dbg_lock_sleep = 0;
namespace daemonize {
struct t_internals {

47
src/daemon/daemon_commands_handler.h
View file

@ -1,7 +1,35 @@
// Copyright (c) 2012-2013 The Cryptonote developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
/* This isn't a header file, may want to refactor this... */
#pragma once
#include <boost/lexical_cast.hpp>
@ -12,6 +40,7 @@
#include "common/util.h"
#include "crypto/hash.h"
#include "version.h"
#include "../../contrib/otshell_utils/utils.hpp"
//#include "net/net_helper.h"
//#include "../p2p/p2p_protocol_defs.h"
@ -43,10 +72,14 @@ public:
m_cmd_binder.set_handler("save", boost::bind(&daemon_cmmands_handler::save, this, _1), "Save blockchain");
m_cmd_binder.set_handler("set_log", boost::bind(&daemon_cmmands_handler::set_log, this, _1), "set_log <level> - Change current log detalization level, <level> is a number 0-4");
m_cmd_binder.set_handler("diff", boost::bind(&daemon_cmmands_handler::diff, this, _1), "Show difficulty");
m_cmd_binder.set_handler("limit-up", boost::bind(&daemon_cmmands_handler::limit_up, this, _1), "Set upload limit");
m_cmd_binder.set_handler("limit-down", boost::bind(&daemon_cmmands_handler::limit_down, this, _1), "Set download limit");
m_cmd_binder.set_handler("limit", boost::bind(&daemon_cmmands_handler::limit, this, _1), "Set download and upload limit");
m_cmd_binder.set_handler("out_peers", boost::bind(&daemon_cmmands_handler::out_peers_limit, this, _1), "Set max limit of out peers");
m_cmd_binder.set_handler("limit_up", boost::bind(&daemon_cmmands_handler::limit_up, this, _1), "Set upload limit [kB/s]");
m_cmd_binder.set_handler("limit_down", boost::bind(&daemon_cmmands_handler::limit_down, this, _1), "Set download limit [kB/s]");
m_cmd_binder.set_handler("limit", boost::bind(&daemon_cmmands_handler::limit, this, _1), "Set download and upload limit [kB/s]");
m_cmd_binder.set_handler("fast_exit", boost::bind(&daemon_cmmands_handler::fast_exit, this, _1), "Exit");
m_cmd_binder.set_handler("test_drop_download", boost::bind(&daemon_cmmands_handler::test_drop_download, this, _1), "For network testing, drop downloaded blocks instead checking/adding them to blockchain. Can fake-download blocks very fast.");
m_cmd_binder.set_handler("start_save_graph", boost::bind(&daemon_cmmands_handler::start_save_graph, this, _1), "");
m_cmd_binder.set_handler("stop_save_graph", boost::bind(&daemon_cmmands_handler::stop_save_graph, this, _1), "");
}
bool start_handling()
@ -327,6 +360,8 @@ private:
PUSH_WARNINGS
DISABLE_GCC_WARNING(maybe-uninitialized)
log_space::log_singletone::get_set_log_detalisation_level(true, l);
int otshell_utils_log_level = 100 - (l * 25);
gCurrentLogger.setDebugLevel(otshell_utils_log_level);
POP_WARNINGS
return true;

12
src/daemon/main.cpp
View file

@ -50,6 +50,9 @@ int main(int argc, char const * argv[])
{
try {
_note_c("dbg/main", "Begin of main()");
// TODO parse the debug options like set log level right here at start
epee::string_tools::set_module_name_and_folder(argv[0]);
// Build argument description
@ -71,6 +74,9 @@ int main(int argc, char const * argv[])
command_line::add_arg(visible_options, command_line::arg_testnet_data_dir, default_testnet_data_dir.string());
bf::path default_conf = default_data_dir / std::string(CRYPTONOTE_NAME ".conf");
command_line::add_arg(visible_options, daemon_args::arg_config_file, default_conf.string());
command_line::add_arg(visible_options, command_line::arg_test_drop_download);
command_line::add_arg(visible_options, command_line::arg_test_dbg_lock_sleep);
command_line::add_arg(visible_options, command_line::arg_test_drop_download_height);
// Settings
bf::path default_log = default_data_dir / std::string(CRYPTONOTE_NAME ".log");
@ -128,6 +134,8 @@ int main(int argc, char const * argv[])
return 0;
}
epee::g_test_dbg_lock_sleep = command_line::get_arg(vm, command_line::arg_test_dbg_lock_sleep);
bool testnet_mode = command_line::get_arg(vm, daemon_args::arg_testnet_on);
auto data_dir_arg = testnet_mode ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
@ -209,6 +217,8 @@ int main(int argc, char const * argv[])
else if (epee::log_space::get_set_log_detalisation_level(false) != new_log_level)
{
epee::log_space::get_set_log_detalisation_level(true, new_log_level);
int otshell_utils_log_level = 100 - (new_log_level * 25);
gCurrentLogger.setDebugLevel(otshell_utils_log_level);
LOG_PRINT_L0("LOG_LEVEL set to " << new_log_level);
}
}
@ -224,6 +234,8 @@ int main(int argc, char const * argv[])
);
}
_note_c("dbg/main", "Moving from main() into the daemonize now.");
return daemonizer::daemonize(argc, argv, daemonize::t_executor{}, vm);
}
catch (std::exception const & ex)

150
src/daemon/rpc_command_executor.cpp
View file

@ -32,6 +32,7 @@
#include "common/scoped_message_writer.h"
#include "daemon/rpc_command_executor.h"
#include "rpc/core_rpc_server_commands_defs.h"
#include "cryptonote_core/cryptonote_core.h"
#include <boost/format.hpp>
#include <ctime>
@ -267,11 +268,38 @@ bool t_rpc_command_executor::print_connections() {
}
}
tools::msg_writer() << std::setw(30) << std::left << "Remote Host"
<< std::setw(20) << "Peer id"
<< std::setw(30) << "Recv/Sent (inactive,sec)"
<< std::setw(25) << "State"
<< std::setw(20) << "Livetime(sec)"
<< std::setw(12) << "Down (kB/s)"
<< std::setw(14) << "Down(now)"
<< std::setw(10) << "Up (kB/s)"
<< std::setw(13) << "Up(now)"
<< std::endl;
for (auto & info : res.connections)
{
std::string address = info.ip + ":" + info.port;
std::string in_out = info.incoming ? "INC" : "OUT";
tools::msg_writer() << boost::format("%-25s peer_id: %-25s %s") % address % info.peer_id % in_out;
std::string address = info.incoming ? "INC " : "OUT ";
address += info.ip + ":" + info.port;
//std::string in_out = info.incoming ? "INC " : "OUT ";
tools::msg_writer()
//<< std::setw(30) << std::left << in_out
<< std::setw(30) << std::left << address
<< std::setw(20) << info.peer_id
<< std::setw(30) << std::to_string(info.recv_count) + "(" + std::to_string(info.recv_idle_time) + ")/" + std::to_string(info.send_count) + "(" + std::to_string(info.send_idle_time) + ")"
<< std::setw(25) << info.state
<< std::setw(20) << info.live_time
<< std::setw(12) << info.avg_download
<< std::setw(14) << info.current_download
<< std::setw(10) << info.avg_upload
<< std::setw(13) << info.current_upload
<< std::left << (info.localhost ? "[LOCALHOST]" : "")
<< std::left << (info.local_ip ? "[LAN]" : "");
//tools::msg_writer() << boost::format("%-25s peer_id: %-25s %s") % address % info.peer_id % in_out;
}
return true;
@ -659,34 +687,134 @@ bool t_rpc_command_executor::print_status()
bool t_rpc_command_executor::set_limit(int limit)
{
/*
epee::net_utils::connection_basic::set_rate_down_limit( limit );
epee::net_utils::connection_basic::set_rate_up_limit( limit );
std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl;
std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl;
*/
return true;
}
bool t_rpc_command_executor::set_limit_up(int limit)
{
/*
epee::net_utils::connection_basic::set_rate_up_limit( limit );
std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl;
*/
return true;
}
bool t_rpc_command_executor::set_limit_down(int limit)
{
/*
epee::net_utils::connection_basic::set_rate_down_limit( limit );
std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl;
*/
return true;
}
bool t_rpc_command_executor::fast_exit()
{
cryptonote::COMMAND_RPC_FAST_EXIT::request req;
cryptonote::COMMAND_RPC_FAST_EXIT::response res;
std::string fail_message = "Daemon did not stop";
if (m_is_rpc)
{
if (!m_rpc_client->rpc_request(req, res, "/fast_exit", fail_message.c_str()))
{
return true;
}
}
else
{
if (!m_rpc_server->on_fast_exit(req, res))
{
tools::fail_msg_writer() << fail_message.c_str();
return true;
}
}
tools::success_msg_writer() << "Daemon stopped";
return true;
}
bool t_rpc_command_executor::out_peers(uint64_t limit)
{
cryptonote::COMMAND_RPC_OUT_PEERS::request req;
cryptonote::COMMAND_RPC_OUT_PEERS::response res;
epee::json_rpc::error error_resp;
req.out_peers = limit;
std::string fail_message = "Unsuccessful";
if (m_is_rpc)
{
if (!m_rpc_client->json_rpc_request(req, res, "/out_peers", fail_message.c_str()))
{
return true;
}
}
else
{
if (!m_rpc_server->on_out_peers(req, res))
{
tools::fail_msg_writer() << fail_message.c_str();
return true;
}
}
return true;
}
bool t_rpc_command_executor::start_save_graph()
{
cryptonote::COMMAND_RPC_START_SAVE_GRAPH::request req;
cryptonote::COMMAND_RPC_START_SAVE_GRAPH::response res;
std::string fail_message = "Unsuccessful";
if (m_is_rpc)
{
if (!m_rpc_client->rpc_request(req, res, "/start_save_graph", fail_message.c_str()))
{
return true;
}
}
else
{
if (!m_rpc_server->on_start_save_graph(req, res))
{
tools::fail_msg_writer() << fail_message.c_str();
return true;
}
}
return true;
}
bool t_rpc_command_executor::stop_save_graph()
{
cryptonote::COMMAND_RPC_STOP_SAVE_GRAPH::request req;
cryptonote::COMMAND_RPC_STOP_SAVE_GRAPH::response res;
std::string fail_message = "Unsuccessful";
if (m_is_rpc)
{
if (!m_rpc_client->rpc_request(req, res, "/stop_save_graph", fail_message.c_str()))
{
return true;
}
}
else
{
if (!m_rpc_server->on_stop_save_graph(req, res))
{
tools::fail_msg_writer() << fail_message.c_str();
return true;
}
}
return true;
}
}// namespace daemonize

6
src/daemon/rpc_command_executor.h
View file

@ -105,7 +105,13 @@ public:
bool set_limit_down(int limit);
bool fast_exit();
bool out_peers(uint64_t limit);
bool start_save_graph();
bool stop_save_graph();
};
} // namespace daemonize

1
src/miner/simpleminer.cpp
View file

@ -41,6 +41,7 @@
using namespace epee;
namespace po = boost::program_options;
unsigned int epee::g_test_dbg_lock_sleep = 0;
int main(int argc, char** argv)
{

46
src/p2p/CMakeLists.txt Normal file
View file

@ -0,0 +1,46 @@
# Copyright (c) 2014, The Monero Project
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are
# permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
# of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cmake_minimum_required (VERSION 2.6)
project (bitmonero CXX)
file(GLOB P2P *)
source_group(p2p FILES ${P2P})
#add_library(p2p ${P2P})
#bitmonero_private_headers(p2p ${P2P})
bitmonero_add_library(p2p ${P2P})
#target_link_libraries(p2p)
# LINK_PRIVATE
# ${Boost_CHRONO_LIBRARY}
# ${Boost_REGEX_LIBRARY}
# ${Boost_SYSTEM_LIBRARY}
# ${Boost_THREAD_LIBRARY}
# ${EXTRA_LIBRARIES})
add_dependencies(p2p
version)

287
src/p2p/connection_basic.cpp Normal file
View file

@ -0,0 +1,287 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief base for connection, contains e.g. the ratelimit hooks
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/* rfree: implementation for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */
#include "connection_basic.hpp"
#include <boost/asio.hpp>
#include <string>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/interprocess/detail/atomic.hpp>
#include <boost/thread/thread.hpp>
#include <memory>
#include "syncobj.h"
#include "../../contrib/epee/include/net/net_utils_base.h"
#include "../../contrib/epee/include/misc_log_ex.h"
#include <boost/lambda/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/lambda/lambda.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/chrono.hpp>
#include <boost/utility/value_init.hpp>
#include <boost/asio/deadline_timer.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/thread/thread.hpp>
#include <boost/filesystem.hpp>
#include "misc_language.h"
#include "pragma_comp_defs.h"
#include <fstream>
#include <sstream>
#include <iomanip>
#include <algorithm>
#include <mutex>
#include <boost/asio/basic_socket.hpp>
#include <boost/asio/ip/unicast.hpp>
#include "../../contrib/epee/include/net/abstract_tcp_server2.h"
#include "../../contrib/otshell_utils/utils.hpp"
#include "data_logger.hpp"
using namespace nOT::nUtils;
// TODO:
#include "../../src/p2p/network_throttle-detail.hpp"
#include "../../src/cryptonote_core/cryptonote_core.h"
// ################################################################################################
// local (TU local) headers
// ################################################################################################
namespace epee
{
namespace net_utils
{
/* ============================================================================ */
class connection_basic_pimpl {
public:
connection_basic_pimpl(const std::string &name);
static int m_default_tos;
network_throttle_bw m_throttle; // per-perr
critical_section m_throttle_lock;
int m_peer_number; // e.g. for debug/stats
};
} // namespace
} // namespace
// ################################################################################################
// The implementation part
// ################################################################################################
namespace epee
{
namespace net_utils
{
// ================================================================================================
// connection_basic_pimpl
// ================================================================================================
connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_throttle(name) { }
// ================================================================================================
// connection_basic
// ================================================================================================
// static variables:
int connection_basic_pimpl::m_default_tos;
// methods:
connection_basic::connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number)
:
mI( new connection_basic_pimpl("peer") ),
strand_(io_service),
socket_(io_service),
m_want_close_connection(false),
m_was_shutdown(false),
m_ref_sock_count(ref_sock_count)
{
++ref_sock_count; // increase the global counter
mI->m_peer_number = sock_number.fetch_add(1); // use, and increase the generated number
string remote_addr_str = "?";
try { remote_addr_str = socket_.remote_endpoint().address().to_string(); } catch(...){} ;
_note("Spawned connection p2p#"<<mI->m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_ref_sock_count);
//boost::filesystem::create_directories("log/dr-monero/net/");
}
connection_basic::~connection_basic() {
string remote_addr_str = "?";
try { remote_addr_str = socket_.remote_endpoint().address().to_string(); } catch(...){} ;
_note("Destructing connection p2p#"<<mI->m_peer_number << " to " << remote_addr_str);
}
void connection_basic::set_rate_up_limit(uint64_t limit) {
// TODO remove __SCALING_FACTOR...
const double SCALING_FACTOR = 2.1; // to acheve the best performance
limit *= SCALING_FACTOR;
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
network_throttle_manager::get_global_throttle_out().set_target_speed(limit);
network_throttle_manager::get_global_throttle_out().set_real_target_speed(limit / SCALING_FACTOR);
}
save_limit_to_file(limit);
}
void connection_basic::set_rate_down_limit(uint64_t limit) {
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in );
network_throttle_manager::get_global_throttle_in().set_target_speed(limit);
}
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq );
network_throttle_manager::get_global_throttle_inreq().set_target_speed(limit);
}
save_limit_to_file(limit);
}
void connection_basic::save_limit_to_file(int limit) {
// saving limit to file
if (!epee::net_utils::data_logger::m_save_graph)
return;
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
epee::net_utils::data_logger::get_instance().add_data("upload_limit", network_throttle_manager::get_global_throttle_out().get_terget_speed() / 1024);
}
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in );
epee::net_utils::data_logger::get_instance().add_data("download_limit", network_throttle_manager::get_global_throttle_in().get_terget_speed() / 1024);
}
}
void connection_basic::set_tos_flag(int tos) {
connection_basic_pimpl::m_default_tos = tos;
}
int connection_basic::get_tos_flag() {
return connection_basic_pimpl::m_default_tos;
}
void connection_basic::sleep_before_packet(size_t packet_size, int phase, int q_len) {
double delay=0; // will be calculated
do
{ // rate limiting
if (m_was_shutdown) {
_dbg2("m_was_shutdown - so abort sleep");
return;
}
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global
}
delay *= 0.50;
if (delay > 0) {
long int ms = (long int)(delay * 1000);
_info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<packet_size); // debug sleep
_dbg1("sleep in sleep_before_packet");
epee::net_utils::data_logger::get_instance().add_data("sleep_up", ms);
boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) );
}
} while(delay > 0);
// XXX LATER XXX
{
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
network_throttle_manager::get_global_throttle_out().handle_trafic_exact( packet_size * 700); // increase counter - global
}
}
void connection_basic::set_start_time() {
CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out );
m_start_time = network_throttle_manager::get_global_throttle_out().get_time_seconds();
}
void connection_basic::do_send_handler_write(const void* ptr , size_t cb ) {
sleep_before_packet(cb,1,-1);
_info_c("net/out/size", "handler_write (direct) - before ASIO write, for packet="<<cb<<" B (after sleep)");
set_start_time();
}
void connection_basic::do_send_handler_write_from_queue( const boost::system::error_code& e, size_t cb, int q_len ) {
sleep_before_packet(cb,2,q_len);
_info_c("net/out/size", "handler_write (after write, from queue="<<q_len<<") - before ASIO write, for packet="<<cb<<" B (after sleep)");
set_start_time();
}
void connection_basic::logger_handle_net_read(size_t size) { // network data read
size /= 1024;
epee::net_utils::data_logger::get_instance().add_data("download", size);
}
void connection_basic::logger_handle_net_write(size_t size) {
size /= 1024;
epee::net_utils::data_logger::get_instance().add_data("upload", size);
}
double connection_basic::get_sleep_time(size_t cb) {
CRITICAL_REGION_LOCAL(epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_out);
auto t = network_throttle_manager::get_global_throttle_out().get_sleep_time(cb);
return t;
}
void connection_basic::set_save_graph(bool save_graph) {
epee::net_utils::data_logger::m_save_graph = save_graph;
}
} // namespace
} // namespace

132
src/p2p/connection_basic.hpp Normal file
View file

@ -0,0 +1,132 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief base for connection, contains e.g. the ratelimit hooks
// ! This file might contain variable names same as in template class connection<>
// ! from files contrib/epee/include/net/abstract_tcp_server2.*
// ! I am not a lawyer; afaik APIs, var names etc are not copyrightable ;)
// ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part)
// ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as:
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
/* rfree: place for hanlers for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */
#ifndef INCLUDED_p2p_connection_basic_hpp
#define INCLUDED_p2p_connection_basic_hpp
#include <boost/asio.hpp>
#include <string>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/interprocess/detail/atomic.hpp>
#include <boost/thread/thread.hpp>
#include <memory>
#include "../../contrib/epee/include/net/net_utils_base.h"
#include "../../contrib/epee/include/syncobj.h"
namespace epee
{
namespace net_utils
{
/************************************************************************/
/* */
/************************************************************************/
/// Represents a single connection from a client.
class connection_basic_pimpl; // PIMPL for this class
class connection_basic { // not-templated base class for rapid developmet of some code parts
public:
std::unique_ptr< connection_basic_pimpl > mI; // my Implementation
// moved here from orginal connecton<> - common member variables that do not depend on template in connection<>
volatile uint32_t m_want_close_connection;
std::atomic<bool> m_was_shutdown;
critical_section m_send_que_lock;
std::list<std::string> m_send_que;
volatile bool m_is_multithreaded;
double m_start_time;
/// Strand to ensure the connection's handlers are not called concurrently.
boost::asio::io_service::strand strand_;
/// Socket for the connection.
boost::asio::ip::tcp::socket socket_;
std::atomic<long> &m_ref_sock_count; // reference to external counter of existing sockets that we will ++/--
public:
// first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator
connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number);
virtual ~connection_basic();
// various handlers to be called from connection class:
void do_send_handler_write(const void * ptr , size_t cb);
void do_send_handler_write_from_queue(const boost::system::error_code& e, size_t cb , int q_len); // from handle_write, sending next part
void logger_handle_net_write(size_t size); // network data written
void logger_handle_net_read(size_t size); // network data read
void set_start_time();
// config for rate limit
static void set_rate_up_limit(uint64_t limit);
static void set_rate_down_limit(uint64_t limit);
// config misc
static void set_tos_flag(int tos); // ToS / QoS flag
static int get_tos_flag();
// handlers and sleep
void sleep_before_packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?)
static void save_limit_to_file(int limit); ///< for dr-monero
static double get_sleep_time(size_t cb);
static void set_save_graph(bool save_graph);
};
} // nameserver
} // nameserver
#endif

173
src/p2p/data_logger.cpp Normal file
View file

@ -0,0 +1,173 @@
#include "data_logger.hpp"
#include <stdexcept>
#include <boost/chrono.hpp>
#include <boost/filesystem.hpp>
#include <chrono>
#include "../../contrib/otshell_utils/utils.hpp"
namespace epee
{
namespace net_utils
{
data_logger &data_logger::get_instance() {
std::call_once(m_singleton,
[] {
_info_c("dbg/data","Creating singleton of data_logger");
if (m_state != data_logger_state::state_before_init) { _erro_c("dbg/data","Internal error in singleton"); throw std::runtime_error("data_logger singleton"); }
m_state = data_logger_state::state_during_init;
m_obj.reset(new data_logger());
m_state = data_logger_state::state_ready_to_use;
}
);
if (m_state != data_logger_state::state_ready_to_use) {
_erro ("trying to use not working data_logger");
throw std::runtime_error("data_logger ctor state");
}
return * m_obj;
}
data_logger::data_logger() {
_note_c("dbg/data","Starting data logger (for graphs data)");
if (m_state != data_logger_state::state_during_init) { _erro_c("dbg/data","Singleton ctor state"); throw std::runtime_error("data_logger ctor state"); }
std::lock_guard<std::mutex> lock(mMutex); // lock
// prepare all the files for given data channels:
mFilesMap["peers"] = data_logger::fileData("log/dr-monero/peers.data");
mFilesMap["download"] = data_logger::fileData("log/dr-monero/net/in-all.data");
mFilesMap["upload"] = data_logger::fileData("log/dr-monero/net/out-all.data");
mFilesMap["request"] = data_logger::fileData("log/dr-monero/net/req-all.data");
mFilesMap["sleep_down"] = data_logger::fileData("log/dr-monero/down_sleep_log.data");
mFilesMap["sleep_up"] = data_logger::fileData("log/dr-monero/up_sleep_log.data");
mFilesMap["calc_time"] = data_logger::fileData("log/dr-monero/get_objects_calc_time.data");
mFilesMap["blockchain_processing_time"] = data_logger::fileData("log/dr-monero/blockchain_log.data");
mFilesMap["block_processing"] = data_logger::fileData("log/dr-monero/block_proc.data");
mFilesMap["peers_limit"] = data_logger::fileData("log/dr-monero/peers_limit.info");
mFilesMap["download_limit"] = data_logger::fileData("log/dr-monero/limit_down.info");
mFilesMap["upload_limit"] = data_logger::fileData("log/dr-monero/limit_up.info");
mFilesMap["peers_limit"].mLimitFile = true;
mFilesMap["download_limit"].mLimitFile = true;
mFilesMap["upload_limit"].mLimitFile = true;
// do NOT modify mFilesMap below this point, since there is no locking for this used (yet)
_info_c("dbg/data","Creating thread for data logger"); // create timer thread
m_thread_maybe_running=true;
std::shared_ptr<std::thread> logger_thread(new std::thread([&]() {
_info_c("dbg/data","Inside thread for data logger");
while (m_state == data_logger_state::state_during_init) { // wait for creation to be done (in other thread, in singleton) before actually running
std::this_thread::sleep_for(std::chrono::seconds(1));
}
_info_c("dbg/data","Inside thread for data logger - going into main loop");
while (m_state == data_logger_state::state_ready_to_use) { // run as long as we are not closing the single object
std::this_thread::sleep_for(std::chrono::seconds(1));
saveToFile(); // save all the pending data
}
_info_c("dbg/data","Inside thread for data logger - done the main loop");
m_thread_maybe_running=false;
}));
logger_thread->detach();
_info_c("dbg/data","Data logger constructed");
}
data_logger::~data_logger() {
_note_c("dbg/data","Destructor of the data logger");
{
std::lock_guard<std::mutex> lock(mMutex);
m_state = data_logger_state::state_dying;
}
_info_c("dbg/data","State was set to dying");
while(m_thread_maybe_running) { // wait for the thread to exit
std::this_thread::sleep_for(std::chrono::seconds(1));
_info_c("dbg/data","Waiting for background thread to exit");
}
_info_c("dbg/data","Thread exited");
}
void data_logger::kill_instance() {
m_state = data_logger_state::state_dying;
m_obj.reset();
}
void data_logger::add_data(std::string filename, unsigned int data) {
std::lock_guard<std::mutex> lock(mMutex);
if (m_state != data_logger_state::state_ready_to_use) { _info_c("dbg/data","Data logger is not ready, returning."); return; }
if (mFilesMap.find(filename) == mFilesMap.end()) { // no such file/counter
_erro_c("dbg/data","Trying to use not opened data file filename="<<filename);
_erro_c("dbg/data","Disabling saving of graphs due to error");
m_save_graph=false; // <--- disabling saving graphs
return;
}
if (mFilesMap[filename].mLimitFile) { // this holds a number (that is not additive) - e.g. the limit setting
mFilesMap[filename].mDataToSave = data;
} else {
mFilesMap[filename].mDataToSave += data; // this holds a number that should be sum of all accumulated samples
}
}
bool data_logger::is_dying() {
if (m_state == data_logger_state::state_dying) {
return true;
}
else {
return false;
}
}
void data_logger::saveToFile() {
_dbg2_c("dbg/data","saving to files");
std::lock_guard<std::mutex> lock(mMutex);
if (m_state != data_logger_state::state_ready_to_use) { _info_c("dbg/data","Data logger is not ready, returning."); return; }
nOT::nUtils::cFilesystemUtils::CreateDirTree("log/dr-monero/net/");
for (auto &element : mFilesMap)
{
element.second.save();
if (!element.second.mLimitFile) element.second.mDataToSave = 0;
}
}
// the inner class:
double data_logger::fileData::get_current_time() {
#if defined(__APPLE__)
auto point = std::chrono::system_clock::now();
#else
auto point = std::chrono::steady_clock::now();
#endif
auto time_from_epoh = point.time_since_epoch();
auto ms = std::chrono::duration_cast< std::chrono::milliseconds >( time_from_epoh ).count();
double ms_f = ms;
return ms_f / 1000.;
}
data_logger::fileData::fileData(std::string pFile) {
_dbg3_c("dbg/data","opening data file named pFile="<<pFile<<" for this="<<this);
mFile = std::make_shared<std::ofstream> (pFile);
_dbg1_c("dbg/data","opened data file named pFile="<<pFile<<" in mFile="<<mFile<<" for this="<<this);
mPath = pFile;
}
void data_logger::fileData::save() {
if (!data_logger::m_save_graph) return; // <--- disabled
_dbg2_c("dbg/data","saving to the file now, mFile="<<mFile);
mFile->open(mPath, std::ios::app);
*mFile << static_cast<int>(get_current_time()) << " " << mDataToSave << std::endl;
mFile->close();
}
data_logger_state data_logger::m_state(data_logger_state::state_before_init); ///< (static) state of the singleton object
std::atomic<bool> data_logger::m_save_graph(false); // (static)
std::atomic<bool> data_logger::m_thread_maybe_running(false); // (static)
std::once_flag data_logger::m_singleton; // (static)
std::unique_ptr<data_logger> data_logger::m_obj; // (static)
} // namespace
} // namespace

76
src/p2p/data_logger.hpp Normal file
View file

@ -0,0 +1,76 @@
#ifndef INCLUDED_p2p_data_logger_hpp
#define INCLUDED_p2p_data_logger_hpp
#include <string>
#include <map>
#include <fstream>
#include <memory>
#include <thread>
#include <mutex>
#include <atomic>
namespace epee
{
namespace net_utils
{
enum class data_logger_state { state_before_init, state_during_init, state_ready_to_use, state_dying };
/***
@note: use it ONLY via singleton! It will be spawned then, and will auto destruct on program exit.
@note: do call ::kill_instance() before exiting main, at end of main. But before make sure no one else (e.g. no other threads) will try to use this/singleton
@note: it is not allowed to use this class from code "runnig before or after main", e.g. from ctors of static objects, because of static-creation-order races
@note: on creation (e.g. from singleton), it spawns a thread that saves all data in background
*/
class data_logger {
public:
static data_logger &get_instance(); ///< singleton
static void kill_instance(); ///< call this before ending main to allow more gracefull shutdown of the main singleton and it's background thread
~data_logger(); ///< destr, will be called when singleton is killed when global m_obj dies. will kill theads etc
private:
data_logger(); ///< constructor is private, use only via singleton get_instance
public:
data_logger(const data_logger &ob) = delete; // use only one per program
data_logger(data_logger &&ob) = delete;
data_logger & operator=(const data_logger&) = delete;
data_logger & operator=(data_logger&&) = delete;
void add_data(std::string filename, unsigned int data); ///< use this to append data here. Use it only the singleton. It locks itself.
static std::atomic<bool> m_save_graph; ///< global setting flag, should we save all the data or not (can disable logging graphs data)
static bool is_dying();
private:
static std::once_flag m_singleton; ///< to guarantee singleton creates the object exactly once
static data_logger_state m_state; ///< state of the singleton object
static std::atomic<bool> m_thread_maybe_running; ///< is the background thread (more or less) running, or is it fully finished
static std::unique_ptr<data_logger> m_obj; ///< the singleton object. Only use it via get_instance(). Can be killed by kill_instance()
/***
* one graph/file with data
*/
class fileData {
public:
fileData() = default;
fileData(const fileData &ob) = delete;
fileData(std::string pFile);
std::shared_ptr<std::ofstream> mFile;
long int mDataToSave = 0; ///< sum of the data (in current interval, will be counted from 0 on next interval)
static double get_current_time();
void save();
std::string mPath;
bool mLimitFile = false; ///< this holds a number (that is not additive) - e.g. the limit setting
};
std::map<std::string, fileData> mFilesMap;
std::mutex mMutex;
void saveToFile(); ///< write data to the target files. do not use this directly
};
} // namespace
} // namespace
#endif

51
src/p2p/net_node.h
View file

@ -80,13 +80,16 @@ namespace nodetool
public:
typedef t_payload_net_handler payload_net_handler;
node_server(
t_payload_net_handler& payload_handler
)
: m_payload_handler(payload_handler)
, m_allow_local_ip(false)
, m_hide_my_port(false)
{}
node_server(t_payload_net_handler& payload_handler)
:m_payload_handler(payload_handler),
m_allow_local_ip(false),
m_no_igd(false),
m_hide_my_port(false)
{
m_current_number_of_out_peers = 0;
m_save_graph = false;
is_closing = false;
}
static void init_options(boost::program_options::options_description& desc);
@ -109,6 +112,7 @@ namespace nodetool
virtual uint64_t get_connections_count();
size_t get_outgoing_connections_count();
peerlist_manager& get_peerlist_manager(){return m_peerlist;}
void delete_connections(size_t count);
private:
const std::vector<std::string> m_seed_nodes_list =
{ "seeds.moneroseeds.se"
@ -117,6 +121,9 @@ namespace nodetool
, "seeds.moneroseeds.li"
};
bool islimitup=false;
bool islimitdown=false;
typedef COMMAND_REQUEST_STAT_INFO_T<typename t_payload_net_handler::stat_info> COMMAND_REQUEST_STAT_INFO;
CHAIN_LEVIN_INVOKE_MAP2(p2p_connection_context); //move levin_commands_handler interface invoke(...) callbacks into invoke map
@ -195,6 +202,20 @@ namespace nodetool
template <class Container>
bool parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor<std::vector<std::string> > & arg, Container& container);
bool set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max);
bool set_tos_flag(const boost::program_options::variables_map& vm, int limit);
bool set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit);
bool set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit);
bool set_rate_limit(const boost::program_options::variables_map& vm, uint64_t limit);
void kill() { ///< will be called e.g. from deinit()
_info("Killing the net_node");
is_closing = true;
mPeersLoggerThread->join(); // make sure the thread finishes
_info("Joined extra background net_node threads");
}
//debug functions
std::string print_connections_container();
@ -212,7 +233,16 @@ namespace nodetool
END_KV_SERIALIZE_MAP()
};
config m_config;
public:
config m_config; // TODO was private, add getters?
std::atomic<unsigned int> m_current_number_of_out_peers;
void set_save_graph(bool save_graph)
{
m_save_graph = save_graph;
epee::net_utils::connection_basic::set_save_graph(save_graph);
}
private:
std::string m_config_folder;
bool m_have_address;
@ -222,7 +252,10 @@ namespace nodetool
uint32_t m_ip_address;
bool m_allow_local_ip;
bool m_hide_my_port;
bool m_no_igd;
std::atomic<bool> m_save_graph;
std::atomic<bool> is_closing;
std::unique_ptr<std::thread> mPeersLoggerThread;
//critical_section m_connections_lock;
//connections_indexed_container m_connections;

172
src/p2p/net_node.inl
View file

@ -46,6 +46,7 @@
#include "net/local_ip.h"
#include "crypto/crypto.h"
#include "storages/levin_abstract_invoke2.h"
#include "data_logger.hpp"
#include "daemon/command_line_args.h"
// We have to look for miniupnpc headers in different places, dependent on if its compiled or external
@ -85,6 +86,16 @@ namespace nodetool
" If this option is given the options add-priority-node and seed-node are ignored"};
const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_seed_node = {"seed-node", "Connect to a node to retrieve peer addresses, and disconnect"};
const command_line::arg_descriptor<bool> arg_p2p_hide_my_port = {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true};
const command_line::arg_descriptor<bool> arg_no_igd = {"no-igd", "Disable UPnP port mapping"};
const command_line::arg_descriptor<int64_t> arg_out_peers = {"out-peers", "set max limit of out peers", -1};
const command_line::arg_descriptor<int> arg_tos_flag = {"tos-flag", "set TOS flag", -1};
const command_line::arg_descriptor<int64_t> arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", -1};
const command_line::arg_descriptor<int64_t> arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", -1};
const command_line::arg_descriptor<uint64_t> arg_limit_rate = {"limit-rate", "set limit-rate [kB/s]", 128};
const command_line::arg_descriptor<bool> arg_save_graph = {"save-graph", "Save data for dr monero", false};
}
//-----------------------------------------------------------------------------------
@ -100,7 +111,15 @@ namespace nodetool
command_line::add_arg(desc, arg_p2p_add_priority_node);
command_line::add_arg(desc, arg_p2p_add_exclusive_node);
command_line::add_arg(desc, arg_p2p_seed_node);
command_line::add_arg(desc, arg_p2p_hide_my_port); }
command_line::add_arg(desc, arg_p2p_hide_my_port);
command_line::add_arg(desc, arg_no_igd);
command_line::add_arg(desc, arg_out_peers);
command_line::add_arg(desc, arg_tos_flag);
command_line::add_arg(desc, arg_limit_rate_up);
command_line::add_arg(desc, arg_limit_rate_down);
command_line::add_arg(desc, arg_limit_rate);
command_line::add_arg(desc, arg_save_graph);
}
//-----------------------------------------------------------------------------------
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::init_config()
@ -121,7 +140,6 @@ namespace nodetool
//at this moment we have hardcoded config
m_config.m_net_config.handshake_interval = P2P_DEFAULT_HANDSHAKE_INTERVAL;
m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT;
m_config.m_net_config.packet_max_size = P2P_DEFAULT_PACKET_MAX_SIZE; //20 MB limit
m_config.m_net_config.config_id = 0; // initial config
m_config.m_net_config.connection_timeout = P2P_DEFAULT_CONNECTION_TIMEOUT;
@ -166,6 +184,7 @@ namespace nodetool
m_port = command_line::get_arg(vm, p2p_bind_arg);
m_external_port = command_line::get_arg(vm, arg_p2p_external_port);
m_allow_local_ip = command_line::get_arg(vm, arg_p2p_allow_local_ip);
m_no_igd = command_line::get_arg(vm, arg_no_igd);
if (command_line::has_arg(vm, arg_p2p_add_peer))
{
@ -180,16 +199,23 @@ namespace nodetool
}
}
if(command_line::has_arg(vm, arg_save_graph))
{
set_save_graph(true);
}
if (command_line::has_arg(vm,arg_p2p_add_exclusive_node))
{
if (!parse_peers_and_add_to_container(vm, arg_p2p_add_exclusive_node, m_exclusive_peers))
return false;
}
if (command_line::has_arg(vm, arg_p2p_add_priority_node))
{
if (!parse_peers_and_add_to_container(vm, arg_p2p_add_priority_node, m_priority_peers))
return false;
}
if (command_line::has_arg(vm, arg_p2p_seed_node))
{
if (!parse_peers_and_add_to_container(vm, arg_p2p_seed_node, m_seed_nodes))
@ -199,6 +225,21 @@ namespace nodetool
if(command_line::has_arg(vm, arg_p2p_hide_my_port))
m_hide_my_port = true;
if ( !set_max_out_peers(vm, command_line::get_arg(vm, arg_out_peers) ) )
return false;
if ( !set_tos_flag(vm, command_line::get_arg(vm, arg_tos_flag) ) )
return false;
if ( !set_rate_up_limit(vm, command_line::get_arg(vm, arg_limit_rate_up) ) )
return false;
if ( !set_rate_down_limit(vm, command_line::get_arg(vm, arg_limit_rate_down) ) )
return false;
if ( !set_rate_limit(vm, command_line::get_arg(vm, arg_limit_rate) ) )
return false;
return true;
}
//-----------------------------------------------------------------------------------
@ -393,6 +434,7 @@ namespace nodetool
LOG_PRINT_L0("External port defined as " << m_external_port);
// Add UPnP port mapping
if(m_no_igd == false) {
LOG_PRINT_L0("Attempting to add IGD port mapping.");
int result;
UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, &result);
@ -428,7 +470,7 @@ namespace nodetool
} else {
LOG_PRINT_L0("No IGD was found.");
}
}
return res;
}
//-----------------------------------------------------------------------------------
@ -441,6 +483,30 @@ namespace nodetool
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::run()
{
// creating thread to log number of connections
mPeersLoggerThread.reset(new std::thread([&]()
{
_note("Thread monitor number of peers - start");
while (!is_closing)
{ // main loop of thread
//number_of_peers = m_net_server.get_config_object().get_connections_count();
unsigned int number_of_peers = 0;
m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt)
{
if (!cntxt.m_is_income) ++number_of_peers;
return true;
}); // lambda
m_current_number_of_out_peers = number_of_peers;
if (epee::net_utils::data_logger::is_dying())
break;
epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers);
std::this_thread::sleep_for(std::chrono::seconds(1));
} // main loop of thread
_note("Thread monitor number of peers - done");
})); // lambda
//here you can set worker threads count
int thrds_count = 10;
@ -471,6 +537,7 @@ namespace nodetool
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::deinit()
{
kill();
m_peerlist.deinit();
m_net_server.deinit_server();
return store_config();
@ -683,6 +750,16 @@ namespace nodetool
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::try_to_connect_and_handshake_with_new_peer(const net_address& na, bool just_take_peerlist, uint64_t last_seen_stamp, bool white)
{
if (m_current_number_of_out_peers == m_config.m_net_config.connections_count) // out peers limit
{
return false;
}
else if (m_current_number_of_out_peers > m_config.m_net_config.connections_count)
{
m_net_server.get_config_object().del_out_connections(1);
m_current_number_of_out_peers --; // atomic variable, update time = 1s
return false;
}
LOG_PRINT_L1("Connecting to " << epee::string_tools::get_ip_string_from_int32(na.ip) << ":"
<< epee::string_tools::num_to_string_fast(na.port) << "(white=" << white << ", last_seen: "
<< (last_seen_stamp ? epee::misc_utils::get_time_interval_string(time(NULL) - last_seen_stamp):"never")
@ -770,16 +847,22 @@ namespace nodetool
++try_count;
if(is_peer_used(pe))
_note("Considering connecting (out) to peer: " << pe.id << " " << epee::string_tools::get_ip_string_from_int32(pe.adr.ip) << ":" << boost::lexical_cast<std::string>(pe.adr.port));
if(is_peer_used(pe)) {
_note("Peer is used");
continue;
}
LOG_PRINT_L1("Selected peer: " << pe.id << " " << epee::string_tools::get_ip_string_from_int32(pe.adr.ip)
<< ":" << boost::lexical_cast<std::string>(pe.adr.port)
<< "[white=" << use_white_list
<< "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never"));
if(!try_to_connect_and_handshake_with_new_peer(pe.adr, false, pe.last_seen, use_white_list))
if(!try_to_connect_and_handshake_with_new_peer(pe.adr, false, pe.last_seen, use_white_list)) {
_note("Handshake failed");
continue;
}
return true;
}
@ -1318,4 +1401,83 @@ namespace nodetool
return true;
}
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max)
{
if(max == -1) {
m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT;
epee::net_utils::data_logger::get_instance().add_data("peers_limit", m_config.m_net_config.connections_count);
return true;
}
epee::net_utils::data_logger::get_instance().add_data("peers_limit", max);
m_config.m_net_config.connections_count = max;
return true;
}
template<class t_payload_net_handler>
void node_server<t_payload_net_handler>::delete_connections(size_t count)
{
m_net_server.get_config_object().del_out_connections(count);
}
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_tos_flag(const boost::program_options::variables_map& vm, int flag)
{
if(flag==-1){
return true;
}
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_tos_flag(flag);
_dbg1("Set ToS flag " << flag);
return true;
}
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit)
{
this->islimitup=true;
if (limit==-1) {
limit=128;
this->islimitup=false;
}
limit *= 1024;
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit );
LOG_PRINT_L0("Set limit-up to " << limit/1024 << " kB/s");
return true;
}
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit)
{
this->islimitdown=true;
if(limit==-1) {
limit=128;
this->islimitdown=false;
}
limit *= 1024;
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit );
LOG_PRINT_L0("Set limit-down to " << limit/1024 << " kB/s");
return true;
}
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_rate_limit(const boost::program_options::variables_map& vm, uint64_t limit)
{
limit *= 1024;
if(this->islimitdown==false && this->islimitup==false) {
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit );
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit );
LOG_PRINT_L0("Set limit to " << limit/1024 << " kB/s");
}
else if(this->islimitdown==false && this->islimitup==true ) {
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit );
}
else if(this->islimitdown==true && this->islimitup==false ) {
epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit );
}
return true;
}
}

382
src/p2p/network_throttle-detail.cpp Normal file
View file

@ -0,0 +1,382 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief implementaion for throttling of connection (count and rate-limit speed etc)
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/* rfree: implementation for throttle details */
#include <boost/asio.hpp>
#include <string>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/interprocess/detail/atomic.hpp>
#include <boost/thread/thread.hpp>
#include <memory>
#include "syncobj.h"
#include "../../contrib/epee/include/net/net_utils_base.h"
#include "../../contrib/epee/include/misc_log_ex.h"
#include <boost/lambda/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/lambda/lambda.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/chrono.hpp>
#include <boost/utility/value_init.hpp>
#include <boost/asio/deadline_timer.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/thread/thread.hpp>
#include "misc_language.h"
#include "pragma_comp_defs.h"
#include <sstream>
#include <iomanip>
#include <algorithm>
#include <boost/asio/basic_socket.hpp>
#include <boost/asio/ip/unicast.hpp>
#include "../../contrib/epee/include/net/abstract_tcp_server2.h"
// TODO:
#include "../../src/p2p/network_throttle-detail.hpp"
#include "../../contrib/otshell_utils/utils.hpp"
#include "data_logger.hpp"
using namespace nOT::nUtils;
// ################################################################################################
// ################################################################################################
// the "header part". Not separeted out for .hpp because point of this modification is
// to rebuild just 1 translation unit while working on this code.
// (But maybe common parts will be separated out later though - if needed)
// ################################################################################################
// ################################################################################################
using namespace nOT::nUtils;
namespace epee
{
namespace net_utils
{
/* ============================================================================ */
class connection_basic_pimpl {
public:
connection_basic_pimpl(const std::string &name);
static int m_default_tos;
network_throttle_bw m_throttle; // per-perr
critical_section m_throttle_lock;
void _packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) could be used for different kinds of sleep e.g. direct/queue write
};
} // namespace
} // namespace
// ################################################################################################
// ################################################################################################
// The implementation part
// ################################################################################################
// ################################################################################################
namespace epee
{
namespace net_utils
{
// ================================================================================================
// network_throttle
// ================================================================================================
network_throttle::~network_throttle() { }
network_throttle::packet_info::packet_info()
: m_size(0)
{
}
network_throttle::network_throttle(const std::string &nameshort, const std::string &name, int window_size)
: m_window_size( (window_size==-1) ? 10 : window_size ),
m_history( m_window_size ), m_nameshort(nameshort)
{
set_name(name);
m_network_add_cost = 128;
m_network_minimal_segment = 256;
m_network_max_segment = 1024*1024;
m_any_packet_yet = false;
m_slot_size = 1.0; // hard coded in few places
m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle
}
void network_throttle::set_name(const std::string &name)
{
m_name = name;
}
void network_throttle::set_target_speed( network_speed_kbps target )
{
m_target_speed = target * 1024;
_note_c("net/"+m_nameshort, "Setting LIMIT: " << target << " kbps");
set_real_target_speed(target);
}
void network_throttle::set_real_target_speed( network_speed_kbps real_target )
{
m_real_target_speed = real_target * 1024;
}
network_speed_kbps network_throttle::get_terget_speed()
{
return m_real_target_speed / 1024;
}
void network_throttle::tick()
{
double time_now = get_time_seconds();
if (!m_any_packet_yet) m_start_time = time_now; // starting now
network_time_seconds current_sample_time_slot = time_to_slot( time_now ); // T=13.7 --> 13 (for 1-second smallwindow)
network_time_seconds last_sample_time_slot = time_to_slot( m_last_sample_time );
// moving to next position, and filling gaps
// !! during this loop the m_last_sample_time and last_sample_time_slot mean the variable moved in +1
// TODO optimize when moving few slots at once
while ( (!m_any_packet_yet) || (last_sample_time_slot < current_sample_time_slot))
{
_dbg3("Moving counter buffer by 1 second " << last_sample_time_slot << " < " << current_sample_time_slot << " (last time " << m_last_sample_time<<")");
// rotate buffer
for (size_t i=m_history.size()-1; i>=1; --i) m_history[i] = m_history[i-1];
m_history[0] = packet_info();
if (! m_any_packet_yet)
{
m_last_sample_time = time_now;
}
m_last_sample_time += 1; last_sample_time_slot = time_to_slot( m_last_sample_time ); // increase and recalculate time, time slot
m_any_packet_yet=true;
}
m_last_sample_time = time_now; // the real exact last time
}
void network_throttle::handle_trafic_exact(size_t packet_size)
{
_handle_trafic_exact(packet_size, packet_size);
}
void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_size)
{
tick();
calculate_times_struct cts ; calculate_times(packet_size, cts , false, -1);
calculate_times_struct cts2; calculate_times(packet_size, cts2, false, 5);
m_history[0].m_size += packet_size;
std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends;
std::string history_str = oss.str();
_dbg2_c( "net/" + m_nameshort , "Throttle " << m_name << ": packet of ~"<<packet_size<<"b " << " (from "<<orginal_size<<" b)"
<< " Speed AVG=" << std::setw(4) << ((long int)(cts .average/1024)) <<"[w="<<cts .window<<"]"
<< " " << std::setw(4) << ((long int)(cts2.average/1024)) <<"[w="<<cts2.window<<"]"
<<" / " << " Limit="<< ((long int)(m_target_speed/1024)) <<" KiB/sec "
<< " " << history_str
);
}
void network_throttle::handle_trafic_tcp(size_t packet_size)
{
size_t all_size = packet_size + m_network_add_cost;
all_size = std::max( m_network_minimal_segment , all_size);
_handle_trafic_exact( all_size , packet_size );
}
network_time_seconds network_throttle::get_sleep_time_after_tick(size_t packet_size) {
tick();
return get_sleep_time(packet_size);
}
void network_throttle::logger_handle_net(const std::string &filename, double time, size_t size) {
if (! epee::net_utils::data_logger::m_save_graph)
return;
std::mutex mutex;
mutex.lock(); {
std::fstream file;
file.open(filename.c_str(), std::ios::app | std::ios::out );
file.precision(6);
if(!file.is_open())
_warn("Can't open file " << filename);
file << static_cast<int>(time) << " " << static_cast<double>(size/1024) << "\n";
file.close();
} mutex.unlock();
}
// fine tune this to decide about sending speed:
network_time_seconds network_throttle::get_sleep_time(size_t packet_size) const
{
double D2=0;
calculate_times_struct cts = { 0, 0, 0, 0};
calculate_times(packet_size, cts, true, m_window_size); D2=cts.delay;
return D2;
}
// MAIN LOGIC:
void network_throttle::calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const
{
const double the_window_size = std::max( (double)m_window_size ,
((force_window>0) ? force_window : m_window_size)
);
if (!m_any_packet_yet) {
cts.window=0; cts.average=0; cts.delay=0;
cts.recomendetDataSize = m_network_minimal_segment; // should be overrided by caller anyway
return ; // no packet yet, I can not decide about sleep time
}
network_time_seconds window_len = (the_window_size-1) * m_slot_size ; // -1 since current slot is not finished
window_len += (m_last_sample_time - time_to_slot(m_last_sample_time)); // add the time for current slot e.g. 13.7-13 = 0.7
auto time_passed = get_time_seconds() - m_start_time;
cts.window = std::max( std::min( window_len , time_passed ) , m_slot_size ) ; // window length resulting from size of history but limited by how long ago history was started,
// also at least slot size (e.g. 1 second) to not be ridiculous
// window_len e.g. 5.7 because takes into account current slot time
size_t Epast = 0; // summ of traffic till now
for (auto sample : m_history) Epast += sample.m_size;
const size_t E = Epast;
const size_t Enow = Epast + packet_size ; // including the data we're about to send now
const double M = m_target_speed; // max
const double D1 = (Epast - M*cts.window) / M; // delay - how long to sleep to get back to target speed
const double D2 = (Enow - M*cts.window) / M; // delay - how long to sleep to get back to target speed (including current packet)
cts.delay = (D1*0.80 + D2*0.20); // finall sleep depends on both with/without current packet
// update_overheat();
cts.average = Epast/cts.window; // current avg. speed (for info)
if (Epast <= 0) {
if (cts.delay>=0) cts.delay = 0; // no traffic in history so we will not wait
}
double Wgood=-1;
{ // how much data we recommend now to download
Wgood = the_window_size + 1;
cts.recomendetDataSize = M*cts.window - E;
}
if (dbg) {
std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends;
std::string history_str = oss.str();
_dbg1_c( "net/"+m_nameshort+"_c" ,
(cts.delay > 0 ? "SLEEP" : "")
<< "dbg " << m_name << ": "
<< "speed is A=" << std::setw(8) <<cts.average<<" vs "
<< "Max=" << std::setw(8) <<M<<" "
<< " so sleep: "
<< "D=" << std::setw(8) <<cts.delay<<" sec "
<< "E="<< std::setw(8) << E << " (Enow="<<std::setw(8)<<Enow<<") "
<< "M=" << std::setw(8) << M <<" W="<< std::setw(8) << cts.window << " "
<< "R=" << std::setw(8) << cts.recomendetDataSize << " Wgood" << std::setw(8) << Wgood << " "
<< "History: " << std::setw(8) << history_str << " "
<< "m_last_sample_time=" << std::setw(8) << m_last_sample_time
);
}
}
double network_throttle::get_time_seconds() const {
#if defined(__APPLE__)
auto point = std::chrono::system_clock::now();
#else
auto point = std::chrono::steady_clock::now();
#endif
auto time_from_epoh = point.time_since_epoch();
auto ms = std::chrono::duration_cast< std::chrono::milliseconds >( time_from_epoh ).count();
double ms_f = ms;
return ms_f / 1000.;
}
size_t network_throttle::get_recommended_size_of_planned_transport_window(double force_window) const {
calculate_times_struct cts = { 0, 0, 0, 0};
network_throttle::calculate_times(0, cts, true, force_window);
cts.recomendetDataSize += m_network_add_cost;
if (cts.recomendetDataSize<0) cts.recomendetDataSize=0;
if (cts.recomendetDataSize>m_network_max_segment) cts.recomendetDataSize=m_network_max_segment;
size_t RI = (long int)cts.recomendetDataSize;
return RI;
}
size_t network_throttle::get_recommended_size_of_planned_transport() const {
size_t R1=0,R2=0,R3=0;
R1 = get_recommended_size_of_planned_transport_window( -1 );
R2 = get_recommended_size_of_planned_transport_window(m_window_size / 2);
R3 = get_recommended_size_of_planned_transport_window( 5 );
auto RM = std::min(R1, std::min(R2,R3));
const double a1=20, a2=10, a3=10, am=10; // weight of the various windows in decisssion // TODO 70 => 20
return (R1*a1 + R2*a2 + R3*a3 + RM*am) / (a1+a2+a3+am);
}
double network_throttle::get_current_speed() const {
unsigned int bytes_transferred = 0;
if (m_history.size() == 0 || m_slot_size == 0)
return 0;
auto it = m_history.begin();
while (it < m_history.end() - 1)
{
bytes_transferred += it->m_size;
it ++;
}
return bytes_transferred / ((m_history.size() - 1) * m_slot_size);
}
} // namespace
} // namespace

131
src/p2p/network_throttle-detail.hpp Normal file
View file

@ -0,0 +1,131 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief implementaion for throttling of connection (count and rate-limit speed etc)
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
/* rfree: throttle details, implementing rate limiting */
#ifndef INCLUDED_src_p2p_throttle_detail_hpp
#define INCLUDED_src_p2p_throttle_detail_hpp
#include "../../src/p2p/network_throttle.hpp"
namespace epee
{
namespace net_utils
{
class network_throttle : public i_network_throttle {
private:
struct packet_info {
size_t m_size; // octets sent. Summary for given small-window (e.g. for all packaged in 1 second)
packet_info();
};
network_speed_kbps m_target_speed;
network_speed_kbps m_real_target_speed;
size_t m_network_add_cost; // estimated add cost of headers
size_t m_network_minimal_segment; // estimated minimal cost of sending 1 byte to round up to
size_t m_network_max_segment; // recommended max size of 1 TCP transmission
const size_t m_window_size; // the number of samples to average over
network_time_seconds m_slot_size; // the size of one slot. TODO: now hardcoded for 1 second e.g. in time_to_slot()
// TODO for big window size, for performance better the substract on change of m_last_sample_time instead of recalculating average of eg >100 elements
std::vector< packet_info > m_history; // the history of bw usage
network_time_seconds m_last_sample_time; // time of last history[0] - so we know when to rotate the buffer
network_time_seconds m_start_time; // when we were created
bool m_any_packet_yet; // did we yet got any packet to count
double m_overheat; // last overheat
double m_overheat_time; // time in seconds after epoch
std::string m_name; // my name for debug and logs
std::string m_nameshort; // my name for debug and logs (used in log file name)
// each sample is now 1 second
public:
network_throttle(const std::string &nameshort, const std::string &name, int window_size=-1);
virtual ~network_throttle();
virtual void set_name(const std::string &name);
virtual void set_target_speed( network_speed_kbps target );
virtual void set_real_target_speed( network_speed_kbps real_target ); // only for throttle_out
virtual network_speed_kbps get_terget_speed();
// add information about events:
virtual void handle_trafic_exact(size_t packet_size); ///< count the new traffic/packet; the size is exact considering all network costs
virtual void handle_trafic_tcp(size_t packet_size); ///< count the new traffic/packet; the size is as TCP, we will consider MTU etc
virtual void tick(); ///< poke and update timers/history (recalculates, moves the history if needed, checks the real clock etc)
virtual double get_time_seconds() const ; ///< timer that we use, time in seconds, monotionic
// time calculations:
virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const; ///< MAIN LOGIC (see base class for info)
virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size); ///< increase the timer if needed, and get the package size
virtual network_time_seconds get_sleep_time(size_t packet_size) const; ///< gets the Delay (recommended Delay time) from calc. (not safe: only if time didnt change?) TODO
virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported
virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const; ///< ditto, but for given windows time frame
virtual double get_current_speed() const;
private:
virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13
virtual void _handle_trafic_exact(size_t packet_size, size_t orginal_size);
virtual void logger_handle_net(const std::string &filename, double time, size_t size);
};
/***
* The complete set of traffic throttle for one typical connection
*/
struct network_throttle_bw {
public:
network_throttle m_in; ///< for incomming traffic (this we can not controll directly as it depends of what others send to us - usually)
network_throttle m_inreq; ///< for requesting incomming traffic (this is exact usually)
network_throttle m_out; ///< for outgoing traffic that we just sent (this is exact usually)
public:
network_throttle_bw(const std::string &name1);
};
} // namespace net_utils
} // namespace epee
#endif

120
src/p2p/network_throttle.cpp Normal file
View file

@ -0,0 +1,120 @@
/**
@file
@author rfree (current maintainer in monero.cc project)
@brief interface for throttling of connection (count and rate-limit speed etc)
@details <PRE>
Throttling work by:
1) taking note of all traffic (hooks added e.g. to connection class) and measuring speed
2) depending on that information we sleep before sending out data (or send smaller portions of data)
3) depending on the information we can also sleep before sending requests or ask for smaller sets of data to download
</PRE>
@image html images/net/rate1-down-1k.png
@image html images/net/rate1-down-full.png
@image html images/net/rate1-up-10k.png
@image html images/net/rate1-up-full.png
@image html images/net/rate2-down-100k.png
@image html images/net/rate2-down-10k.png
@image html images/net/rate2-down-50k.png
@image html images/net/rate2-down-full.png
@image html images/net/rate2-up-100k.png
@image html images/net/rate2-up-10k.png
@image html images/net/rate3-up-10k.png
*/
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
#include "../../src/p2p/network_throttle-detail.hpp"
namespace epee
{
namespace net_utils
{
// ================================================================================================
// network_throttle_manager
// ================================================================================================
// ================================================================================================
// static:
std::mutex network_throttle_manager::m_lock_get_global_throttle_in;
std::mutex network_throttle_manager::m_lock_get_global_throttle_inreq;
std::mutex network_throttle_manager::m_lock_get_global_throttle_out;
int network_throttle_manager::xxx;
// ================================================================================================
// methods:
i_network_throttle & network_throttle_manager::get_global_throttle_in() {
std::call_once(m_once_get_global_throttle_in, [] { m_obj_get_global_throttle_in.reset(new network_throttle("in/all","<<< global-IN",10)); } );
return * m_obj_get_global_throttle_in;
}
std::once_flag network_throttle_manager::m_once_get_global_throttle_in;
std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_in;
i_network_throttle & network_throttle_manager::get_global_throttle_inreq() {
std::call_once(m_once_get_global_throttle_inreq, [] { m_obj_get_global_throttle_inreq.reset(new network_throttle("inreq/all", "<== global-IN-REQ",10)); } );
return * m_obj_get_global_throttle_inreq;
}
std::once_flag network_throttle_manager::m_once_get_global_throttle_inreq;
std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_inreq;
i_network_throttle & network_throttle_manager::get_global_throttle_out() {
std::call_once(m_once_get_global_throttle_out, [] { m_obj_get_global_throttle_out.reset(new network_throttle("out/all", ">>> global-OUT",10)); } );
return * m_obj_get_global_throttle_out;
}
std::once_flag network_throttle_manager::m_once_get_global_throttle_out;
std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_out;
network_throttle_bw::network_throttle_bw(const std::string &name1)
: m_in("in/"+name1, name1+"-DOWNLOAD"), m_inreq("inreq/"+name1, name1+"-DOWNLOAD-REQUESTS"), m_out("out/"+name1, name1+"-UPLOAD")
{ }
} // namespace
} // namespace

185
src/p2p/network_throttle.hpp Normal file
View file

@ -0,0 +1,185 @@
/// @file
/// @author rfree (current maintainer in monero.cc project)
/// @brief interface for throttling of connection (count and rate-limit speed etc)
// Copyright (c) 2014, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
/* rfree: throttle basic interface */
/* rfree: also includes the manager for singeton/global such objects */
#ifndef INCLUDED_p2p_network_throttle_hpp
#define INCLUDED_p2p_network_throttle_hpp
#include <boost/asio.hpp>
#include <string>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <atomic>
#include <boost/asio.hpp>
#include <boost/array.hpp>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/interprocess/detail/atomic.hpp>
#include <boost/thread/thread.hpp>
#include "syncobj.h"
#include "../../contrib/epee/include/net/net_utils_base.h"
#include "../../contrib/epee/include/misc_log_ex.h"
#include <boost/lambda/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/lambda/lambda.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/chrono.hpp>
#include <boost/utility/value_init.hpp>
#include <boost/asio/deadline_timer.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/thread/thread.hpp>
#include "misc_language.h"
#include "pragma_comp_defs.h"
#include <sstream>
#include <iomanip>
#include <algorithm>
#include <memory>
#include <mutex>
#include <fstream>
namespace epee
{
namespace net_utils
{
// just typedefs to in code define the units used. TODO later it will be enforced that casts to other numericals are only explicit to avoid mistakes? use boost::chrono?
typedef double network_speed_kbps;
typedef double network_time_seconds;
typedef double network_MB;
class i_network_throttle;
/***
@brief All information about given throttle - speed calculations
*/
struct calculate_times_struct {
double average;
double window;
double delay;
double recomendetDataSize;
};
typedef calculate_times_struct calculate_times_struct;
namespace cryptonote { class cryptonote_protocol_handler_base; } // a friend class // TODO friend not working
/***
@brief Access to simple throttles, with singlton to access global network limits
*/
class network_throttle_manager {
// provides global (singleton) in/inreq/out throttle access
// [[note1]] see also http://www.nuonsoft.com/blog/2012/10/21/implementing-a-thread-safe-singleton-with-c11/
// [[note2]] _inreq is the requested in traffic - we anticipate we will get in-bound traffic soon as result of what we do (e.g. that we sent network downloads requests)
//protected:
public: // XXX
// [[note1]]
static std::once_flag m_once_get_global_throttle_in;
static std::once_flag m_once_get_global_throttle_inreq; // [[note2]]
static std::once_flag m_once_get_global_throttle_out;
static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_in;
static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_inreq;
static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_out;
static std::mutex m_lock_get_global_throttle_in;
static std::mutex m_lock_get_global_throttle_inreq;
static std::mutex m_lock_get_global_throttle_out;
friend class cryptonote::cryptonote_protocol_handler_base; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS!
friend class connection_basic; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS!
friend class connection_basic_pimpl; // ditto
static int xxx;
public:
static i_network_throttle & get_global_throttle_in(); ///< singleton ; for friend class ; caller MUST use proper locks! like m_lock_get_global_throttle_in
static i_network_throttle & get_global_throttle_inreq(); ///< ditto ; use lock ... use m_lock_get_global_throttle_inreq obviously
static i_network_throttle & get_global_throttle_out(); ///< ditto ; use lock ... use m_lock_get_global_throttle_out obviously
};
/***
@brief interface for the throttle, see the derivated class
*/
class i_network_throttle {
public:
virtual void set_name(const std::string &name)=0;
virtual void set_target_speed( network_speed_kbps target )=0;
virtual void set_real_target_speed(network_speed_kbps real_target)=0;
virtual network_speed_kbps get_terget_speed()=0;
virtual void handle_trafic_exact(size_t packet_size) =0; // count the new traffic/packet; the size is exact considering all network costs
virtual void handle_trafic_tcp(size_t packet_size) =0; // count the new traffic/packet; the size is as TCP, we will consider MTU etc
virtual void tick() =0; // poke and update timers/history
// time calculations:
virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const =0; // assuming sending new package (or 0), calculate:
// Average, Window, Delay, Recommended data size ; also gets dbg=debug flag, and forced widnow size if >0 or -1 for not forcing window size
// Average speed, Window size, recommended Delay to sleep now, Recommended size of data to send now
virtual network_time_seconds get_sleep_time(size_t packet_size) const =0; // gets the D (recommended Delay time) from calc
virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size) =0; // ditto, but first tick the timer
virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future
virtual double get_time_seconds() const =0; // a timer
virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0;
};
// ... more in the -advanced.h file
} // namespace net_utils
} // namespace epee
#endif

1
src/rpc/CMakeLists.txt
View file

@ -45,6 +45,7 @@ bitmonero_add_library(rpc
target_link_libraries(rpc
LINK_PRIVATE
cryptonote_core
cryptonote_protocol
${Boost_CHRONO_LIBRARY}
${Boost_REGEX_LIBRARY}
${Boost_SYSTEM_LIBRARY}

39
src/rpc/core_rpc_server.cpp
View file

@ -753,6 +753,45 @@ namespace cryptonote
return true;
}
//------------------------------------------------------------------------------------------------------------------------------
bool core_rpc_server::on_fast_exit(const COMMAND_RPC_FAST_EXIT::request& req, COMMAND_RPC_FAST_EXIT::response& res)
{
cryptonote::core::set_fast_exit();
m_p2p.deinit();
m_core.deinit();
return true;
}
//------------------------------------------------------------------------------------------------------------------------------
bool core_rpc_server::on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res)
{
// TODO
/*if (m_p2p.get_outgoing_connections_count() > req.out_peers)
{
m_p2p.m_config.m_net_config.connections_count = req.out_peers;
if (m_p2p.get_outgoing_connections_count() > req.out_peers)
{
int count = m_p2p.get_outgoing_connections_count() - req.out_peers;
m_p2p.delete_connections(count);
}
}
else
m_p2p.m_config.m_net_config.connections_count = req.out_peers;
*/
return true;
}
//------------------------------------------------------------------------------------------------------------------------------
bool core_rpc_server::on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res)
{
m_p2p.set_save_graph(true);
return true;
}
//------------------------------------------------------------------------------------------------------------------------------
bool core_rpc_server::on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res)
{
m_p2p.set_save_graph(false);
return true;
}
//------------------------------------------------------------------------------------------------------------------------------
const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_bind_ip = {
"rpc-bind-ip"

8
src/rpc/core_rpc_server.h
View file

@ -87,6 +87,10 @@ namespace cryptonote
MAP_URI_AUTO_JON2("/get_transaction_pool", on_get_transaction_pool, COMMAND_RPC_GET_TRANSACTION_POOL)
MAP_URI_AUTO_JON2("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON)
MAP_URI_AUTO_JON2("/getinfo", on_get_info, COMMAND_RPC_GET_INFO)
MAP_URI_AUTO_JON2("/fast_exit", on_fast_exit, COMMAND_RPC_FAST_EXIT)
MAP_URI_AUTO_JON2("/out_peers", on_out_peers, COMMAND_RPC_OUT_PEERS)
MAP_URI_AUTO_JON2("/start_save_graph", on_start_save_graph, COMMAND_RPC_START_SAVE_GRAPH)
MAP_URI_AUTO_JON2("/stop_save_graph", on_stop_save_graph, COMMAND_RPC_STOP_SAVE_GRAPH)
BEGIN_JSON_RPC_MAP("/json_rpc")
MAP_JON_RPC("getblockcount", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT)
MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH)
@ -116,6 +120,10 @@ namespace cryptonote
bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res);
bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res);
bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res);
bool on_fast_exit(const COMMAND_RPC_FAST_EXIT::request& req, COMMAND_RPC_FAST_EXIT::response& res);
bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res);
bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res);
bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res);
//json_rpc
bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res);

74
src/rpc/core_rpc_server_commands_defs.h
View file

@ -703,5 +703,79 @@ namespace cryptonote
END_KV_SERIALIZE_MAP()
};
};
struct COMMAND_RPC_FAST_EXIT
{
struct request
{
BEGIN_KV_SERIALIZE_MAP()
END_KV_SERIALIZE_MAP()
};
struct response
{
std::string status;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(status)
END_KV_SERIALIZE_MAP()
};
};
struct COMMAND_RPC_OUT_PEERS
{
struct request
{
uint64_t out_peers;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(out_peers)
END_KV_SERIALIZE_MAP()
};
struct response
{
std::string status;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(status)
END_KV_SERIALIZE_MAP()
};
};
struct COMMAND_RPC_START_SAVE_GRAPH
{
struct request
{
BEGIN_KV_SERIALIZE_MAP()
END_KV_SERIALIZE_MAP()
};
struct response
{
std::string status;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(status)
END_KV_SERIALIZE_MAP()
};
};
struct COMMAND_RPC_STOP_SAVE_GRAPH
{
struct request
{
BEGIN_KV_SERIALIZE_MAP()
END_KV_SERIALIZE_MAP()
};
struct response
{
std::string status;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(status)
END_KV_SERIALIZE_MAP()
};
};
}

4
src/serialization/binary_archive.h
View file

@ -42,8 +42,8 @@
#include "warnings.h"
/* I have no clue what these lines means */
PUSH_WARNINGS;
DISABLE_VS_WARNINGS(4244);
PUSH_WARNINGS
DISABLE_VS_WARNINGS(4244)
//TODO: fix size_t warning in x32 platform

1
src/simplewallet/CMakeLists.txt
View file

@ -50,6 +50,7 @@ target_link_libraries(simplewallet
crypto
common
mnemonics
p2p
${UNBOUND_LIBRARY}
${UPNP_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}

2
src/simplewallet/simplewallet.cpp
View file

@ -67,6 +67,8 @@ namespace po = boost::program_options;
#define EXTENDED_LOGS_FILE "wallet_details.log"
unsigned int epee::g_test_dbg_lock_sleep = 0;
#define DEFAULT_MIX 3
namespace

2
tests/core_proxy/CMakeLists.txt
View file

@ -38,6 +38,8 @@ add_executable(core_proxy
target_link_libraries(core_proxy
LINK_PRIVATE
cryptonote_core
cryptonote_protocol
p2p
${UPNP_LIBRARIES}
${Boost_CHRONO_LIBRARY}
${Boost_FILESYSTEM_LIBRARY}

3
tests/core_proxy/core_proxy.cpp
View file

@ -62,6 +62,8 @@ using namespace crypto;
BOOST_CLASS_VERSION(nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<tests::proxy_core> >, 1);
unsigned int epee::g_test_dbg_lock_sleep = 0;
int main(int argc, char* argv[])
{
@ -147,6 +149,7 @@ int main(int argc, char* argv[])
LOG_PRINT("Node stopped.", LOG_LEVEL_0);
epee::net_utils::data_logger::get_instance().kill_instance();
return 0;
CATCH_ENTRY_L0("main", 1);

3
tests/core_proxy/core_proxy.h
View file

@ -81,5 +81,8 @@ namespace tests
bool on_idle(){return true;}
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;}
bool handle_get_objects(cryptonote::NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote::cryptonote_connection_context& context){return true;}
cryptonote::blockchain_storage &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class proxy_core."); }
bool get_test_drop_download() {return true;}
bool get_test_drop_download_height() {return true;}
};
}

2
tests/core_tests/CMakeLists.txt
View file

@ -60,6 +60,8 @@ add_executable(coretests
target_link_libraries(coretests
LINK_PRIVATE
cryptonote_core
p2p
${Boost_CHRONO_LIBRARY}
${Boost_FILESYSTEM_LIBRARY}
${Boost_SYSTEM_LIBRARY}
${CMAKE_THREAD_LIBS_INIT}

2
tests/core_tests/chaingen_main.cpp
View file

@ -44,6 +44,8 @@ namespace
const command_line::arg_descriptor<bool> arg_test_transactions = {"test_transactions", ""};
}
unsigned int epee::g_test_dbg_lock_sleep = 0;
int main(int argc, char* argv[])
{
TRY_ENTRY();

2
tests/functional_tests/main.cpp
View file

@ -55,6 +55,8 @@ namespace
const command_line::arg_descriptor<size_t> arg_test_repeat_count = {"test_repeat_count", "", 1};
}
unsigned int epee::g_test_dbg_lock_sleep = 0;
int main(int argc, char* argv[])
{
TRY_ENTRY();

6
tests/net_load_tests/CMakeLists.txt
View file

@ -37,6 +37,9 @@ add_executable(net_load_tests_clt
${clt_headers})
target_link_libraries(net_load_tests_clt
LINK_PRIVATE
otshell_utils
p2p
cryptonote_core
${GTEST_MAIN_LIBRARIES}
${Boost_CHRONO_LIBRARY}
${Boost_DATE_TIME_LIBRARY}
@ -56,6 +59,9 @@ add_executable(net_load_tests_srv
${srv_headers})
target_link_libraries(net_load_tests_srv
LINK_PRIVATE
otshell_utils
p2p
cryptonote_core
${GTEST_MAIN_LIBRARIES}
${Boost_CHRONO_LIBRARY}
${Boost_DATE_TIME_LIBRARY}

6
tests/net_load_tests/clt.cpp
View file

@ -44,7 +44,10 @@
#include "net_load_tests.h"
#include "../../contrib/otshell_utils/utils.hpp"
using namespace net_load_tests;
using namespace nOT::nUtils;
namespace
{
@ -620,6 +623,8 @@ TEST_F(net_load_test_clt, permament_open_and_close_and_connections_closed_by_ser
ASSERT_EQ(RESERVED_CONN_CNT, m_tcp_server.get_config_object().get_connections_count());
}
unsigned int epee::g_test_dbg_lock_sleep = 0;
int main(int argc, char** argv)
{
epee::debug::get_set_enable_assert(true, false);
@ -628,5 +633,6 @@ int main(int argc, char** argv)
epee::log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL);
::testing::InitGoogleTest(&argc, argv);
epee::net_utils::data_logger::get_instance().kill_instance();
return RUN_ALL_TESTS();
}

4
tests/net_load_tests/srv.cpp
View file

@ -213,6 +213,8 @@ namespace
};
}
unsigned int epee::g_test_dbg_lock_sleep = 0;
int main(int argc, char** argv)
{
//set up logging options
@ -232,6 +234,6 @@ int main(int argc, char** argv)
if (!tcp_server.run_server(thread_count, true))
return 2;
epee::net_utils::data_logger::get_instance().kill_instance();
return 0;
}

2
tests/performance_tests/main.cpp
View file

@ -42,6 +42,8 @@
#include "generate_key_image_helper.h"
#include "is_out_to_acc.h"
unsigned int epee::g_test_dbg_lock_sleep = 0;
int main(int argc, char** argv)
{
set_process_affinity(1);

1
tests/unit_tests/CMakeLists.txt
View file

@ -58,6 +58,7 @@ target_link_libraries(unit_tests
cryptonote_core
rpc
wallet
p2p
${GTEST_MAIN_LIBRARIES}
${Boost_CHRONO_LIBRARY}
${Boost_REGEX_LIBRARY}

2
tests/unit_tests/main.cpp
View file

@ -32,6 +32,8 @@
#include "include_base_utils.h"
unsigned int epee::g_test_dbg_lock_sleep = 0;
int main(int argc, char** argv)
{
epee::debug::get_set_enable_assert(true, false);

25
tools/doxygen-publish.sh Executable file
View file

@ -0,0 +1,25 @@
#!/bin/bash -e
# maintainer (ask me any questions): rfree
if [[ ! -r "Doxyfile" ]] ; then
echo "Error, can not read the Doxyfile - make sure to run this script from top of monero project, where the Doxyfile file is located"
exit 1
fi
wwwdir="$HOME/monero-www/"
if [[ ! -w "$wwwdir" ]] ; then
echo "Error, can not write into wwwdir=$wwwdir. It should be a directory readable/connected to your webserver, or a symlink to such directory"
exit 1
fi
if [[ ! -d "$wwwdir/doc" ]] ; then
echo "Creating subdirs"
mkdir "$wwwdir/doc"
fi
echo "Generating:"
doxygen Doxyfile && echo "Backup previous version:" && rm -rf ~/monero-www-previous && mv "$wwwdir/doc" ~/monero-www-previous && cp -ar doc/ "$wwwdir/" && echo "Done, builded and copied to public - the doxygen docs" && echo "size:" && du -Dsh "$wwwdir/" && echo "files:" && find "$wwwdir/" | wc -l