2015-01-02 16:52:46 +00:00
|
|
|
// Copyright (c) 2014-2015, The Monero Project
|
2014-07-23 13:03:52 +00:00
|
|
|
//
|
|
|
|
// All rights reserved.
|
|
|
|
//
|
|
|
|
// Redistribution and use in source and binary forms, with or without modification, are
|
|
|
|
// permitted provided that the following conditions are met:
|
|
|
|
//
|
|
|
|
// 1. Redistributions of source code must retain the above copyright notice, this list of
|
|
|
|
// conditions and the following disclaimer.
|
|
|
|
//
|
|
|
|
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
|
|
|
|
// of conditions and the following disclaimer in the documentation and/or other
|
|
|
|
// materials provided with the distribution.
|
|
|
|
//
|
|
|
|
// 3. Neither the name of the copyright holder nor the names of its contributors may be
|
|
|
|
// used to endorse or promote products derived from this software without specific
|
|
|
|
// prior written permission.
|
|
|
|
//
|
|
|
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
|
|
|
|
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
|
|
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
|
|
|
|
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
|
|
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
|
|
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
|
|
|
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
|
|
|
|
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
//
|
|
|
|
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
|
2014-03-03 22:07:58 +00:00
|
|
|
|
|
|
|
#pragma once
|
|
|
|
#include <boost/thread.hpp>
|
|
|
|
#include <boost/bind.hpp>
|
|
|
|
#include <boost/foreach.hpp>
|
|
|
|
#include <boost/bimap.hpp>
|
|
|
|
#include <boost/multi_index_container.hpp>
|
|
|
|
#include <boost/multi_index/ordered_index.hpp>
|
|
|
|
#include <boost/multi_index/identity.hpp>
|
|
|
|
#include <boost/multi_index/member.hpp>
|
|
|
|
#include <boost/multi_index/global_fun.hpp>
|
|
|
|
#include <boost/program_options/options_description.hpp>
|
|
|
|
#include <boost/program_options/variables_map.hpp>
|
|
|
|
#include <boost/serialization/version.hpp>
|
2014-09-08 17:40:28 +00:00
|
|
|
#include <boost/uuid/uuid.hpp>
|
2014-03-03 22:07:58 +00:00
|
|
|
|
2014-09-05 02:14:36 +00:00
|
|
|
#include "cryptonote_config.h"
|
2014-03-03 22:07:58 +00:00
|
|
|
#include "warnings.h"
|
|
|
|
#include "net/levin_server_cp2.h"
|
|
|
|
#include "p2p_protocol_defs.h"
|
|
|
|
#include "storages/levin_abstract_invoke2.h"
|
|
|
|
#include "net_peerlist.h"
|
|
|
|
#include "math_helper.h"
|
|
|
|
#include "net_node_common.h"
|
2014-05-25 17:06:40 +00:00
|
|
|
#include "common/command_line.h"
|
2014-03-03 22:07:58 +00:00
|
|
|
|
|
|
|
PUSH_WARNINGS
|
|
|
|
DISABLE_VS_WARNINGS(4355)
|
|
|
|
|
|
|
|
namespace nodetool
|
|
|
|
{
|
|
|
|
template<class base_type>
|
|
|
|
struct p2p_connection_context_t: base_type //t_payload_net_handler::connection_context //public net_utils::connection_context_base
|
|
|
|
{
|
|
|
|
peerid_type peer_id;
|
|
|
|
};
|
|
|
|
|
|
|
|
template<class t_payload_net_handler>
|
2014-05-25 17:06:40 +00:00
|
|
|
class node_server: public epee::levin::levin_commands_handler<p2p_connection_context_t<typename t_payload_net_handler::connection_context> >,
|
2014-03-03 22:07:58 +00:00
|
|
|
public i_p2p_endpoint<typename t_payload_net_handler::connection_context>
|
|
|
|
{
|
|
|
|
struct by_conn_id{};
|
|
|
|
struct by_peer_id{};
|
|
|
|
struct by_addr{};
|
|
|
|
|
|
|
|
typedef p2p_connection_context_t<typename t_payload_net_handler::connection_context> p2p_connection_context;
|
|
|
|
|
|
|
|
typedef COMMAND_HANDSHAKE_T<typename t_payload_net_handler::payload_type> COMMAND_HANDSHAKE;
|
|
|
|
typedef COMMAND_TIMED_SYNC_T<typename t_payload_net_handler::payload_type> COMMAND_TIMED_SYNC;
|
|
|
|
|
|
|
|
public:
|
|
|
|
typedef t_payload_net_handler payload_net_handler;
|
2014-09-08 17:40:28 +00:00
|
|
|
|
2015-04-01 17:00:45 +00:00
|
|
|
node_server(t_payload_net_handler& payload_handler)
|
|
|
|
:m_payload_handler(payload_handler),
|
2015-05-06 16:10:51 +00:00
|
|
|
m_current_number_of_out_peers(0),
|
2015-04-01 17:00:45 +00:00
|
|
|
m_allow_local_ip(false),
|
2015-04-08 17:54:07 +00:00
|
|
|
m_hide_my_port(false),
|
2015-05-06 16:10:51 +00:00
|
|
|
m_no_igd(false),
|
|
|
|
m_save_graph(false),
|
|
|
|
is_closing(false),
|
2015-04-08 17:54:07 +00:00
|
|
|
m_net_server( epee::net_utils::e_connection_type_P2P ) // this is a P2P connection of the main p2p node server, because this is class node_server<>
|
2014-03-03 22:07:58 +00:00
|
|
|
{}
|
2015-04-20 17:17:11 +00:00
|
|
|
virtual ~node_server()
|
|
|
|
{}
|
2014-03-03 22:07:58 +00:00
|
|
|
|
|
|
|
static void init_options(boost::program_options::options_description& desc);
|
|
|
|
|
|
|
|
bool run();
|
2015-01-29 22:10:53 +00:00
|
|
|
bool init(const boost::program_options::variables_map& vm);
|
2014-03-03 22:07:58 +00:00
|
|
|
bool deinit();
|
|
|
|
bool send_stop_signal();
|
|
|
|
uint32_t get_this_peer_port(){return m_listenning_port;}
|
|
|
|
t_payload_net_handler& get_payload_object();
|
|
|
|
|
|
|
|
template <class Archive, class t_version_type>
|
|
|
|
void serialize(Archive &a, const t_version_type ver)
|
|
|
|
{
|
|
|
|
a & m_peerlist;
|
|
|
|
a & m_config.m_peer_id;
|
|
|
|
}
|
|
|
|
// debug functions
|
|
|
|
bool log_peerlist();
|
|
|
|
bool log_connections();
|
|
|
|
virtual uint64_t get_connections_count();
|
|
|
|
size_t get_outgoing_connections_count();
|
|
|
|
peerlist_manager& get_peerlist_manager(){return m_peerlist;}
|
2015-01-05 19:30:17 +00:00
|
|
|
void delete_connections(size_t count);
|
2014-03-03 22:07:58 +00:00
|
|
|
private:
|
2014-09-17 19:35:52 +00:00
|
|
|
const std::vector<std::string> m_seed_nodes_list =
|
|
|
|
{ "seeds.moneroseeds.se"
|
|
|
|
, "seeds.moneroseeds.ae.org"
|
|
|
|
, "seeds.moneroseeds.ch"
|
|
|
|
, "seeds.moneroseeds.li"
|
|
|
|
};
|
2015-01-05 19:30:17 +00:00
|
|
|
|
|
|
|
bool islimitup=false;
|
|
|
|
bool islimitdown=false;
|
2014-09-17 19:35:52 +00:00
|
|
|
|
2014-03-03 22:07:58 +00:00
|
|
|
typedef COMMAND_REQUEST_STAT_INFO_T<typename t_payload_net_handler::stat_info> COMMAND_REQUEST_STAT_INFO;
|
|
|
|
|
|
|
|
CHAIN_LEVIN_INVOKE_MAP2(p2p_connection_context); //move levin_commands_handler interface invoke(...) callbacks into invoke map
|
|
|
|
CHAIN_LEVIN_NOTIFY_MAP2(p2p_connection_context); //move levin_commands_handler interface notify(...) callbacks into nothing
|
|
|
|
|
|
|
|
BEGIN_INVOKE_MAP2(node_server)
|
|
|
|
HANDLE_INVOKE_T2(COMMAND_HANDSHAKE, &node_server::handle_handshake)
|
|
|
|
HANDLE_INVOKE_T2(COMMAND_TIMED_SYNC, &node_server::handle_timed_sync)
|
|
|
|
HANDLE_INVOKE_T2(COMMAND_PING, &node_server::handle_ping)
|
|
|
|
#ifdef ALLOW_DEBUG_COMMANDS
|
|
|
|
HANDLE_INVOKE_T2(COMMAND_REQUEST_STAT_INFO, &node_server::handle_get_stat_info)
|
|
|
|
HANDLE_INVOKE_T2(COMMAND_REQUEST_NETWORK_STATE, &node_server::handle_get_network_state)
|
|
|
|
HANDLE_INVOKE_T2(COMMAND_REQUEST_PEER_ID, &node_server::handle_get_peer_id)
|
|
|
|
#endif
|
|
|
|
CHAIN_INVOKE_MAP_TO_OBJ_FORCE_CONTEXT(m_payload_handler, typename t_payload_net_handler::connection_context&)
|
|
|
|
END_INVOKE_MAP2()
|
|
|
|
|
|
|
|
//----------------- commands handlers ----------------------------------------------
|
|
|
|
int handle_handshake(int command, typename COMMAND_HANDSHAKE::request& arg, typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context);
|
|
|
|
int handle_timed_sync(int command, typename COMMAND_TIMED_SYNC::request& arg, typename COMMAND_TIMED_SYNC::response& rsp, p2p_connection_context& context);
|
|
|
|
int handle_ping(int command, COMMAND_PING::request& arg, COMMAND_PING::response& rsp, p2p_connection_context& context);
|
|
|
|
#ifdef ALLOW_DEBUG_COMMANDS
|
|
|
|
int handle_get_stat_info(int command, typename COMMAND_REQUEST_STAT_INFO::request& arg, typename COMMAND_REQUEST_STAT_INFO::response& rsp, p2p_connection_context& context);
|
|
|
|
int handle_get_network_state(int command, COMMAND_REQUEST_NETWORK_STATE::request& arg, COMMAND_REQUEST_NETWORK_STATE::response& rsp, p2p_connection_context& context);
|
|
|
|
int handle_get_peer_id(int command, COMMAND_REQUEST_PEER_ID::request& arg, COMMAND_REQUEST_PEER_ID::response& rsp, p2p_connection_context& context);
|
|
|
|
#endif
|
|
|
|
bool init_config();
|
|
|
|
bool make_default_config();
|
|
|
|
bool store_config();
|
|
|
|
bool check_trust(const proof_of_trust& tr);
|
|
|
|
|
|
|
|
|
|
|
|
//----------------- levin_commands_handler -------------------------------------------------------------
|
|
|
|
virtual void on_connection_new(p2p_connection_context& context);
|
|
|
|
virtual void on_connection_close(p2p_connection_context& context);
|
|
|
|
virtual void callback(p2p_connection_context& context);
|
|
|
|
//----------------- i_p2p_endpoint -------------------------------------------------------------
|
|
|
|
virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context);
|
|
|
|
virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context);
|
|
|
|
virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context);
|
|
|
|
virtual bool drop_connection(const epee::net_utils::connection_context_base& context);
|
|
|
|
virtual void request_callback(const epee::net_utils::connection_context_base& context);
|
2014-03-20 11:46:11 +00:00
|
|
|
virtual void for_each_connection(std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type)> f);
|
2014-03-03 22:07:58 +00:00
|
|
|
//-----------------------------------------------------------------------------------------------
|
|
|
|
bool parse_peer_from_string(nodetool::net_address& pe, const std::string& node_addr);
|
2014-09-08 16:51:04 +00:00
|
|
|
bool handle_command_line(
|
|
|
|
const boost::program_options::variables_map& vm
|
|
|
|
, bool testnet
|
|
|
|
);
|
2014-03-03 22:07:58 +00:00
|
|
|
bool idle_worker();
|
2014-05-25 17:06:40 +00:00
|
|
|
bool handle_remote_peerlist(const std::list<peerlist_entry>& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context);
|
2014-03-03 22:07:58 +00:00
|
|
|
bool get_local_node_data(basic_node_data& node_data);
|
|
|
|
//bool get_local_handshake_data(handshake_data& hshd);
|
|
|
|
|
|
|
|
bool merge_peerlist_with_local(const std::list<peerlist_entry>& bs);
|
|
|
|
bool fix_time_delta(std::list<peerlist_entry>& local_peerlist, time_t local_time, int64_t& delta);
|
|
|
|
|
|
|
|
bool connections_maker();
|
|
|
|
bool peer_sync_idle_maker();
|
|
|
|
bool do_handshake_with_peer(peerid_type& pi, p2p_connection_context& context, bool just_take_peerlist = false);
|
2014-05-25 17:06:40 +00:00
|
|
|
bool do_peer_timed_sync(const epee::net_utils::connection_context_base& context, peerid_type peer_id);
|
2014-03-03 22:07:58 +00:00
|
|
|
|
|
|
|
bool make_new_connection_from_peerlist(bool use_white_list);
|
|
|
|
bool try_to_connect_and_handshake_with_new_peer(const net_address& na, bool just_take_peerlist = false, uint64_t last_seen_stamp = 0, bool white = true);
|
|
|
|
size_t get_random_index_with_fixed_probability(size_t max_index);
|
|
|
|
bool is_peer_used(const peerlist_entry& peer);
|
|
|
|
bool is_addr_connected(const net_address& peer);
|
|
|
|
template<class t_callback>
|
|
|
|
bool try_ping(basic_node_data& node_data, p2p_connection_context& context, t_callback cb);
|
|
|
|
bool make_expected_connections_count(bool white_list, size_t expected_connections);
|
2014-05-25 17:06:40 +00:00
|
|
|
bool is_priority_node(const net_address& na);
|
|
|
|
|
|
|
|
template <class Container>
|
|
|
|
bool connect_to_peerlist(const Container& peers);
|
|
|
|
|
|
|
|
template <class Container>
|
|
|
|
bool parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor<std::vector<std::string> > & arg, Container& container);
|
2014-03-03 22:07:58 +00:00
|
|
|
|
2015-01-05 19:30:17 +00:00
|
|
|
bool set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max);
|
|
|
|
bool set_tos_flag(const boost::program_options::variables_map& vm, int limit);
|
|
|
|
|
|
|
|
bool set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit);
|
|
|
|
bool set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit);
|
** CHANGES ARE EXPERIMENTAL (FOR TESTING ONLY)
Bockchain:
1. Optim: Multi-thread long-hash computation when encountering groups of blocks.
2. Optim: Cache verified txs and return result from cache instead of re-checking whenever possible.
3. Optim: Preload output-keys when encoutering groups of blocks. Sort by amount and global-index before bulk querying database and multi-thread when possible.
4. Optim: Disable double spend check on block verification, double spend is already detected when trying to add blocks.
5. Optim: Multi-thread signature computation whenever possible.
6. Patch: Disable locking (recursive mutex) on called functions from check_tx_inputs which causes slowdowns (only seems to happen on ubuntu/VMs??? Reason: TBD)
7. Optim: Removed looped full-tx hash computation when retrieving transactions from pool (???).
8. Optim: Cache difficulty/timestamps (735 blocks) for next-difficulty calculations so that only 2 db reads per new block is needed when a new block arrives (instead of 1470 reads).
Berkeley-DB:
1. Fix: 32-bit data errors causing wrong output global indices and failure to send blocks to peers (etc).
2. Fix: Unable to pop blocks on reorganize due to transaction errors.
3. Patch: Large number of transaction aborts when running multi-threaded bulk queries.
4. Patch: Insufficient locks error when running full sync.
5. Patch: Incorrect db stats when returning from an immediate exit from "pop block" operation.
6. Optim: Add bulk queries to get output global indices.
7. Optim: Modified output_keys table to store public_key+unlock_time+height for single transaction lookup (vs 3)
8. Optim: Used output_keys table retrieve public_keys instead of going through output_amounts->output_txs+output_indices->txs->output:public_key
9. Optim: Added thread-safe buffers used when multi-threading bulk queries.
10. Optim: Added support for nosync/write_nosync options for improved performance (*see --db-sync-mode option for details)
11. Mod: Added checkpoint thread and auto-remove-logs option.
12. *Now usable on 32-bit systems like RPI2.
LMDB:
1. Optim: Added custom comparison for 256-bit key tables (minor speed-up, TBD: get actual effect)
2. Optim: Modified output_keys table to store public_key+unlock_time+height for single transaction lookup (vs 3)
3. Optim: Used output_keys table retrieve public_keys instead of going through output_amounts->output_txs+output_indices->txs->output:public_key
4. Optim: Added support for sync/writemap options for improved performance (*see --db-sync-mode option for details)
5. Mod: Auto resize to +1GB instead of multiplier x1.5
ETC:
1. Minor optimizations for slow-hash for ARM (RPI2). Incomplete.
2. Fix: 32-bit saturation bug when computing next difficulty on large blocks.
[PENDING ISSUES]
1. Berkely db has a very slow "pop-block" operation. This is very noticeable on the RPI2 as it sometimes takes > 10 MINUTES to pop a block during reorganization.
This does not happen very often however, most reorgs seem to take a few seconds but it possibly depends on the number of outputs present. TBD.
2. Berkeley db, possible bug "unable to allocate memory". TBD.
[NEW OPTIONS] (*Currently all enabled for testing purposes)
1. --fast-block-sync arg=[0:1] (default: 1)
a. 0 = Compute long hash per block (may take a while depending on CPU)
b. 1 = Skip long-hash and verify blocks based on embedded known good block hashes (faster, minimal CPU dependence)
2. --db-sync-mode arg=[[safe|fast|fastest]:[sync|async]:[nblocks_per_sync]] (default: fastest:async:1000)
a. safe = fdatasync/fsync (or equivalent) per stored block. Very slow, but safest option to protect against power-out/crash conditions.
b. fast/fastest = Enables asynchronous fdatasync/fsync (or equivalent). Useful for battery operated devices or STABLE systems with UPS and/or systems with battery backed write cache/solid state cache.
Fast - Write meta-data but defer data flush.
Fastest - Defer meta-data and data flush.
Sync - Flush data after nblocks_per_sync and wait.
Async - Flush data after nblocks_per_sync but do not wait for the operation to finish.
3. --prep-blocks-threads arg=[n] (default: 4 or system max threads, whichever is lower)
Max number of threads to use when computing long-hash in groups.
4. --show-time-stats arg=[0:1] (default: 1)
Show benchmark related time stats.
5. --db-auto-remove-logs arg=[0:1] (default: 1)
For berkeley-db only. Auto remove logs if enabled.
**Note: lmdb and berkeley-db have changes to the tables and are not compatible with official git head version.
At the moment, you need a full resync to use this optimized version.
[PERFORMANCE COMPARISON]
**Some figures are approximations only.
Using a baseline machine of an i7-2600K+SSD+(with full pow computation):
1. The optimized lmdb/blockhain core can process blocks up to 585K for ~1.25 hours + download time, so it usually takes 2.5 hours to sync the full chain.
2. The current head with memory can process blocks up to 585K for ~4.2 hours + download time, so it usually takes 5.5 hours to sync the full chain.
3. The current head with lmdb can process blocks up to 585K for ~32 hours + download time and usually takes 36 hours to sync the full chain.
Averate procesing times (with full pow computation):
lmdb-optimized:
1. tx_ave = 2.5 ms / tx
2. block_ave = 5.87 ms / block
memory-official-repo:
1. tx_ave = 8.85 ms / tx
2. block_ave = 19.68 ms / block
lmdb-official-repo (0f4a036437fd41a5498ee5e74e2422ea6177aa3e)
1. tx_ave = 47.8 ms / tx
2. block_ave = 64.2 ms / block
**Note: The following data denotes processing times only (does not include p2p download time)
lmdb-optimized processing times (with full pow computation):
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 1.25 hours processing time (--db-sync-mode=fastest:async:1000).
2. Laptop, Dual-core / 4-threads U4200 (3Mb) - 4.90 hours processing time (--db-sync-mode=fastest:async:1000).
3. Embedded, Quad-core / 4-threads Z3735F (2x1Mb) - 12.0 hours processing time (--db-sync-mode=fastest:async:1000).
lmdb-optimized processing times (with per-block-checkpoint)
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 10 minutes processing time (--db-sync-mode=fastest:async:1000).
berkeley-db optimized processing times (with full pow computation)
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 1.8 hours processing time (--db-sync-mode=fastest:async:1000).
2. RPI2. Improved from estimated 3 months(???) into 2.5 days (*Need 2AMP supply + Clock:1Ghz + [usb+ssd] to achieve this speed) (--db-sync-mode=fastest:async:1000).
berkeley-db optimized processing times (with per-block-checkpoint)
1. RPI2. 12-15 hours (*Need 2AMP supply + Clock:1Ghz + [usb+ssd] to achieve this speed) (--db-sync-mode=fastest:async:1000).
2015-07-10 20:09:32 +00:00
|
|
|
bool set_rate_limit(const boost::program_options::variables_map& vm, int64_t limit);
|
2015-01-05 19:30:17 +00:00
|
|
|
|
2015-02-24 20:02:48 +00:00
|
|
|
void kill() { ///< will be called e.g. from deinit()
|
|
|
|
_info("Killing the net_node");
|
|
|
|
is_closing = true;
|
** CHANGES ARE EXPERIMENTAL (FOR TESTING ONLY)
Bockchain:
1. Optim: Multi-thread long-hash computation when encountering groups of blocks.
2. Optim: Cache verified txs and return result from cache instead of re-checking whenever possible.
3. Optim: Preload output-keys when encoutering groups of blocks. Sort by amount and global-index before bulk querying database and multi-thread when possible.
4. Optim: Disable double spend check on block verification, double spend is already detected when trying to add blocks.
5. Optim: Multi-thread signature computation whenever possible.
6. Patch: Disable locking (recursive mutex) on called functions from check_tx_inputs which causes slowdowns (only seems to happen on ubuntu/VMs??? Reason: TBD)
7. Optim: Removed looped full-tx hash computation when retrieving transactions from pool (???).
8. Optim: Cache difficulty/timestamps (735 blocks) for next-difficulty calculations so that only 2 db reads per new block is needed when a new block arrives (instead of 1470 reads).
Berkeley-DB:
1. Fix: 32-bit data errors causing wrong output global indices and failure to send blocks to peers (etc).
2. Fix: Unable to pop blocks on reorganize due to transaction errors.
3. Patch: Large number of transaction aborts when running multi-threaded bulk queries.
4. Patch: Insufficient locks error when running full sync.
5. Patch: Incorrect db stats when returning from an immediate exit from "pop block" operation.
6. Optim: Add bulk queries to get output global indices.
7. Optim: Modified output_keys table to store public_key+unlock_time+height for single transaction lookup (vs 3)
8. Optim: Used output_keys table retrieve public_keys instead of going through output_amounts->output_txs+output_indices->txs->output:public_key
9. Optim: Added thread-safe buffers used when multi-threading bulk queries.
10. Optim: Added support for nosync/write_nosync options for improved performance (*see --db-sync-mode option for details)
11. Mod: Added checkpoint thread and auto-remove-logs option.
12. *Now usable on 32-bit systems like RPI2.
LMDB:
1. Optim: Added custom comparison for 256-bit key tables (minor speed-up, TBD: get actual effect)
2. Optim: Modified output_keys table to store public_key+unlock_time+height for single transaction lookup (vs 3)
3. Optim: Used output_keys table retrieve public_keys instead of going through output_amounts->output_txs+output_indices->txs->output:public_key
4. Optim: Added support for sync/writemap options for improved performance (*see --db-sync-mode option for details)
5. Mod: Auto resize to +1GB instead of multiplier x1.5
ETC:
1. Minor optimizations for slow-hash for ARM (RPI2). Incomplete.
2. Fix: 32-bit saturation bug when computing next difficulty on large blocks.
[PENDING ISSUES]
1. Berkely db has a very slow "pop-block" operation. This is very noticeable on the RPI2 as it sometimes takes > 10 MINUTES to pop a block during reorganization.
This does not happen very often however, most reorgs seem to take a few seconds but it possibly depends on the number of outputs present. TBD.
2. Berkeley db, possible bug "unable to allocate memory". TBD.
[NEW OPTIONS] (*Currently all enabled for testing purposes)
1. --fast-block-sync arg=[0:1] (default: 1)
a. 0 = Compute long hash per block (may take a while depending on CPU)
b. 1 = Skip long-hash and verify blocks based on embedded known good block hashes (faster, minimal CPU dependence)
2. --db-sync-mode arg=[[safe|fast|fastest]:[sync|async]:[nblocks_per_sync]] (default: fastest:async:1000)
a. safe = fdatasync/fsync (or equivalent) per stored block. Very slow, but safest option to protect against power-out/crash conditions.
b. fast/fastest = Enables asynchronous fdatasync/fsync (or equivalent). Useful for battery operated devices or STABLE systems with UPS and/or systems with battery backed write cache/solid state cache.
Fast - Write meta-data but defer data flush.
Fastest - Defer meta-data and data flush.
Sync - Flush data after nblocks_per_sync and wait.
Async - Flush data after nblocks_per_sync but do not wait for the operation to finish.
3. --prep-blocks-threads arg=[n] (default: 4 or system max threads, whichever is lower)
Max number of threads to use when computing long-hash in groups.
4. --show-time-stats arg=[0:1] (default: 1)
Show benchmark related time stats.
5. --db-auto-remove-logs arg=[0:1] (default: 1)
For berkeley-db only. Auto remove logs if enabled.
**Note: lmdb and berkeley-db have changes to the tables and are not compatible with official git head version.
At the moment, you need a full resync to use this optimized version.
[PERFORMANCE COMPARISON]
**Some figures are approximations only.
Using a baseline machine of an i7-2600K+SSD+(with full pow computation):
1. The optimized lmdb/blockhain core can process blocks up to 585K for ~1.25 hours + download time, so it usually takes 2.5 hours to sync the full chain.
2. The current head with memory can process blocks up to 585K for ~4.2 hours + download time, so it usually takes 5.5 hours to sync the full chain.
3. The current head with lmdb can process blocks up to 585K for ~32 hours + download time and usually takes 36 hours to sync the full chain.
Averate procesing times (with full pow computation):
lmdb-optimized:
1. tx_ave = 2.5 ms / tx
2. block_ave = 5.87 ms / block
memory-official-repo:
1. tx_ave = 8.85 ms / tx
2. block_ave = 19.68 ms / block
lmdb-official-repo (0f4a036437fd41a5498ee5e74e2422ea6177aa3e)
1. tx_ave = 47.8 ms / tx
2. block_ave = 64.2 ms / block
**Note: The following data denotes processing times only (does not include p2p download time)
lmdb-optimized processing times (with full pow computation):
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 1.25 hours processing time (--db-sync-mode=fastest:async:1000).
2. Laptop, Dual-core / 4-threads U4200 (3Mb) - 4.90 hours processing time (--db-sync-mode=fastest:async:1000).
3. Embedded, Quad-core / 4-threads Z3735F (2x1Mb) - 12.0 hours processing time (--db-sync-mode=fastest:async:1000).
lmdb-optimized processing times (with per-block-checkpoint)
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 10 minutes processing time (--db-sync-mode=fastest:async:1000).
berkeley-db optimized processing times (with full pow computation)
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 1.8 hours processing time (--db-sync-mode=fastest:async:1000).
2. RPI2. Improved from estimated 3 months(???) into 2.5 days (*Need 2AMP supply + Clock:1Ghz + [usb+ssd] to achieve this speed) (--db-sync-mode=fastest:async:1000).
berkeley-db optimized processing times (with per-block-checkpoint)
1. RPI2. 12-15 hours (*Need 2AMP supply + Clock:1Ghz + [usb+ssd] to achieve this speed) (--db-sync-mode=fastest:async:1000).
2015-07-10 20:09:32 +00:00
|
|
|
if(mPeersLoggerThread != nullptr)
|
2015-02-24 20:02:48 +00:00
|
|
|
mPeersLoggerThread->join(); // make sure the thread finishes
|
|
|
|
_info("Joined extra background net_node threads");
|
|
|
|
}
|
|
|
|
|
2014-03-03 22:07:58 +00:00
|
|
|
//debug functions
|
|
|
|
std::string print_connections_container();
|
|
|
|
|
|
|
|
|
2014-05-25 17:06:40 +00:00
|
|
|
typedef epee::net_utils::boosted_tcp_server<epee::levin::async_protocol_handler<p2p_connection_context> > net_server;
|
2014-03-03 22:07:58 +00:00
|
|
|
|
|
|
|
struct config
|
|
|
|
{
|
|
|
|
network_config m_net_config;
|
|
|
|
uint64_t m_peer_id;
|
|
|
|
|
|
|
|
BEGIN_KV_SERIALIZE_MAP()
|
|
|
|
KV_SERIALIZE(m_net_config)
|
|
|
|
KV_SERIALIZE(m_peer_id)
|
|
|
|
END_KV_SERIALIZE_MAP()
|
|
|
|
};
|
|
|
|
|
2015-01-05 19:30:17 +00:00
|
|
|
public:
|
|
|
|
config m_config; // TODO was private, add getters?
|
2015-02-20 21:28:03 +00:00
|
|
|
std::atomic<unsigned int> m_current_number_of_out_peers;
|
2015-04-01 17:00:45 +00:00
|
|
|
|
2015-02-12 19:59:39 +00:00
|
|
|
void set_save_graph(bool save_graph)
|
|
|
|
{
|
|
|
|
m_save_graph = save_graph;
|
|
|
|
epee::net_utils::connection_basic::set_save_graph(save_graph);
|
|
|
|
}
|
2015-01-05 19:30:17 +00:00
|
|
|
private:
|
2014-03-03 22:07:58 +00:00
|
|
|
std::string m_config_folder;
|
|
|
|
|
|
|
|
bool m_have_address;
|
|
|
|
bool m_first_connection_maker_call;
|
|
|
|
uint32_t m_listenning_port;
|
|
|
|
uint32_t m_external_port;
|
|
|
|
uint32_t m_ip_address;
|
|
|
|
bool m_allow_local_ip;
|
|
|
|
bool m_hide_my_port;
|
2015-01-05 19:30:17 +00:00
|
|
|
bool m_no_igd;
|
2015-02-12 19:59:39 +00:00
|
|
|
std::atomic<bool> m_save_graph;
|
2015-02-24 20:02:48 +00:00
|
|
|
std::atomic<bool> is_closing;
|
|
|
|
std::unique_ptr<std::thread> mPeersLoggerThread;
|
2014-03-03 22:07:58 +00:00
|
|
|
//critical_section m_connections_lock;
|
|
|
|
//connections_indexed_container m_connections;
|
|
|
|
|
|
|
|
t_payload_net_handler& m_payload_handler;
|
|
|
|
peerlist_manager m_peerlist;
|
|
|
|
|
2014-05-25 17:06:40 +00:00
|
|
|
epee::math_helper::once_a_time_seconds<P2P_DEFAULT_HANDSHAKE_INTERVAL> m_peer_handshake_idle_maker_interval;
|
|
|
|
epee::math_helper::once_a_time_seconds<1> m_connections_maker_interval;
|
|
|
|
epee::math_helper::once_a_time_seconds<60*30, false> m_peerlist_store_interval;
|
2014-03-03 22:07:58 +00:00
|
|
|
|
|
|
|
std::string m_bind_ip;
|
|
|
|
std::string m_port;
|
|
|
|
#ifdef ALLOW_DEBUG_COMMANDS
|
|
|
|
uint64_t m_last_stat_request_time;
|
|
|
|
#endif
|
|
|
|
std::list<net_address> m_priority_peers;
|
2014-05-25 17:06:40 +00:00
|
|
|
std::vector<net_address> m_exclusive_peers;
|
2014-03-03 22:07:58 +00:00
|
|
|
std::vector<net_address> m_seed_nodes;
|
|
|
|
std::list<nodetool::peerlist_entry> m_command_line_peers;
|
|
|
|
uint64_t m_peer_livetime;
|
|
|
|
//keep connections to initiate some interactions
|
|
|
|
net_server m_net_server;
|
2014-07-16 17:30:15 +00:00
|
|
|
boost::uuids::uuid m_network_id;
|
2014-03-03 22:07:58 +00:00
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
#include "net_node.inl"
|
|
|
|
|
|
|
|
POP_WARNINGS
|