diff --git a/CMakeLists.txt b/CMakeLists.txt index 6dd3638a..102399c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,14 +12,22 @@ if(APPLE) include_directories(SYSTEM /usr/include/malloc) endif() +if(MSVC) +include_directories(src/Platform/Windows) +elseif(APPLE) +include_directories(src/Platform/OSX) +else() +include_directories(src/Platform/Linux) +endif() + + set(STATIC ${MSVC} CACHE BOOL "Link libraries statically") if(MSVC) - add_definitions("/bigobj /MP /W3 /GS- /D_CRT_SECURE_NO_WARNINGS /wd4996 /wd4345 /D_WIN32_WINNT=0x0600 /DWIN32_LEAN_AND_MEAN /DGTEST_HAS_TR1_TUPLE=0 /D_VARIADIC_MAX=8 /FIinline_c.h /D__SSE4_1__") - # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Dinline=__inline") + add_definitions("/bigobj /MP /W3 /GS- /D_CRT_SECURE_NO_WARNINGS /wd4996 /wd4345 /D_WIN32_WINNT=0x0600 /DWIN32_LEAN_AND_MEAN /DGTEST_HAS_TR1_TUPLE=0 /D_VARIADIC_MAX=8 /D__SSE4_1__") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /STACK:10485760") if(STATIC) - foreach(VAR CMAKE_C_FLAGS_DEBUG CMAKE_CXX_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE CMAKE_CXX_FLAGS_RELEASE) + foreach(VAR CMAKE_C_FLAGS_DEBUG CMAKE_CXX_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE CMAKE_CXX_FLAGS_RELEASE CMAKE_C_FLAGS_RELWITHDEBINFO CMAKE_CXX_FLAGS_RELWITHDEBINFO) string(REPLACE "/MD" "/MT" ${VAR} "${${VAR}}") endforeach() endif() @@ -31,11 +39,11 @@ else() else() set(ARCH_FLAG "-march=${ARCH}") endif() - set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Werror -Wno-error=extra -Wno-error=deprecated-declarations -Wno-error=sign-compare -Wno-error=strict-aliasing -Wno-error=type-limits -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized") + set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Werror -Wno-error=extra -Wno-error=deprecated-declarations -Wno-error=sign-compare -Wno-error=strict-aliasing -Wno-error=type-limits -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized -Wno-error=unused-result") if(CMAKE_C_COMPILER_ID STREQUAL "Clang") - set(WARNINGS "${WARNINGS} -Wno-error=mismatched-tags -Wno-error=null-conversion -Wno-overloaded-shift-op-parentheses -Wno-error=shift-count-overflow -Wno-error=tautological-constant-out-of-range-compare -Wno-error=unused-private-field -Wno-error=unneeded-internal-declaration") + set(WARNINGS "${WARNINGS} -Wno-error=mismatched-tags -Wno-error=null-conversion -Wno-overloaded-shift-op-parentheses -Wno-error=shift-count-overflow -Wno-error=tautological-constant-out-of-range-compare -Wno-error=unused-private-field -Wno-error=unneeded-internal-declaration -Wno-error=unused-function") else() - set(WARNINGS "${WARNINGS} -Wlogical-op -Wno-error=maybe-uninitialized") + set(WARNINGS "${WARNINGS} -Wlogical-op -Wno-error=maybe-uninitialized -Wno-error=clobbered -Wno-error=unused-but-set-variable") endif() if(MINGW) set(WARNINGS "${WARNINGS} -Wno-error=unused-value") @@ -46,13 +54,7 @@ else() endif() set(C_WARNINGS "-Waggregate-return -Wnested-externs -Wold-style-definition -Wstrict-prototypes") set(CXX_WARNINGS "-Wno-reorder -Wno-missing-field-initializers") - try_compile(STATIC_ASSERT_RES "${CMAKE_CURRENT_BINARY_DIR}/static-assert" "${CMAKE_CURRENT_SOURCE_DIR}/utils/test-static-assert.c" COMPILE_DEFINITIONS "-std=c11") - if(STATIC_ASSERT_RES) - set(STATIC_ASSERT_FLAG "") - else() - set(STATIC_ASSERT_FLAG "-Dstatic_assert=_Static_assert") - endif() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${ARCH_FLAG} -maes") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${WARNINGS} ${C_WARNINGS} ${ARCH_FLAG} -maes") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${ARCH_FLAG} -maes") if(APPLE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGTEST_HAS_TR1_TUPLE=0") @@ -83,7 +85,7 @@ if(STATIC) set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_RUNTIME ON) endif() -find_package(Boost 1.53 REQUIRED COMPONENTS system filesystem thread date_time chrono regex serialization program_options) +find_package(Boost 1.53 REQUIRED COMPONENTS system filesystem thread date_time chrono regex serialization program_options coroutine context) if((${Boost_MAJOR_VERSION} EQUAL 1) AND (${Boost_MINOR_VERSION} EQUAL 54)) message(SEND_ERROR "Boost version 1.54 is unsupported, more details are available here http://goo.gl/RrCFmA") endif() @@ -122,6 +124,7 @@ else() endif() endif() +add_subdirectory(contrib) add_subdirectory(external) add_subdirectory(src) add_subdirectory(tests) diff --git a/README.md b/README.md index a31442cb..0f818b31 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ This is the reference code for [CryptoNote](https://cryptonote.org) cryptocurrency protocol. + * Launch your own CryptoNote currency: [CryptoNote Starter](https://cryptonotestarter.org/) * CryptoNote reference implementation: [CryptoNoteCoin](https://cryptonote-coin.org) * Discussion board and support: [CryptoNote Forum](https://forum.cryptonote.org) @@ -15,18 +16,18 @@ This is the reference code for [CryptoNote](https://cryptonote.org) cryptocurren ### First step. Give a name to your coin -**Good name must be unique.*** Check uniqueness with [google](http://google.com) and [Map of Coins](mapofcoins.com) or any other similar service. +**Good name must be unique.** Check uniqueness with [google](http://google.com) and [Map of Coins](mapofcoins.com) or any other similar service. Name must be specified twice: -**1. in file src/cryptonote_config.h** - CRYPTONOTE_NAME macro +**1. in file src/cryptonote_config.h** - `CRYPTONOTE_NAME` constant Example: ``` -#define CRYPTONOTE_NAME "furiouscoin" +const char CRYPTONOTE_NAME[] = "furiouscoin"; ``` -**2. in CMakeList.txt file** - set_property(TARGET daemon PROPERTY OUTPUT_NAME "YOURCOINNAMEd") +**2. in src/CMakeList.txt file** - set_property(TARGET daemon PROPERTY OUTPUT_NAME "YOURCOINNAME**d**") Example: ``` @@ -40,22 +41,22 @@ set_property(TARGET daemon PROPERTY OUTPUT_NAME "furiouscoind") **1. Total money supply** (src/cryptonote_config.h) -Total amount of coins to be emitted. Most of CryptoNote based coins use `(uint64_t)(-1)` (equals to 18446744073709551616). You can define number explicitly (for example UINT64_C(858986905600000000)). +Total amount of coins to be emitted. Most of CryptoNote based coins use `(uint64_t)(-1)` (equals to 18446744073709551616). You can define number explicitly (for example `UINT64_C(858986905600000000)`). Example: ``` -#define MONEY_SUPPLY ((uint64_t)(-1)) +const uint64_t MONEY_SUPPLY = (uint64_t)(-1); ``` **2. Emission curve** (src/cryptonote_config.h) Be default CryptoNote provides emission formula with slight decrease of block reward with each block. This is different from Bitcoin where block reward halves every 4 years. -EMISSION_SPEED_FACTOR macro defines emission curve slope. This parameter is required to calulate block reward. +`EMISSION_SPEED_FACTOR` constant defines emission curve slope. This parameter is required to calulate block reward. Example: ``` -#define EMISSION_SPEED_FACTOR (18) +const unsigned EMISSION_SPEED_FACTOR = 18; ``` **3. Difficulty target** (src/cryptonote_config.h) @@ -72,19 +73,19 @@ For most coins difficulty target is 60 or 120 seconds. Example: ``` -#define DIFFICULTY_TARGET 120 +const uint64_t DIFFICULTY_TARGET = 120; ``` **4. Block reward formula** -In case you are not satisfied with CryptoNote default implementation of block reward logic you can also change it. The implementation is in src/cryptonote_core/cryptonote_basic_impl.cpp: +In case you are not satisfied with CryptoNote default implementation of block reward logic you can also change it. The implementation is in `src/cryptonote_core/Currency.cpp`: ``` -bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward) +bool Currency::getBlockReward(size_t medianSize, size_t currentBlockSize, uint64_t alreadyGeneratedCoins, uint64_t fee, uint64_t& reward, int64_t& emissionChange) const ``` This function has two parts: -- basic block reward calculation: `uint64_t base_reward = (MONEY_SUPPLY - already_generated_coins) >> EMISSION_SPEED_FACTOR;` +- basic block reward calculation: `uint64_t baseReward = (m_moneySupply - alreadyGeneratedCoins) >> m_emissionSpeedFactor;` - big block penalty calculation: this is the way CryptoNote protects the block chain from transaction flooding attacks and preserves opportunities for organic network growth at the same time. Only the first part of this function is directly related to the emission logic. You can change it the way you want. See MonetaVerde and DuckNote as the examples where this function is modified. @@ -105,8 +106,8 @@ It's better to choose ports that aren't used by other software or coins. See kno Example: ``` -#define P2P_DEFAULT_PORT 17236 -#define RPC_DEFAULT_PORT 18236 +const int P2P_DEFAULT_PORT = 17236; +const int RPC_DEFAULT_PORT = 18236; ``` @@ -114,72 +115,84 @@ Example: This identifier is used in network packages in order not to mix two different cryptocoin networks. Change all the bytes to random values for your network: ``` -const static boost::uuids::uuid CRYPTONOTE_NETWORK = { { 0xA1 ,0x1A, 0xA1, 0x1A , 0xA1, 0x0A , 0xA1, 0x0A, 0xA0, 0x1A, 0xA0, 0x1A, 0xA0, 0x1A, 0xA1, 0x1A} }; +const static boost::uuids::uuid CRYPTONOTE_NETWORK = { { 0xA1, 0x1A, 0xA1, 0x1A, 0xA1, 0x0A, 0xA1, 0x0A, 0xA0, 0x1A, 0xA0, 0x1A, 0xA0, 0x1A, 0xA1, 0x1A } }; ``` -**3. Seed nodes** (src/p2p/net_node.inl) +**3. Seed nodes** (src/cryptonote_config.h) -Add ip addresses of your seed nodes to the beginning of `node_server::init(const boost::program_options::variables_map& vm)` function. +Add IP addresses of your seed nodes. Example: ``` -ADD_HARDCODED_SEED_NODE("111.11.11.11:17236"); -ADD_HARDCODED_SEED_NODE("222.22.22.22:17236"); +const std::initializer_list SEED_NODES = { + "111.11.11.11:17236", + "222.22.22.22:17236", +}; ``` ### Fourth step. Transaction fee and related parameters -**1. Default transaction fee** (src/cryptonote_config.h) +**1. Minimum transaction fee** (src/cryptonote_config.h) -Zero default fee can lead to transaction flooding. Transactions cheaper than the default transaction fee wouldn't be accepted by daemons. 100000 value for DEFAULT_FEE is usually enough. +Zero minimum fee can lead to transaction flooding. Transactions cheaper than the minimum transaction fee wouldn't be accepted by daemons. 100000 value for `MINIMUM_FEE` is usually enough. Example: ``` -#define DEFAULT_FEE 100000 +const uint64_t MINIMUM_FEE = 100000; ``` **2. Penalty free block size** (src/cryptonote_config.h) -CryptoNote protects chain from tx flooding by reducing block reward for blocks larger than the median block size. However, this rule applies for blocks larger than CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE bytes. +CryptoNote protects chain from tx flooding by reducing block reward for blocks larger than the median block size. However, this rule applies for blocks larger than `CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE` bytes. Example: ``` -#define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE 20000 +const size_t CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE = 20000; ``` -### Fifth step. Genesis block +### Fifth step. Address prefix + +You may choose a letter (in some cases several letters) all the coin's public addresses will start with. It is defined by `CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX` constant. Since the rules for address prefixes are nontrivial you may use the [prefix generator tool](https://cryptonotestarter.org/tools.html). + +Example: +``` +const uint64_t CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX = 0xe9; // addresses start with "f" +``` + + +### Sixth step. Genesis block **1. Build the binaries with blank genesis tx hex** (src/cryptonote_config.h) -You should leave #define GENESIS_COINBASE_TX_HEX blank and compile the binaries without it. +You should leave `const char GENESIS_COINBASE_TX_HEX[]` blank and compile the binaries without it. Example: ``` -#define GENESIS_COINBASE_TX_HEX "" +const char GENESIS_COINBASE_TX_HEX[] = ""; ``` **2. Start the daemon to print out the genesis block** -Run your daemon with --print-genesis-tx argument. It will print out the genesis block coinbase transaction hash. +Run your daemon with `--print-genesis-tx` argument. It will print out the genesis block coinbase transaction hash. Example: ``` -cryptonotecoind --print-genesis-tx +furiouscoind --print-genesis-tx ``` **3. Copy the printed transaction hash** (src/cryptonote_config.h) -Copy the tx hash that has been printed by the daemon to GENESIS_COINBASE_TX_HEX in /src/cryptonote_config.h +Copy the tx hash that has been printed by the daemon to `GENESIS_COINBASE_TX_HEX` in `src/cryptonote_config.h` Example: ``` -#define GENESIS_COINBASE_TX_HEX "013c01ff0001ffff...785a33d9ebdba68b0" +const char GENESIS_COINBASE_TX_HEX[] = "013c01ff0001ffff...785a33d9ebdba68b0"; ``` @@ -192,25 +205,27 @@ Recompile everything again. Your coin code is ready now. Make an announcement fo ### On *nix -Dependencies: GCC 4.7.3 or later, CMake 2.8.6 or later, and Boost 1.53 or later (except 1.54, more details here: http://goo.gl/RrCFmA). +Dependencies: GCC 4.7.3 or later, CMake 2.8.6 or later, and Boost 1.55. You may download them from: + * http://gcc.gnu.org/ * http://www.cmake.org/ * http://www.boost.org/ -* -Alternatively, it may be possible to install them using a package manager. +* Alternatively, it may be possible to install them using a package manager. -To build, change to a directory where this file is located, and run `make'. The resulting executables can be found in build/release/src. +To build, change to a directory where this file is located, and run `make`. The resulting executables can be found in `build/release/src`. **Advanced options:** + * Parallel build: run `make -j` instead of `make`. * Debug build: run `make build-debug`. * Test suite: run `make test-release` to run tests in addition to building. Running `make test-debug` will do the same to the debug version. * Building with Clang: it may be possible to use Clang instead of GCC, but this may not work everywhere. To build, run `export CC=clang CXX=clang++` before running `make`. ### On Windows -Dependencies: MSVC 2012 or later, CMake 2.8.6 or later, and Boost 1.53 or later. You may download them from: +Dependencies: MSVC 2012 or later, CMake 2.8.6 or later, and Boost 1.55. You may download them from: + * http://www.microsoft.com/ * http://www.cmake.org/ * http://www.boost.org/ diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 43ec47c6..e01b4d2e 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -1,4 +1,25 @@ -Release notes 0.8.11 +Release notes 1.0.2 + +- Multisignature with API +- Low level API +- High level API improvements +- Updated block reward scheme +- Dynamic maximum block size limit + +- Instant transaction pool notifications +- Transaction priority based on tx fee +- Transactions are returned from tx pools after 24 hours + +- Fully refactored simplewallet +- Transaction history for simplewallet +- Reset command for simplewallet +- Faster wallet refresh + +- Further optimization in daemon RAM consumption +- Various network health updates +- Config and CryptoNote forking how-to update + +Release notes 1.0.1 - High level API implementation - CryptoNight hash function optimization diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt new file mode 100644 index 00000000..7222cce5 --- /dev/null +++ b/contrib/CMakeLists.txt @@ -0,0 +1,7 @@ +file(GLOB_RECURSE EPEE epee/include/*) + +source_group(epee FILES ${EPEE}) + +add_library(epee ${EPEE}) + +set_property(TARGET epee PROPERTY FOLDER "external") diff --git a/contrib/epee/demo/demo_levin_server/stdafx.h b/contrib/epee/demo/demo_levin_server/stdafx.h index f69d5922..cc455843 100644 --- a/contrib/epee/demo/demo_levin_server/stdafx.h +++ b/contrib/epee/demo/demo_levin_server/stdafx.h @@ -35,7 +35,4 @@ #define BOOST_FILESYSTEM_VERSION 3 #define ENABLE_RELEASE_LOGGING -#include "log_opt_defs.h" #include "misc_log_ex.h" - - diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index fcab35aa..a1df7839 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -32,6 +32,11 @@ #include #include +#include +#include + +#include "string_tools.h" + namespace epee { class async_stdin_reader @@ -294,7 +299,7 @@ namespace epee bool start_default_console(t_server* ptsrv, t_handler handlr, const std::string& prompt, const std::string& usage = "") { std::shared_ptr console_handler = std::make_shared(); - boost::thread([=](){console_handler->run(ptsrv, handlr, prompt, usage);}).detach(); + std::thread([=](){console_handler->run(ptsrv, handlr, prompt, usage);}).detach(); return true; } @@ -314,46 +319,24 @@ namespace epee bool run_default_console_handler_no_srv_param(t_server* ptsrv, t_handler handlr, const std::string& prompt, const std::string& usage = "") { async_console_handler console_handler; - return console_handler.run(ptsrv, boost::bind(no_srv_param_adapter, _1, _2, handlr), prompt, usage); + return console_handler.run(ptsrv, std::bind(no_srv_param_adapter, std::placeholders::_1, std::placeholders::_2, handlr), prompt, usage); } template bool start_default_console_handler_no_srv_param(t_server* ptsrv, t_handler handlr, const std::string& prompt, const std::string& usage = "") { - boost::thread( boost::bind(run_default_console_handler_no_srv_param, ptsrv, handlr, prompt, usage) ); + std::thread( std::bind(run_default_console_handler_no_srv_param, ptsrv, handlr, prompt, usage) ); return true; } - /*template - bool f(int i, a l) - { - return true; - }*/ - /* - template - bool default_console_handler2(chain_handler ch_handler, const std::string usage) - */ - - - /*template - bool start_default_console2(t_handler handlr, const std::string& usage = "") - { - //std::string usage_local = usage; - boost::thread( boost::bind(default_console_handler2, handlr, usage) ); - //boost::function p__ = boost::bind(f, 1, handlr); - //boost::function p__ = boost::bind(default_console_handler2, handlr, usage); - //boost::thread tr(p__); - return true; - }*/ - /************************************************************************/ /* */ /************************************************************************/ class console_handlers_binder { - typedef boost::function &)> console_command_handler; + typedef std::function &)> console_command_handler; typedef std::map > command_handlers_map; - std::unique_ptr m_console_thread; + std::unique_ptr m_console_thread; command_handlers_map m_command_handlers; async_console_handler m_console_handler; public: @@ -396,16 +379,9 @@ namespace epee return process_command_vec(cmd_v); } - /*template - bool start_handling(t_srv& srv, const std::string& usage_string = "") - { - start_default_console_handler_no_srv_param(&srv, boost::bind(&console_handlers_binder::process_command_str, this, _1)); - return true; - }*/ - bool start_handling(const std::string& prompt, const std::string& usage_string = "") { - m_console_thread.reset(new boost::thread(boost::bind(&console_handlers_binder::run_handling, this, prompt, usage_string))); + m_console_thread.reset(new std::thread(std::bind(&console_handlers_binder::run_handling, this, prompt, usage_string))); m_console_thread->detach(); return true; } @@ -417,14 +393,8 @@ namespace epee bool run_handling(const std::string& prompt, const std::string& usage_string) { - return m_console_handler.run(boost::bind(&console_handlers_binder::process_command_str, this, _1), prompt, usage_string); + return m_console_handler.run(std::bind(&console_handlers_binder::process_command_str, this, std::placeholders::_1), prompt, usage_string); } - - /*template - bool run_handling(t_srv& srv, const std::string& usage_string) - { - return run_default_console_handler_no_srv_param(&srv, boost::bind(&console_handlers_binder::process_command_str, this, _1), usage_string); - }*/ }; /* work around because of broken boost bind */ @@ -438,13 +408,14 @@ namespace epee public: bool start_handling(t_server* psrv, const std::string& prompt, const std::string& usage_string = "") { - boost::thread(boost::bind(&srv_console_handlers_binder::run_handling, this, psrv, prompt, usage_string)).detach(); + std::thread(std::bind(&srv_console_handlers_binder::run_handling, this, psrv, prompt, usage_string)).detach(); return true; } bool run_handling(t_server* psrv, const std::string& prompt, const std::string& usage_string) { - return m_console_handler.run(psrv, boost::bind(&srv_console_handlers_binder::process_command_str, this, _1, _2), prompt, usage_string); + return m_console_handler.run(psrv, std::bind(&srv_console_handlers_binder::process_command_str, this, + std::placeholders::_1, std::placeholders::_2), prompt, usage_string); } void stop_handling() diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index 7e852183..01d70db3 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -28,10 +28,7 @@ #ifndef _FILE_IO_UTILS_H_ #define _FILE_IO_UTILS_H_ - -//#include -//#include - +#include #include #include @@ -75,7 +72,7 @@ namespace file_io_utils #ifdef BOOST_LEXICAL_CAST_INCLUDED inline - bool get_not_used_filename(const std::string& folder, OUT std::string& result_name) + bool get_not_used_filename(const std::string& folder, std::string& result_name) { DWORD folder_attr = ::GetFileAttributesA(folder.c_str()); if(folder_attr == INVALID_FILE_ATTRIBUTES) @@ -302,7 +299,7 @@ namespace file_io_utils } */ inline - bool get_file_time(const std::string& path_to_file, OUT time_t& ft) + bool get_file_time(const std::string& path_to_file, time_t& ft) { boost::system::error_code ec; ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); @@ -408,7 +405,7 @@ namespace file_io_utils } */ #ifdef WINDOWS_PLATFORM - inline bool get_folder_content(const std::string& path, std::list& OUT target_list) + inline bool get_folder_content(const std::string& path, std::list& target_list) { WIN32_FIND_DATAA find_data = {0}; HANDLE hfind = ::FindFirstFileA((path + "\\*.*").c_str(), &find_data); @@ -426,7 +423,7 @@ namespace file_io_utils return true; } #endif - inline bool get_folder_content(const std::string& path, std::list& OUT target_list, bool only_files = false) + inline bool get_folder_content(const std::string& path, std::list& target_list, bool only_files = false) { try { diff --git a/contrib/epee/include/math_helper.h b/contrib/epee/include/math_helper.h index 349d6d82..11faa976 100644 --- a/contrib/epee/include/math_helper.h +++ b/contrib/epee/include/math_helper.h @@ -37,6 +37,7 @@ #include #include "misc_os_dependent.h" +#include "pragma_comp_defs.h" namespace epee { diff --git a/contrib/epee/include/misc_log_ex.cpp b/contrib/epee/include/misc_log_ex.cpp new file mode 100644 index 00000000..0c0b441b --- /dev/null +++ b/contrib/epee/include/misc_log_ex.cpp @@ -0,0 +1,1029 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "misc_log_ex.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(WIN32) +#include +#else +#include +#endif + +#include "static_initializer.h" +#include "string_tools.h" +#include "time_helper.h" +#include "misc_os_dependent.h" + +#include "syncobj.h" + + +#define LOG_LEVEL_SILENT -1 +#define LOG_LEVEL_0 0 +#define LOG_LEVEL_1 1 +#define LOG_LEVEL_2 2 +#define LOG_LEVEL_3 3 +#define LOG_LEVEL_4 4 +#define LOG_LEVEL_MIN LOG_LEVEL_SILENT +#define LOG_LEVEL_MAX LOG_LEVEL_4 + + +#define LOGGER_NULL 0 +#define LOGGER_FILE 1 +#define LOGGER_DEBUGGER 2 +#define LOGGER_CONSOLE 3 +#define LOGGER_DUMP 4 + + +#ifndef LOCAL_ASSERT +#include +#if (defined _MSC_VER) +#define LOCAL_ASSERT(expr) {if(epee::debug::get_set_enable_assert()){_ASSERTE(expr);}} +#else +#define LOCAL_ASSERT(expr) +#endif + +#endif + +namespace epee { +namespace log_space { + //---------------------------------------------------------------------------- + bool is_stdout_a_tty() + { + static std::atomic initialized(false); + static std::atomic is_a_tty(false); + + if (!initialized.load(std::memory_order_acquire)) + { +#if defined(WIN32) + is_a_tty.store(0 != _isatty(_fileno(stdout)), std::memory_order_relaxed); +#else + is_a_tty.store(0 != isatty(fileno(stdout)), std::memory_order_relaxed); +#endif + initialized.store(true, std::memory_order_release); + } + + return is_a_tty.load(std::memory_order_relaxed); + } + //---------------------------------------------------------------------------- + void set_console_color(int color, bool bright) + { + if (!is_stdout_a_tty()) + return; + + switch(color) + { + case console_color_default: + { +#ifdef WIN32 + HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE| (bright ? FOREGROUND_INTENSITY:0)); +#else + if(bright) + std::cout << "\033[1;37m"; + else + std::cout << "\033[0m"; +#endif + } + break; + case console_color_white: + { +#ifdef WIN32 + HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0)); +#else + if(bright) + std::cout << "\033[1;37m"; + else + std::cout << "\033[0;37m"; +#endif + } + break; + case console_color_red: + { +#ifdef WIN32 + HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | (bright ? FOREGROUND_INTENSITY:0)); +#else + if(bright) + std::cout << "\033[1;31m"; + else + std::cout << "\033[0;31m"; +#endif + } + break; + case console_color_green: + { +#ifdef WIN32 + HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | (bright ? FOREGROUND_INTENSITY:0)); +#else + if(bright) + std::cout << "\033[1;32m"; + else + std::cout << "\033[0;32m"; +#endif + } + break; + + case console_color_blue: + { +#ifdef WIN32 + HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_INTENSITY);//(bright ? FOREGROUND_INTENSITY:0)); +#else + if(bright) + std::cout << "\033[1;34m"; + else + std::cout << "\033[0;34m"; +#endif + } + break; + + case console_color_cyan: + { +#ifdef WIN32 + HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0)); +#else + if(bright) + std::cout << "\033[1;36m"; + else + std::cout << "\033[0;36m"; +#endif + } + break; + + case console_color_magenta: + { +#ifdef WIN32 + HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_RED | (bright ? FOREGROUND_INTENSITY:0)); +#else + if(bright) + std::cout << "\033[1;35m"; + else + std::cout << "\033[0;35m"; +#endif + } + break; + + case console_color_yellow: + { +#ifdef WIN32 + HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | (bright ? FOREGROUND_INTENSITY:0)); +#else + if(bright) + std::cout << "\033[1;33m"; + else + std::cout << "\033[0;33m"; +#endif + } + break; + + } + } + //---------------------------------------------------------------------------- + void reset_console_color() { + if (!is_stdout_a_tty()) + return; + +#ifdef WIN32 + HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE); +#else + std::cout << "\033[0m"; + std::cout.flush(); +#endif + } + //---------------------------------------------------------------------------- + bool rotate_log_file(const char* pfile_path) + { +#ifdef _MSC_VER + if(!pfile_path) + return false; + + std::string file_path = pfile_path; + std::string::size_type a = file_path .rfind('.'); + if ( a != std::string::npos ) + file_path .erase( a, file_path .size()); + + ::DeleteFileA( (file_path + ".0").c_str() ); + ::MoveFileA( (file_path + ".log").c_str(), (file_path + ".0").c_str() ); +#else + return false;//not implemented yet +#endif + return true; + } + //---------------------------------------------------------------------------- +#ifdef _MSC_VER + bool debug_output_stream::out_buffer( const char* buffer, int buffer_len , int log_level, int color, const char* plog_name/* = NULL*/) + { + for ( int i = 0; i < buffer_len; i = i + max_dbg_str_len ) + { + std::string s( buffer + i, buffer_len- i < max_dbg_str_len ? + buffer_len - i : max_dbg_str_len ); + + ::OutputDebugStringA( s.c_str() ); + } + return true; + } +#endif + //---------------------------------------------------------------------------- + console_output_stream::console_output_stream() + { +#ifdef _MSC_VER + + if(!::GetStdHandle(STD_OUTPUT_HANDLE)) + m_have_to_kill_console = true; + else + m_have_to_kill_console = false; + + ::AllocConsole(); +#endif + } + //---------------------------------------------------------------------------- + console_output_stream::~console_output_stream() + { +#ifdef _MSC_VER + if(m_have_to_kill_console) + ::FreeConsole(); +#endif + } + //---------------------------------------------------------------------------- + bool console_output_stream::out_buffer( const char* buffer, int buffer_len , int log_level, int color, const char* plog_name/* = NULL*/) + { + if(plog_name) + return true; //skip alternative logs from console + + set_console_color(color, log_level < 1); + +#ifdef _MSC_VER + const char* ptarget_buf = NULL; + char* pallocated_buf = NULL; + + // + int i = 0; + for(; i < buffer_len; i++) + if(buffer[i] == '\a') break; + if(i == buffer_len) + ptarget_buf = buffer; + else + { + pallocated_buf = new char[buffer_len]; + ptarget_buf = pallocated_buf; + for(i = 0; i < buffer_len; i++) + { + if(buffer[i] == '\a') + pallocated_buf[i] = '^'; + else + pallocated_buf[i] = buffer[i]; + } + } + + //uint32_t b = 0; + //::WriteConsoleA(::GetStdHandle(STD_OUTPUT_HANDLE), ptarget_buf, buffer_len, (DWORD*)&b, 0); + std::cout << ptarget_buf; + if(pallocated_buf) delete [] pallocated_buf; +#else + std::string buf(buffer, buffer_len); + for(size_t i = 0; i!= buf.size(); i++) + { + if(buf[i] == 7 || buf[i] == -107) + buf[i] = '^'; + } + + std::cout << buf; +#endif + reset_console_color(); + return true; + } + //---------------------------------------------------------------------------- + file_output_stream::file_output_stream(std::string default_log_file_name, std::string log_path) + { + m_default_log_filename = default_log_file_name; + m_max_logfile_size = 0; + m_default_log_path = log_path; + m_pdefault_file_stream = add_new_stream_and_open(default_log_file_name.c_str()); + } + //---------------------------------------------------------------------------- + file_output_stream::~file_output_stream() + { + for(named_log_streams::iterator it = m_log_file_names.begin(); it!=m_log_file_names.end(); it++) + { + if ( it->second->is_open() ) + { + it->second->flush(); + it->second->close(); + } + delete it->second; + } + } + //---------------------------------------------------------------------------- + std::ofstream* file_output_stream::add_new_stream_and_open(const char* pstream_name) + { + //log_space::rotate_log_file((m_default_log_path + "\\" + pstream_name).c_str()); + + std::ofstream* pstream = (m_log_file_names[pstream_name] = new std::ofstream); + std::string target_path = m_default_log_path + "/" + pstream_name; + pstream->open( target_path.c_str(), std::ios_base::out | std::ios::app /*ios_base::trunc */); + if(pstream->fail()) + return NULL; + return pstream; + } + //---------------------------------------------------------------------------- + bool file_output_stream::set_max_logfile_size(uint64_t max_size) + { + m_max_logfile_size = max_size; + return true; + } + //---------------------------------------------------------------------------- + bool file_output_stream::set_log_rotate_cmd(const std::string& cmd) + { + m_log_rotate_cmd = cmd; + return true; + } + //---------------------------------------------------------------------------- + bool file_output_stream::out_buffer(const char* buffer, int buffer_len, int log_level, int color, const char* plog_name/* = NULL*/) + { + std::ofstream* m_target_file_stream = m_pdefault_file_stream; + if(plog_name) + { //find named stream + named_log_streams::iterator it = m_log_file_names.find(plog_name); + if(it == m_log_file_names.end()) + m_target_file_stream = add_new_stream_and_open(plog_name); + else + m_target_file_stream = it->second; + } + if(!m_target_file_stream || !m_target_file_stream->is_open()) + return false;//TODO: add assert here + + m_target_file_stream->write(buffer, buffer_len ); + m_target_file_stream->flush(); + + if(m_max_logfile_size) + { + std::ofstream::pos_type pt = m_target_file_stream->tellp(); + uint64_t current_sz = pt; + if(current_sz > m_max_logfile_size) + { + std::cout << "current_sz= " << current_sz << " m_max_logfile_size= " << m_max_logfile_size << std::endl; + std::string log_file_name; + if(!plog_name) + log_file_name = m_default_log_filename; + else + log_file_name = plog_name; + + m_target_file_stream->close(); + std::string new_log_file_name = log_file_name; + + time_t tm = 0; + time(&tm); + + int err_count = 0; + boost::system::error_code ec; + do + { + new_log_file_name = string_tools::cut_off_extension(log_file_name); + if(err_count) + new_log_file_name += misc_utils::get_time_str_v2(tm) + "(" + boost::lexical_cast(err_count) + ")" + ".log"; + else + new_log_file_name += misc_utils::get_time_str_v2(tm) + ".log"; + + err_count++; + }while(boost::filesystem::exists(m_default_log_path + "/" + new_log_file_name, ec)); + + std::string new_log_file_path = m_default_log_path + "/" + new_log_file_name; + boost::filesystem::rename(m_default_log_path + "/" + log_file_name, new_log_file_path, ec); + if(ec) + { + std::cout << "Filed to rename, ec = " << ec.message() << std::endl; + } + + if(m_log_rotate_cmd.size()) + { + + std::string m_log_rotate_cmd_local_copy = m_log_rotate_cmd; + //boost::replace_all(m_log_rotate_cmd, "[*SOURCE*]", new_log_file_path); + boost::replace_all(m_log_rotate_cmd_local_copy, "[*TARGET*]", new_log_file_path); + + misc_utils::call_sys_cmd(m_log_rotate_cmd_local_copy); + } + + m_target_file_stream->open( (m_default_log_path + "/" + log_file_name).c_str(), std::ios_base::out | std::ios::app /*ios_base::trunc */); + if(m_target_file_stream->fail()) + return false; + } + } + + return true; + } + //---------------------------------------------------------------------------- + log_stream_splitter::~log_stream_splitter() + { + //free pointers + std::for_each(m_log_streams.begin(), m_log_streams.end(), delete_ptr()); + } + //---------------------------------------------------------------------------- + bool log_stream_splitter::set_max_logfile_size(uint64_t max_size) + { + for(streams_container::iterator it = m_log_streams.begin(); it!=m_log_streams.end();it++) + it->first->set_max_logfile_size(max_size); + return true; + } + //---------------------------------------------------------------------------- + bool log_stream_splitter::set_log_rotate_cmd(const std::string& cmd) + { + for(streams_container::iterator it = m_log_streams.begin(); it!=m_log_streams.end();it++) + it->first->set_log_rotate_cmd(cmd); + return true; + } + //---------------------------------------------------------------------------- + bool log_stream_splitter::do_log_message(const std::string& rlog_mes, int log_level, int color, const char* plog_name/* = NULL*/) + { + std::string str_mess = rlog_mes; + size_t str_len = str_mess.size(); + const char* pstr = str_mess.c_str(); + for(streams_container::iterator it = m_log_streams.begin(); it!=m_log_streams.end();it++) + if(it->second >= log_level) + it->first->out_buffer(pstr, (int)str_len, log_level, color, plog_name); + return true; + } + //---------------------------------------------------------------------------- + bool log_stream_splitter::add_logger(int type, const char* pdefault_file_name, const char* pdefault_log_folder, int log_level_limit/* = LOG_LEVEL_4*/) + { + ibase_log_stream* ls = NULL; + + switch( type ) + { + case LOGGER_FILE: + ls = new file_output_stream( pdefault_file_name, pdefault_log_folder ); + break; + + case LOGGER_DEBUGGER: +#ifdef _MSC_VER + ls = new debug_output_stream( ); +#else + return false;//not implemented yet +#endif + break; + case LOGGER_CONSOLE: + ls = new console_output_stream( ); + break; + } + + if ( ls ) { + m_log_streams.push_back(streams_container::value_type(ls, log_level_limit)); + return true; + } + return ls ? true:false; + } + //---------------------------------------------------------------------------- + bool log_stream_splitter::add_logger(ibase_log_stream* pstream, int log_level_limit/* = LOG_LEVEL_4*/) + { + m_log_streams.push_back(streams_container::value_type(pstream, log_level_limit) ); + return true; + } + //---------------------------------------------------------------------------- + bool log_stream_splitter::remove_logger(int type) + { + streams_container::iterator it = m_log_streams.begin(); + for(;it!=m_log_streams.end(); it++) + { + if(it->first->get_type() == type) + { + delete it->first; + m_log_streams.erase(it); + return true; + } + } + return false; + } + //---------------------------------------------------------------------------- + std::string get_daytime_string2() + { + boost::posix_time::ptime p = boost::posix_time::microsec_clock::local_time(); + return misc_utils::get_time_str_v3(p); + } + //---------------------------------------------------------------------------- + std::string get_day_time_string() + { + return get_daytime_string2(); + //time_t tm = 0; + //time(&tm); + //return misc_utils::get_time_str(tm); + } + //---------------------------------------------------------------------------- + std::string get_time_string() + { + return get_daytime_string2(); + } + //---------------------------------------------------------------------------- +#ifdef _MSC_VER + std::string get_time_string_adv(SYSTEMTIME* pst/* = NULL*/) + { + SYSTEMTIME st = {0}; + if(!pst) + { + pst = &st; + GetSystemTime(&st); + } + std::stringstream str_str; + str_str.fill('0'); + str_str << std::setw(2) << pst->wHour << "_" + << std::setw(2) << pst->wMinute << "_" + << std::setw(2) << pst->wSecond << "_" + << std::setw(3) << pst->wMilliseconds; + return str_str.str(); + } +#endif + //---------------------------------------------------------------------------- + logger::logger() + { + CRITICAL_REGION_BEGIN(m_critical_sec); + init(); + CRITICAL_REGION_END(); + } + //---------------------------------------------------------------------------- + bool logger::set_max_logfile_size(uint64_t max_size) + { + CRITICAL_REGION_BEGIN(m_critical_sec); + m_log_target.set_max_logfile_size(max_size); + CRITICAL_REGION_END(); + return true; + } + //---------------------------------------------------------------------------- + bool logger::set_log_rotate_cmd(const std::string& cmd) + { + CRITICAL_REGION_BEGIN(m_critical_sec); + m_log_target.set_log_rotate_cmd(cmd); + CRITICAL_REGION_END(); + return true; + } + //---------------------------------------------------------------------------- + bool logger::take_away_journal(std::list& journal) + { + CRITICAL_REGION_BEGIN(m_critical_sec); + m_journal.swap(journal); + CRITICAL_REGION_END(); + return true; + } + //---------------------------------------------------------------------------- + bool logger::do_log_message(const std::string& rlog_mes, int log_level, int color, bool add_to_journal/* = false*/, const char* plog_name/* = NULL*/) + { + CRITICAL_REGION_BEGIN(m_critical_sec); + m_log_target.do_log_message(rlog_mes, log_level, color, plog_name); + if(add_to_journal) + m_journal.push_back(rlog_mes); + + return true; + CRITICAL_REGION_END(); + } + //---------------------------------------------------------------------------- + bool logger::add_logger( int type, const char* pdefault_file_name, const char* pdefault_log_folder, int log_level_limit/* = LOG_LEVEL_4*/) + { + CRITICAL_REGION_BEGIN(m_critical_sec); + return m_log_target.add_logger( type, pdefault_file_name, pdefault_log_folder, log_level_limit); + CRITICAL_REGION_END(); + } + //---------------------------------------------------------------------------- + bool logger::add_logger( ibase_log_stream* pstream, int log_level_limit/* = LOG_LEVEL_4*/) + { + CRITICAL_REGION_BEGIN(m_critical_sec); + return m_log_target.add_logger(pstream, log_level_limit); + CRITICAL_REGION_END(); + } + //---------------------------------------------------------------------------- + bool logger::remove_logger(int type) + { + CRITICAL_REGION_BEGIN(m_critical_sec); + return m_log_target.remove_logger(type); + CRITICAL_REGION_END(); + } + //---------------------------------------------------------------------------- + bool logger::set_thread_prefix(const std::string& prefix) + { + CRITICAL_REGION_BEGIN(m_critical_sec); + m_thr_prefix_strings[misc_utils::get_thread_string_id()] = prefix; + CRITICAL_REGION_END(); + return true; + } + //---------------------------------------------------------------------------- + bool logger::init() + { + m_process_name = string_tools::get_current_module_name(); + + init_log_path_by_default(); + + //init default set of loggers + init_default_loggers(); + + std::stringstream ss; + ss << get_time_string() << " Init logging. Level=" << get_set_log_detalisation_level() + << " Log path=" << m_default_log_folder << std::endl; + this->do_log_message(ss.str(), console_color_white, LOG_LEVEL_0); + return true; + } + //---------------------------------------------------------------------------- + bool logger::init_default_loggers() + { + return true; + } + //---------------------------------------------------------------------------- + bool logger::init_log_path_by_default() + { + //load process name + m_default_log_folder = string_tools::get_current_module_folder(); + + m_default_log_file = m_process_name; + std::string::size_type a = m_default_log_file.rfind('.'); + if ( a != std::string::npos ) + m_default_log_file.erase( a, m_default_log_file.size()); + m_default_log_file += ".log"; + + return true; + } + //---------------------------------------------------------------------------- + int log_singletone::get_log_detalisation_level() + { + get_or_create_instance();//to initialize logger, if it not initialized + return get_set_log_detalisation_level(); + } + //---------------------------------------------------------------------------- + bool log_singletone::is_filter_error(int error_code) + { + return false; + } + //---------------------------------------------------------------------------- + bool log_singletone::do_log_message(const std::string& rlog_mes, int log_level, int color, bool keep_in_journal, const char* plog_name/* = NULL*/) + { + logger* plogger = get_or_create_instance(); + bool res = false; + if(plogger) + res = plogger->do_log_message(rlog_mes, log_level, color, keep_in_journal, plog_name); + else + { //globally uninitialized, create new logger for each call of do_log_message() and then delete it + plogger = new logger(); + //TODO: some extra initialization + res = plogger->do_log_message(rlog_mes, log_level, color, keep_in_journal, plog_name); + delete plogger; + plogger = NULL; + } + return res; + } + //---------------------------------------------------------------------------- + bool log_singletone::take_away_journal(std::list& journal) + { + logger* plogger = get_or_create_instance(); + bool res = false; + if(plogger) + res = plogger->take_away_journal(journal); + + return res; + } + //---------------------------------------------------------------------------- + bool log_singletone::set_max_logfile_size(uint64_t file_size) + { + logger* plogger = get_or_create_instance(); + if(!plogger) return false; + return plogger->set_max_logfile_size(file_size); + } + //---------------------------------------------------------------------------- + bool log_singletone::set_log_rotate_cmd(const std::string& cmd) + { + logger* plogger = get_or_create_instance(); + if(!plogger) return false; + return plogger->set_log_rotate_cmd(cmd); + } + //---------------------------------------------------------------------------- + bool log_singletone::add_logger( int type, const char* pdefault_file_name, const char* pdefault_log_folder, int log_level_limit/* = LOG_LEVEL_4*/) + { + logger* plogger = get_or_create_instance(); + if(!plogger) return false; + return plogger->add_logger(type, pdefault_file_name, pdefault_log_folder, log_level_limit); + } + //---------------------------------------------------------------------------- + std::string log_singletone::get_default_log_file() + { + logger* plogger = get_or_create_instance(); + if(plogger) + return plogger->get_default_log_file(); + + return ""; + } + //---------------------------------------------------------------------------- + std::string log_singletone::get_default_log_folder() + { + logger* plogger = get_or_create_instance(); + if(plogger) + return plogger->get_default_log_folder(); + + return ""; + } + //---------------------------------------------------------------------------- + bool log_singletone::add_logger(ibase_log_stream* pstream, int log_level_limit/* = LOG_LEVEL_4*/) + { + logger* plogger = get_or_create_instance(); + if(!plogger) return false; + return plogger->add_logger(pstream, log_level_limit); + } + //---------------------------------------------------------------------------- + bool log_singletone::remove_logger(int type) + { + logger* plogger = get_or_create_instance(); + if(!plogger) return false; + return plogger->remove_logger(type); + } + //---------------------------------------------------------------------------- +PUSH_WARNINGS +DISABLE_GCC_WARNING(maybe-uninitialized) + int log_singletone::get_set_log_detalisation_level(bool is_need_set/* = false*/, int log_level_to_set/* = LOG_LEVEL_1*/) + { + static int log_detalisation_level = LOG_LEVEL_1; + if(is_need_set) + log_detalisation_level = log_level_to_set; + return log_detalisation_level; + } +POP_WARNINGS + //---------------------------------------------------------------------------- + int log_singletone::get_set_time_level(bool is_need_set/* = false*/, int time_log_level/* = LOG_LEVEL_0*/) + { + static int val_time_log_level = LOG_LEVEL_0; + if(is_need_set) + val_time_log_level = time_log_level; + + return val_time_log_level; + } + //---------------------------------------------------------------------------- + int log_singletone::get_set_process_level(bool is_need_set/* = false*/, int process_log_level/* = LOG_LEVEL_0*/) + { + static int val_process_log_level = LOG_LEVEL_0; + if(is_need_set) + val_process_log_level = process_log_level; + + return val_process_log_level; + } + //---------------------------------------------------------------------------- + bool log_singletone::get_set_need_thread_id(bool is_need_set/* = false*/, bool is_need_val/* = false*/) + { + static bool is_need = false; + if(is_need_set) + is_need = is_need_val; + + return is_need; + } + //---------------------------------------------------------------------------- + bool log_singletone::get_set_need_proc_name(bool is_need_set/* = false*/, bool is_need_val/* = false*/) + { + static bool is_need = true; + if(is_need_set) + is_need = is_need_val; + + return is_need; + } + //---------------------------------------------------------------------------- + uint64_t log_singletone::get_set_err_count(bool is_need_set/* = false*/, uint64_t err_val/* = false*/) + { + static uint64_t err_count = 0; + if(is_need_set) + err_count = err_val; + + return err_count; + } + //---------------------------------------------------------------------------- +#ifdef _MSC_VER + void log_singletone::SetThreadName( DWORD dwThreadID, const char* threadName) + { +#define MS_VC_EXCEPTION 0x406D1388 + +#pragma pack(push,8) + typedef struct tagTHREADNAME_INFO + { + DWORD dwType; // Must be 0x1000. + LPCSTR szName; // Pointer to name (in user addr space). + DWORD dwThreadID; // Thread ID (-1=caller thread). + DWORD dwFlags; // Reserved for future use, must be zero. + } THREADNAME_INFO; +#pragma pack(pop) + + Sleep(10); + THREADNAME_INFO info; + info.dwType = 0x1000; + info.szName = (char*)threadName; + info.dwThreadID = dwThreadID; + info.dwFlags = 0; + + __try + { + RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(ULONG_PTR), (ULONG_PTR*)&info ); + } + __except(EXCEPTION_EXECUTE_HANDLER) + { + } + } +#endif + //---------------------------------------------------------------------------- + bool log_singletone::set_thread_log_prefix(const std::string& prefix) + { +#ifdef _MSC_VER + SetThreadName(-1, prefix.c_str()); +#endif + + logger* plogger = get_or_create_instance(); + if(!plogger) return false; + return plogger->set_thread_prefix(prefix); + } + //---------------------------------------------------------------------------- + std::string log_singletone::get_prefix_entry() + { + std::stringstream str_prefix; + //write time entry + if ( get_set_time_level() <= get_set_log_detalisation_level() ) + str_prefix << get_day_time_string() << " "; + + //write process info + logger* plogger = get_or_create_instance(); + //bool res = false; + if(!plogger) + { //globally uninitialized, create new logger for each call of get_prefix_entry() and then delete it + plogger = new logger(); + } + + //if ( get_set_need_proc_name() && get_set_process_level() <= get_set_log_detalisation_level() ) + // str_prefix << "[" << plogger->m_process_name << " (id=" << GetCurrentProcessId() << ")] "; +//#ifdef _MSC_VER_EX + if ( get_set_need_thread_id() /*&& get_set_tid_level() <= get_set_log_detalisation_level()*/ ) + str_prefix << "tid:" << misc_utils::get_thread_string_id() << " "; +//#endif + + if(plogger->m_thr_prefix_strings.size()) + { + CRITICAL_REGION_LOCAL(plogger->m_critical_sec); + std::string thr_str = misc_utils::get_thread_string_id(); + std::map::iterator it = plogger->m_thr_prefix_strings.find(thr_str); + if(it!=plogger->m_thr_prefix_strings.end()) + { + str_prefix << it->second; + } + } + + if(get_set_is_uninitialized()) + delete plogger; + + return str_prefix.str(); + } + //---------------------------------------------------------------------------- + bool log_singletone::init() + { + return true;/*do nothing here*/ + } + //---------------------------------------------------------------------------- + bool log_singletone::un_init() + { + //delete object + logger* plogger = get_set_instance_internal(); + if(plogger) delete plogger; + //set uninitialized + get_set_is_uninitialized(true, true); + get_set_instance_internal(true, NULL); + return true; + } + //---------------------------------------------------------------------------- + logger* log_singletone::get_or_create_instance() + { + logger* plogger = get_set_instance_internal(); + if(!plogger) + if(!get_set_is_uninitialized()) + get_set_instance_internal(true, plogger = new logger); + + return plogger; + } + //---------------------------------------------------------------------------- + logger* log_singletone::get_set_instance_internal(bool is_need_set/* = false*/, logger* pnew_logger_val/* = NULL*/) + { + static logger* val_plogger = NULL; + + if(is_need_set) + val_plogger = pnew_logger_val; + + return val_plogger; + } + //---------------------------------------------------------------------------- + bool log_singletone::get_set_is_uninitialized(bool is_need_set/* = false*/, bool is_uninitialized/* = false*/) + { + static bool val_is_uninitialized = false; + + if(is_need_set) + val_is_uninitialized = is_uninitialized; + + return val_is_uninitialized; + } + //---------------------------------------------------------------------------- + log_frame::log_frame(const std::string& name, int dlevel/* = LOG_LEVEL_2*/, const char* plog_name/* = NULL*/) + { +#ifdef _MSC_VER + int lasterr=::GetLastError(); +#endif + m_plog_name = plog_name; + if ( dlevel <= log_singletone::get_log_detalisation_level() ) + { + m_name = name; + std::stringstream ss; + ss << log_space::log_singletone::get_prefix_entry() << "-->>" << m_name << std::endl; + log_singletone::do_log_message(ss.str(), dlevel, console_color_default, false, m_plog_name); + } + m_level = dlevel; +#ifdef _MSC_VER + ::SetLastError(lasterr); +#endif + } + //---------------------------------------------------------------------------- + log_frame::~log_frame() + { +#ifdef _MSC_VER + int lasterr=::GetLastError(); +#endif + + if (m_level <= log_singletone::get_log_detalisation_level() ) + { + std::stringstream ss; + ss << log_space::log_singletone::get_prefix_entry() << "<<--" << m_name << std::endl; + log_singletone::do_log_message(ss.str(), m_level, console_color_default, false,m_plog_name); + } +#ifdef _MSC_VER + ::SetLastError(lasterr); +#endif + } + //---------------------------------------------------------------------------- + std::string get_win32_err_descr(int err_no) + { +#ifdef _MSC_VER + LPVOID lpMsgBuf; + + FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + err_no, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (char*) &lpMsgBuf, + 0, NULL ); + + std::string fix_sys_message = "(null)"; + if(lpMsgBuf) fix_sys_message = (char*)lpMsgBuf; + std::string::size_type a; + if ( (a = fix_sys_message.rfind( '\n' )) != std::string::npos ) + fix_sys_message.erase(a); + if ( (a = fix_sys_message.rfind( '\r' )) != std::string::npos ) + fix_sys_message.erase(a); + + LocalFree(lpMsgBuf); + return fix_sys_message; +#else + return "Not implemented yet"; +#endif + } + //---------------------------------------------------------------------------- + bool getwin32_err_text(std::stringstream& ref_message, int error_no) + { + ref_message << "win32 error:" << get_win32_err_descr(error_no); + return true; + } +} +} diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 44f50afa..918bfcf2 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -24,38 +24,27 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // +#pragma once #ifndef _MISC_LOG_EX_H_ #define _MISC_LOG_EX_H_ -//#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #if defined(WIN32) -#include -#else -#include +#if !defined(WIN32_LEAN_AND_MEAN) +#define WIN32_LEAN_AND_MEAN +#endif #endif -#include "static_initializer.h" -#include "string_tools.h" -#include "time_helper.h" -#include "misc_os_dependent.h" +#include +#include +#include +#include +#include +#include "misc_os_dependent.h" +#include "static_initializer.h" #include "syncobj.h" +#include "warnings.h" #define LOG_LEVEL_SILENT -1 @@ -121,50 +110,15 @@ namespace log_space struct ibase_log_stream { - ibase_log_stream(){} - virtual ~ibase_log_stream(){} - virtual bool out_buffer( const char* buffer, int buffer_len , int log_level, int color, const char* plog_name = NULL)=0; - virtual int get_type(){return 0;} + ibase_log_stream() {} + virtual ~ibase_log_stream() {} + virtual bool out_buffer(const char* buffer, int buffer_len , int log_level, int color, const char* plog_name = NULL)=0; + virtual int get_type() const { return 0; } - virtual bool set_max_logfile_size(uint64_t max_size){return true;}; - virtual bool set_log_rotate_cmd(const std::string& cmd){return true;}; + virtual bool set_max_logfile_size(uint64_t max_size) { return true; } + virtual bool set_log_rotate_cmd(const std::string& cmd) { return true; } }; - /************************************************************************/ - /* */ - /************************************************************************/ - /*struct ibase_log_value - { - public: - virtual void debug_out( std::stringstream* p_stream)const = 0; - };*/ - - /************************************************************************/ - /* */ - /************************************************************************/ - /*class log_message: public std::stringstream - { - public: - log_message(const log_message& lm): std::stringstream(), std::stringstream::basic_ios() - {} - log_message(){} - - template - log_message& operator<< (T t) - { - std::stringstream* pstrstr = this; - (*pstrstr) << t; - - return *this; - } - }; - inline - log_space::log_message& operator<<(log_space::log_message& sstream, const ibase_log_value& log_val) - { - log_val.debug_out(&sstream); - return sstream; - } - */ /************************************************************************/ /* */ /************************************************************************/ @@ -180,176 +134,21 @@ namespace log_space /************************************************************************/ /* */ /************************************************************************/ + + bool is_stdout_a_tty(); + void set_console_color(int color, bool bright); + void reset_console_color(); + bool rotate_log_file(const char* pfile_path); + //------------------------------------------------------------------------ #define max_dbg_str_len 80 #ifdef _MSC_VER class debug_output_stream: public ibase_log_stream { - virtual bool out_buffer( const char* buffer, int buffer_len , int log_level, int color, const char* plog_name = NULL) - { - for ( int i = 0; i < buffer_len; i = i + max_dbg_str_len ) - { - std::string s( buffer + i, buffer_len- i < max_dbg_str_len ? - buffer_len - i : max_dbg_str_len ); - - ::OutputDebugStringA( s.c_str() ); - } - return true; - } - + virtual bool out_buffer(const char* buffer, int buffer_len , int log_level, int color, const char* plog_name = NULL) override; }; #endif - inline bool is_stdout_a_tty() - { - static std::atomic initialized(false); - static std::atomic is_a_tty(false); - - if (!initialized.load(std::memory_order_acquire)) - { -#if defined(WIN32) - is_a_tty.store(0 != _isatty(_fileno(stdout)), std::memory_order_relaxed); -#else - is_a_tty.store(0 != isatty(fileno(stdout)), std::memory_order_relaxed); -#endif - initialized.store(true, std::memory_order_release); - } - - return is_a_tty.load(std::memory_order_relaxed); - } - - inline void set_console_color(int color, bool bright) - { - if (!is_stdout_a_tty()) - return; - - switch(color) - { - case console_color_default: - { -#ifdef WIN32 - HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE| (bright ? FOREGROUND_INTENSITY:0)); -#else - if(bright) - std::cout << "\033[1;37m"; - else - std::cout << "\033[0m"; -#endif - } - break; - case console_color_white: - { -#ifdef WIN32 - HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0)); -#else - if(bright) - std::cout << "\033[1;37m"; - else - std::cout << "\033[0;37m"; -#endif - } - break; - case console_color_red: - { -#ifdef WIN32 - HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | (bright ? FOREGROUND_INTENSITY:0)); -#else - if(bright) - std::cout << "\033[1;31m"; - else - std::cout << "\033[0;31m"; -#endif - } - break; - case console_color_green: - { -#ifdef WIN32 - HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | (bright ? FOREGROUND_INTENSITY:0)); -#else - if(bright) - std::cout << "\033[1;32m"; - else - std::cout << "\033[0;32m"; -#endif - } - break; - - case console_color_blue: - { -#ifdef WIN32 - HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_INTENSITY);//(bright ? FOREGROUND_INTENSITY:0)); -#else - if(bright) - std::cout << "\033[1;34m"; - else - std::cout << "\033[0;34m"; -#endif - } - break; - - case console_color_cyan: - { -#ifdef WIN32 - HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0)); -#else - if(bright) - std::cout << "\033[1;36m"; - else - std::cout << "\033[0;36m"; -#endif - } - break; - - case console_color_magenta: - { -#ifdef WIN32 - HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_RED | (bright ? FOREGROUND_INTENSITY:0)); -#else - if(bright) - std::cout << "\033[1;35m"; - else - std::cout << "\033[0;35m"; -#endif - } - break; - - case console_color_yellow: - { -#ifdef WIN32 - HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | (bright ? FOREGROUND_INTENSITY:0)); -#else - if(bright) - std::cout << "\033[1;33m"; - else - std::cout << "\033[0;33m"; -#endif - } - break; - - } - } - - inline void reset_console_color() { - if (!is_stdout_a_tty()) - return; - -#ifdef WIN32 - HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE); -#else - std::cout << "\033[0m"; - std::cout.flush(); -#endif - } - class console_output_stream: public ibase_log_stream { #ifdef _MSC_VER @@ -357,129 +156,22 @@ namespace log_space #endif public: - console_output_stream() - { -#ifdef _MSC_VER - - if(!::GetStdHandle(STD_OUTPUT_HANDLE)) - m_have_to_kill_console = true; - else - m_have_to_kill_console = false; - - ::AllocConsole(); -#endif - } - - ~console_output_stream() - { -#ifdef _MSC_VER - if(m_have_to_kill_console) - ::FreeConsole(); -#endif - } - int get_type(){return LOGGER_CONSOLE;} - - - - virtual bool out_buffer( const char* buffer, int buffer_len , int log_level, int color, const char* plog_name = NULL) - { - if(plog_name) - return true; //skip alternative logs from console - - set_console_color(color, log_level < 1); - -#ifdef _MSC_VER - const char* ptarget_buf = NULL; - char* pallocated_buf = NULL; - - // - int i = 0; - for(; i < buffer_len; i++) - if(buffer[i] == '\a') break; - if(i == buffer_len) - ptarget_buf = buffer; - else - { - pallocated_buf = new char[buffer_len]; - ptarget_buf = pallocated_buf; - for(i = 0; i < buffer_len; i++) - { - if(buffer[i] == '\a') - pallocated_buf[i] = '^'; - else - pallocated_buf[i] = buffer[i]; - } - } - - //uint32_t b = 0; - //::WriteConsoleA(::GetStdHandle(STD_OUTPUT_HANDLE), ptarget_buf, buffer_len, (DWORD*)&b, 0); - std::cout << ptarget_buf; - if(pallocated_buf) delete [] pallocated_buf; -#else - std::string buf(buffer, buffer_len); - for(size_t i = 0; i!= buf.size(); i++) - { - if(buf[i] == 7 || buf[i] == -107) - buf[i] = '^'; - } - - std::cout << buf; -#endif - reset_console_color(); - return true; - } - + console_output_stream(); + virtual ~console_output_stream(); + virtual int get_type() const override { return LOGGER_CONSOLE; } + virtual bool out_buffer(const char* buffer, int buffer_len , int log_level, int color, const char* plog_name = NULL) override; }; - inline bool rotate_log_file(const char* pfile_path) - { -#ifdef _MSC_VER - if(!pfile_path) - return false; - - std::string file_path = pfile_path; - std::string::size_type a = file_path .rfind('.'); - if ( a != std::string::npos ) - file_path .erase( a, file_path .size()); - - ::DeleteFileA( (file_path + ".0").c_str() ); - ::MoveFileA( (file_path + ".log").c_str(), (file_path + ".0").c_str() ); -#else - return false;//not implemented yet -#endif - return true; - } - - - - //--------------------------------------------------------------------------// class file_output_stream : public ibase_log_stream { public: typedef std::map named_log_streams; - file_output_stream( std::string default_log_file_name, std::string log_path ) - { - m_default_log_filename = default_log_file_name; - m_max_logfile_size = 0; - m_default_log_path = log_path; - m_pdefault_file_stream = add_new_stream_and_open(default_log_file_name.c_str()); - } + file_output_stream(std::string default_log_file_name, std::string log_path); + ~file_output_stream(); - ~file_output_stream() - { - for(named_log_streams::iterator it = m_log_file_names.begin(); it!=m_log_file_names.end(); it++) - { - if ( it->second->is_open() ) - { - it->second->flush(); - it->second->close(); - } - delete it->second; - } - } private: named_log_streams m_log_file_names; std::string m_default_log_path; @@ -488,109 +180,14 @@ namespace log_space std::string m_default_log_filename; uint64_t m_max_logfile_size; + virtual int get_type() const override { return LOGGER_FILE; } + virtual bool out_buffer(const char* buffer, int buffer_len, int log_level, int color, const char* plog_name = NULL) override; + virtual bool set_max_logfile_size(uint64_t max_size) override; + virtual bool set_log_rotate_cmd(const std::string& cmd) override; - std::ofstream* add_new_stream_and_open(const char* pstream_name) - { - //log_space::rotate_log_file((m_default_log_path + "\\" + pstream_name).c_str()); - - std::ofstream* pstream = (m_log_file_names[pstream_name] = new std::ofstream); - std::string target_path = m_default_log_path + "/" + pstream_name; - pstream->open( target_path.c_str(), std::ios_base::out | std::ios::app /*ios_base::trunc */); - if(pstream->fail()) - return NULL; - return pstream; - } - - bool set_max_logfile_size(uint64_t max_size) - { - m_max_logfile_size = max_size; - return true; - } - - bool set_log_rotate_cmd(const std::string& cmd) - { - m_log_rotate_cmd = cmd; - return true; - } - - - - virtual bool out_buffer( const char* buffer, int buffer_len, int log_level, int color, const char* plog_name = NULL ) - { - std::ofstream* m_target_file_stream = m_pdefault_file_stream; - if(plog_name) - { //find named stream - named_log_streams::iterator it = m_log_file_names.find(plog_name); - if(it == m_log_file_names.end()) - m_target_file_stream = add_new_stream_and_open(plog_name); - else - m_target_file_stream = it->second; - } - if(!m_target_file_stream || !m_target_file_stream->is_open()) - return false;//TODO: add assert here - - m_target_file_stream->write(buffer, buffer_len ); - m_target_file_stream->flush(); - - if(m_max_logfile_size) - { - std::ofstream::pos_type pt = m_target_file_stream->tellp(); - uint64_t current_sz = pt; - if(current_sz > m_max_logfile_size) - { - std::cout << "current_sz= " << current_sz << " m_max_logfile_size= " << m_max_logfile_size << std::endl; - std::string log_file_name; - if(!plog_name) - log_file_name = m_default_log_filename; - else - log_file_name = plog_name; - - m_target_file_stream->close(); - std::string new_log_file_name = log_file_name; - - time_t tm = 0; - time(&tm); - - int err_count = 0; - boost::system::error_code ec; - do - { - new_log_file_name = string_tools::cut_off_extension(log_file_name); - if(err_count) - new_log_file_name += misc_utils::get_time_str_v2(tm) + "(" + boost::lexical_cast(err_count) + ")" + ".log"; - else - new_log_file_name += misc_utils::get_time_str_v2(tm) + ".log"; - - err_count++; - }while(boost::filesystem::exists(m_default_log_path + "/" + new_log_file_name, ec)); - - std::string new_log_file_path = m_default_log_path + "/" + new_log_file_name; - boost::filesystem::rename(m_default_log_path + "/" + log_file_name, new_log_file_path, ec); - if(ec) - { - std::cout << "Filed to rename, ec = " << ec.message() << std::endl; - } - - if(m_log_rotate_cmd.size()) - { - - std::string m_log_rotate_cmd_local_copy = m_log_rotate_cmd; - //boost::replace_all(m_log_rotate_cmd, "[*SOURCE*]", new_log_file_path); - boost::replace_all(m_log_rotate_cmd_local_copy, "[*TARGET*]", new_log_file_path); - - misc_utils::call_sys_cmd(m_log_rotate_cmd_local_copy); - } - - m_target_file_stream->open( (m_default_log_path + "/" + log_file_name).c_str(), std::ios_base::out | std::ios::app /*ios_base::trunc */); - if(m_target_file_stream->fail()) - return false; - } - } - - return true; - } - int get_type(){return LOGGER_FILE;} + std::ofstream* add_new_stream_and_open(const char* pstream_name); }; + /************************************************************************/ /* */ /************************************************************************/ @@ -599,272 +196,59 @@ namespace log_space public: typedef std::list > streams_container; - log_stream_splitter(){} - ~log_stream_splitter() - { - //free pointers - std::for_each(m_log_streams.begin(), m_log_streams.end(), delete_ptr()); - } + log_stream_splitter() { } + ~log_stream_splitter(); - bool set_max_logfile_size(uint64_t max_size) - { - for(streams_container::iterator it = m_log_streams.begin(); it!=m_log_streams.end();it++) - it->first->set_max_logfile_size(max_size); - return true; - } + bool set_max_logfile_size(uint64_t max_size); + bool set_log_rotate_cmd(const std::string& cmd); + bool do_log_message(const std::string& rlog_mes, int log_level, int color, const char* plog_name = NULL); + bool add_logger(int type, const char* pdefault_file_name, const char* pdefault_log_folder, int log_level_limit = LOG_LEVEL_4); + bool add_logger(ibase_log_stream* pstream, int log_level_limit = LOG_LEVEL_4); + bool remove_logger(int type); - bool set_log_rotate_cmd(const std::string& cmd) - { - for(streams_container::iterator it = m_log_streams.begin(); it!=m_log_streams.end();it++) - it->first->set_log_rotate_cmd(cmd); - return true; - } - - bool do_log_message(const std::string& rlog_mes, int log_level, int color, const char* plog_name = NULL) - { - std::string str_mess = rlog_mes; - size_t str_len = str_mess.size(); - const char* pstr = str_mess.c_str(); - for(streams_container::iterator it = m_log_streams.begin(); it!=m_log_streams.end();it++) - if(it->second >= log_level) - it->first->out_buffer(pstr, (int)str_len, log_level, color, plog_name); - return true; - } - - bool add_logger( int type, const char* pdefault_file_name, const char* pdefault_log_folder, int log_level_limit = LOG_LEVEL_4 ) - { - ibase_log_stream* ls = NULL; - - switch( type ) - { - case LOGGER_FILE: - ls = new file_output_stream( pdefault_file_name, pdefault_log_folder ); - break; - - case LOGGER_DEBUGGER: -#ifdef _MSC_VER - ls = new debug_output_stream( ); -#else - return false;//not implemented yet -#endif - break; - case LOGGER_CONSOLE: - ls = new console_output_stream( ); - break; - } - - if ( ls ) { - m_log_streams.push_back(streams_container::value_type(ls, log_level_limit)); - return true; - } - return ls ? true:false; - } - bool add_logger( ibase_log_stream* pstream, int log_level_limit = LOG_LEVEL_4 ) - { - m_log_streams.push_back(streams_container::value_type(pstream, log_level_limit) ); - return true; - } - - bool remove_logger(int type) - { - streams_container::iterator it = m_log_streams.begin(); - for(;it!=m_log_streams.end(); it++) - { - if(it->first->get_type() == type) - { - delete it->first; - m_log_streams.erase(it); - return true; - } - } - return false; - - } - - protected: private: - - streams_container m_log_streams; + streams_container m_log_streams; }; /************************************************************************/ /* */ /************************************************************************/ - inline int get_set_log_detalisation_level(bool is_need_set = false, int log_level_to_set = LOG_LEVEL_1); - inline int get_set_time_level(bool is_need_set = false, int time_log_level = LOG_LEVEL_0); - inline bool get_set_need_thread_id(bool is_need_set = false, bool is_need_val = false); - inline bool get_set_need_proc_name(bool is_need_set = false, bool is_need_val = false); + int get_set_log_detalisation_level(bool is_need_set = false, int log_level_to_set = LOG_LEVEL_1); + int get_set_time_level(bool is_need_set = false, int time_log_level = LOG_LEVEL_0); + bool get_set_need_thread_id(bool is_need_set = false, bool is_need_val = false); + bool get_set_need_proc_name(bool is_need_set = false, bool is_need_val = false); + std::string get_daytime_string2(); + std::string get_day_time_string(); + std::string get_time_string(); - inline std::string get_daytime_string2() - { - boost::posix_time::ptime p = boost::posix_time::microsec_clock::local_time(); - return misc_utils::get_time_str_v3(p); - } - inline std::string get_day_time_string() - { - return get_daytime_string2(); - //time_t tm = 0; - //time(&tm); - //return misc_utils::get_time_str(tm); - } - - inline std::string get_time_string() - { - return get_daytime_string2(); - - } #ifdef _MSC_VER - inline std::string get_time_string_adv(SYSTEMTIME* pst = NULL) - { - SYSTEMTIME st = {0}; - if(!pst) - { - pst = &st; - GetSystemTime(&st); - } - std::stringstream str_str; - str_str.fill('0'); - str_str << std::setw(2) << pst->wHour << "_" - << std::setw(2) << pst->wMinute << "_" - << std::setw(2) << pst->wSecond << "_" - << std::setw(3) << pst->wMilliseconds; - return str_str.str(); - } + inline std::string get_time_string_adv(SYSTEMTIME* pst = NULL); #endif - - - - - class logger + class logger { public: friend class log_singletone; - logger() - { - CRITICAL_REGION_BEGIN(m_critical_sec); - init(); - CRITICAL_REGION_END(); - } - ~logger() - { - } + logger(); + ~logger() { } - bool set_max_logfile_size(uint64_t max_size) - { - CRITICAL_REGION_BEGIN(m_critical_sec); - m_log_target.set_max_logfile_size(max_size); - CRITICAL_REGION_END(); - return true; - } + bool set_max_logfile_size(uint64_t max_size); + bool set_log_rotate_cmd(const std::string& cmd); + bool take_away_journal(std::list& journal); + bool do_log_message(const std::string& rlog_mes, int log_level, int color, bool add_to_journal = false, const char* plog_name = NULL); + bool add_logger(int type, const char* pdefault_file_name, const char* pdefault_log_folder , int log_level_limit = LOG_LEVEL_4); + bool add_logger(ibase_log_stream* pstream, int log_level_limit = LOG_LEVEL_4); + bool remove_logger(int type); + bool set_thread_prefix(const std::string& prefix); + std::string get_default_log_file() { return m_default_log_file; } + std::string get_default_log_folder() { return m_default_log_folder; } - bool set_log_rotate_cmd(const std::string& cmd) - { - CRITICAL_REGION_BEGIN(m_critical_sec); - m_log_target.set_log_rotate_cmd(cmd); - CRITICAL_REGION_END(); - return true; - } - - bool take_away_journal(std::list& journal) - { - CRITICAL_REGION_BEGIN(m_critical_sec); - m_journal.swap(journal); - CRITICAL_REGION_END(); - return true; - } - - bool do_log_message(const std::string& rlog_mes, int log_level, int color, bool add_to_journal = false, const char* plog_name = NULL) - { - CRITICAL_REGION_BEGIN(m_critical_sec); - m_log_target.do_log_message(rlog_mes, log_level, color, plog_name); - if(add_to_journal) - m_journal.push_back(rlog_mes); - - return true; - CRITICAL_REGION_END(); - } - - bool add_logger( int type, const char* pdefault_file_name, const char* pdefault_log_folder , int log_level_limit = LOG_LEVEL_4) - { - CRITICAL_REGION_BEGIN(m_critical_sec); - return m_log_target.add_logger( type, pdefault_file_name, pdefault_log_folder, log_level_limit); - CRITICAL_REGION_END(); - } - bool add_logger( ibase_log_stream* pstream, int log_level_limit = LOG_LEVEL_4) - { - CRITICAL_REGION_BEGIN(m_critical_sec); - return m_log_target.add_logger(pstream, log_level_limit); - CRITICAL_REGION_END(); - } - - bool remove_logger(int type) - { - CRITICAL_REGION_BEGIN(m_critical_sec); - return m_log_target.remove_logger(type); - CRITICAL_REGION_END(); - } - - - bool set_thread_prefix(const std::string& prefix) - { - CRITICAL_REGION_BEGIN(m_critical_sec); - m_thr_prefix_strings[misc_utils::get_thread_string_id()] = prefix; - CRITICAL_REGION_END(); - return true; - } - - - std::string get_default_log_file() - { - return m_default_log_file; - } - - std::string get_default_log_folder() - { - return m_default_log_folder; - } - - protected: private: - bool init() - { - // - - m_process_name = string_tools::get_current_module_name(); - - init_log_path_by_default(); - - //init default set of loggers - init_default_loggers(); - - std::stringstream ss; - ss << get_time_string() << " Init logging. Level=" << get_set_log_detalisation_level() - << " Log path=" << m_default_log_folder << std::endl; - this->do_log_message(ss.str(), console_color_white, LOG_LEVEL_0); - return true; - } - bool init_default_loggers() - { - //TODO: - return true; - } - - bool init_log_path_by_default() - { - //load process name - m_default_log_folder = string_tools::get_current_module_folder(); - - m_default_log_file = m_process_name; - std::string::size_type a = m_default_log_file.rfind('.'); - if ( a != std::string::npos ) - m_default_log_file.erase( a, m_default_log_file.size()); - m_default_log_file += ".log"; - - return true; - } + bool init(); + bool init_default_loggers(); + bool init_log_path_by_default(); log_stream_splitter m_log_target; @@ -875,6 +259,7 @@ namespace log_space std::list m_journal; critical_section m_critical_sec; }; + /************************************************************************/ /* */ /************************************************************************/ @@ -883,372 +268,63 @@ namespace log_space public: friend class initializer; friend class logger; - static int get_log_detalisation_level() - { - get_or_create_instance();//to initialize logger, if it not initialized - return get_set_log_detalisation_level(); - } - static bool is_filter_error(int error_code) - { - return false; - } + static int get_log_detalisation_level(); + static bool is_filter_error(int error_code); + static bool do_log_message(const std::string& rlog_mes, int log_level, int color, bool keep_in_journal, const char* plog_name = NULL); + static bool take_away_journal(std::list& journal); + static bool set_max_logfile_size(uint64_t file_size); + static bool set_log_rotate_cmd(const std::string& cmd); + static bool add_logger(int type, const char* pdefault_file_name, const char* pdefault_log_folder, int log_level_limit = LOG_LEVEL_4); + static std::string get_default_log_file(); + static std::string get_default_log_folder(); + static bool add_logger( ibase_log_stream* pstream, int log_level_limit = LOG_LEVEL_4); + static bool remove_logger(int type); - - static bool do_log_message(const std::string& rlog_mes, int log_level, int color, bool keep_in_journal, const char* plog_name = NULL) - { - logger* plogger = get_or_create_instance(); - bool res = false; - if(plogger) - res = plogger->do_log_message(rlog_mes, log_level, color, keep_in_journal, plog_name); - else - { //globally uninitialized, create new logger for each call of do_log_message() and then delete it - plogger = new logger(); - //TODO: some extra initialization - res = plogger->do_log_message(rlog_mes, log_level, color, keep_in_journal, plog_name); - delete plogger; - plogger = NULL; - } - return res; - } - - static bool take_away_journal(std::list& journal) - { - logger* plogger = get_or_create_instance(); - bool res = false; - if(plogger) - res = plogger->take_away_journal(journal); - - return res; - } - - static bool set_max_logfile_size(uint64_t file_size) - { - logger* plogger = get_or_create_instance(); - if(!plogger) return false; - return plogger->set_max_logfile_size(file_size); - } - - - static bool set_log_rotate_cmd(const std::string& cmd) - { - logger* plogger = get_or_create_instance(); - if(!plogger) return false; - return plogger->set_log_rotate_cmd(cmd); - } - - - static bool add_logger( int type, const char* pdefault_file_name, const char* pdefault_log_folder, int log_level_limit = LOG_LEVEL_4) - { - logger* plogger = get_or_create_instance(); - if(!plogger) return false; - return plogger->add_logger(type, pdefault_file_name, pdefault_log_folder, log_level_limit); - } - - static std::string get_default_log_file() - { - logger* plogger = get_or_create_instance(); - if(plogger) - return plogger->get_default_log_file(); - - return ""; - } - - static std::string get_default_log_folder() - { - logger* plogger = get_or_create_instance(); - if(plogger) - return plogger->get_default_log_folder(); - - return ""; - } - - static bool add_logger( ibase_log_stream* pstream, int log_level_limit = LOG_LEVEL_4 ) - { - logger* plogger = get_or_create_instance(); - if(!plogger) return false; - return plogger->add_logger(pstream, log_level_limit); - } - - - static bool remove_logger( int type ) - { - logger* plogger = get_or_create_instance(); - if(!plogger) return false; - return plogger->remove_logger(type); - } PUSH_WARNINGS DISABLE_GCC_WARNING(maybe-uninitialized) - static int get_set_log_detalisation_level(bool is_need_set = false, int log_level_to_set = LOG_LEVEL_1) - { - static int log_detalisation_level = LOG_LEVEL_1; - if(is_need_set) - log_detalisation_level = log_level_to_set; - return log_detalisation_level; - } + static int get_set_log_detalisation_level(bool is_need_set = false, int log_level_to_set = LOG_LEVEL_1); POP_WARNINGS - static int get_set_time_level(bool is_need_set = false, int time_log_level = LOG_LEVEL_0) - { - static int val_time_log_level = LOG_LEVEL_0; - if(is_need_set) - val_time_log_level = time_log_level; - - return val_time_log_level; - } - - static int get_set_process_level(bool is_need_set = false, int process_log_level = LOG_LEVEL_0) - { - static int val_process_log_level = LOG_LEVEL_0; - if(is_need_set) - val_process_log_level = process_log_level; - - return val_process_log_level; - } - - /*static int get_set_tid_level(bool is_need_set = false, int tid_log_level = LOG_LEVEL_0) - { - static int val_tid_log_level = LOG_LEVEL_0; - if(is_need_set) - val_tid_log_level = tid_log_level; - - return val_tid_log_level; - }*/ - - static bool get_set_need_thread_id(bool is_need_set = false, bool is_need_val = false) - { - static bool is_need = false; - if(is_need_set) - is_need = is_need_val; - - return is_need; - } - - static bool get_set_need_proc_name(bool is_need_set = false, bool is_need_val = false) - { - static bool is_need = true; - if(is_need_set) - is_need = is_need_val; - - return is_need; - } - static uint64_t get_set_err_count(bool is_need_set = false, uint64_t err_val = false) - { - static uint64_t err_count = 0; - if(is_need_set) - err_count = err_val; - - return err_count; - } + static int get_set_time_level(bool is_need_set = false, int time_log_level = LOG_LEVEL_0); + static int get_set_process_level(bool is_need_set = false, int process_log_level = LOG_LEVEL_0); + static bool get_set_need_thread_id(bool is_need_set = false, bool is_need_val = false); + static bool get_set_need_proc_name(bool is_need_set = false, bool is_need_val = false); + static uint64_t get_set_err_count(bool is_need_set = false, uint64_t err_val = false); #ifdef _MSC_VER - - - static void SetThreadName( DWORD dwThreadID, const char* threadName) - { -#define MS_VC_EXCEPTION 0x406D1388 - -#pragma pack(push,8) - typedef struct tagTHREADNAME_INFO - { - DWORD dwType; // Must be 0x1000. - LPCSTR szName; // Pointer to name (in user addr space). - DWORD dwThreadID; // Thread ID (-1=caller thread). - DWORD dwFlags; // Reserved for future use, must be zero. - } THREADNAME_INFO; -#pragma pack(pop) - - - - Sleep(10); - THREADNAME_INFO info; - info.dwType = 0x1000; - info.szName = (char*)threadName; - info.dwThreadID = dwThreadID; - info.dwFlags = 0; - - __try - { - RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(ULONG_PTR), (ULONG_PTR*)&info ); - } - __except(EXCEPTION_EXECUTE_HANDLER) - { - } - } + static void SetThreadName( DWORD dwThreadID, const char* threadName); #endif - static bool set_thread_log_prefix(const std::string& prefix) - { -#ifdef _MSC_VER - SetThreadName(-1, prefix.c_str()); -#endif - - - logger* plogger = get_or_create_instance(); - if(!plogger) return false; - return plogger->set_thread_prefix(prefix); - } - - - static std::string get_prefix_entry() - { - std::stringstream str_prefix; - //write time entry - if ( get_set_time_level() <= get_set_log_detalisation_level() ) - str_prefix << get_day_time_string() << " "; - - //write process info - logger* plogger = get_or_create_instance(); - //bool res = false; - if(!plogger) - { //globally uninitialized, create new logger for each call of get_prefix_entry() and then delete it - plogger = new logger(); - } - - //if ( get_set_need_proc_name() && get_set_process_level() <= get_set_log_detalisation_level() ) - // str_prefix << "[" << plogger->m_process_name << " (id=" << GetCurrentProcessId() << ")] "; -//#ifdef _MSC_VER_EX - if ( get_set_need_thread_id() /*&& get_set_tid_level() <= get_set_log_detalisation_level()*/ ) - str_prefix << "tid:" << misc_utils::get_thread_string_id() << " "; -//#endif - - if(plogger->m_thr_prefix_strings.size()) - { - CRITICAL_REGION_LOCAL(plogger->m_critical_sec); - std::string thr_str = misc_utils::get_thread_string_id(); - std::map::iterator it = plogger->m_thr_prefix_strings.find(thr_str); - if(it!=plogger->m_thr_prefix_strings.end()) - { - str_prefix << it->second; - } - } - - - if(get_set_is_uninitialized()) - delete plogger; - - return str_prefix.str(); - } + static bool set_thread_log_prefix(const std::string& prefix); + static std::string get_prefix_entry(); private: - log_singletone(){}//restric to create an instance + log_singletone() { } //restric to create an instance //static initializer m_log_initializer;//must be in one .cpp file (for example main.cpp) via DEFINE_LOGGING macro - static bool init() - { - return true;/*do nothing here*/ - } - static bool un_init() - { - //delete object - logger* plogger = get_set_instance_internal(); - if(plogger) delete plogger; - //set uninitialized - get_set_is_uninitialized(true, true); - get_set_instance_internal(true, NULL); - return true; - } + static bool init(); + static bool un_init(); - static logger* get_or_create_instance() - { - logger* plogger = get_set_instance_internal(); - if(!plogger) - if(!get_set_is_uninitialized()) - get_set_instance_internal(true, plogger = new logger); - - return plogger; - } - - static logger* get_set_instance_internal(bool is_need_set = false, logger* pnew_logger_val = NULL) - { - static logger* val_plogger = NULL; - - if(is_need_set) - val_plogger = pnew_logger_val; - - return val_plogger; - } - - static bool get_set_is_uninitialized(bool is_need_set = false, bool is_uninitialized = false) - { - static bool val_is_uninitialized = false; - - if(is_need_set) - val_is_uninitialized = is_uninitialized; - - return val_is_uninitialized; - } - //static int get_set_error_filter(bool is_need_set = false) + static logger* get_or_create_instance(); + static logger* get_set_instance_internal(bool is_need_set = false, logger* pnew_logger_val = NULL); + static bool get_set_is_uninitialized(bool is_need_set = false, bool is_uninitialized = false); }; const static initializer log_initializer; - /************************************************************************/ - /* */ -// /************************************************************************/ -// class log_array_value -// { -// int num; -// log_message& m_ref_log_mes; -// -// public: -// -// log_array_value( log_message& log_mes ) : num(0), m_ref_log_mes(log_mes) {} -// -// void operator ( )( ibase_log_value *val ) { -// m_ref_log_mes << "\n[" << num++ << "] "/* << val*/; } -// -// -// template -// void operator ()(T &value ) -// { -// m_ref_log_mes << "\n[" << num++ << "] " << value; -// } -// }; class log_frame { - std::string m_name; - int m_level; - const char* m_plog_name; + std::string m_name; + int m_level; + const char* m_plog_name; + public: - - log_frame(const std::string& name, int dlevel = LOG_LEVEL_2 , const char* plog_name = NULL) - { -#ifdef _MSC_VER - int lasterr=::GetLastError(); -#endif - m_plog_name = plog_name; - if ( dlevel <= log_singletone::get_log_detalisation_level() ) - { - m_name = name; - std::stringstream ss; - ss << log_space::log_singletone::get_prefix_entry() << "-->>" << m_name << std::endl; - log_singletone::do_log_message(ss.str(), dlevel, console_color_default, false, m_plog_name); - } - m_level = dlevel; -#ifdef _MSC_VER - ::SetLastError(lasterr); -#endif - } - ~log_frame() - { -#ifdef _MSC_VER - int lasterr=::GetLastError(); -#endif - - if (m_level <= log_singletone::get_log_detalisation_level() ) - { - std::stringstream ss; - ss << log_space::log_singletone::get_prefix_entry() << "<<--" << m_name << std::endl; - log_singletone::do_log_message(ss.str(), m_level, console_color_default, false,m_plog_name); - } -#ifdef _MSC_VER - ::SetLastError(lasterr); -#endif - } + log_frame(const std::string& name, int dlevel = LOG_LEVEL_2 , const char* plog_name = NULL); + ~log_frame(); }; - inline int get_set_time_level(bool is_need_set, int time_log_level) + inline int get_set_time_level(bool is_need_set, int time_log_level) { return log_singletone::get_set_time_level(is_need_set, time_log_level); } @@ -1269,41 +345,10 @@ POP_WARNINGS return log_singletone::get_set_need_proc_name(is_need_set, is_need_val); } - inline std::string get_win32_err_descr(int err_no) - { -#ifdef _MSC_VER - LPVOID lpMsgBuf; - - FormatMessageA( - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM, - NULL, - err_no, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (char*) &lpMsgBuf, - 0, NULL ); - - std::string fix_sys_message = "(null)"; - if(lpMsgBuf) fix_sys_message = (char*)lpMsgBuf; - std::string::size_type a; - if ( (a = fix_sys_message.rfind( '\n' )) != std::string::npos ) - fix_sys_message.erase(a); - if ( (a = fix_sys_message.rfind( '\r' )) != std::string::npos ) - fix_sys_message.erase(a); - - LocalFree(lpMsgBuf); - return fix_sys_message; -#else - return "Not implemented yet"; -#endif - } - - inline bool getwin32_err_text(std::stringstream& ref_message, int error_no) - { - ref_message << "win32 error:" << get_win32_err_descr(error_no); - return true; - } + inline std::string get_win32_err_descr(int err_no); + inline bool getwin32_err_text(std::stringstream& ref_message, int error_no); } + #if defined(_DEBUG) || defined(__GNUC__) #define ENABLE_LOGGING_INTERNAL #endif @@ -1342,6 +387,9 @@ POP_WARNINGS #define LOG_FRAME2(log_name, x, y) epee::log_space::log_frame frame(x, y, log_name) +#define LOG_WARNING2(log_name, x, y) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() )\ + {std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << "WARNING " << __FILE__ << ":" << __LINE__ << " " << x << std::endl; epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_red, true, log_name);LOCAL_ASSERT(0); epee::log_space::log_singletone::get_set_err_count(true, epee::log_space::log_singletone::get_set_err_count()+1);}} + #else @@ -1362,6 +410,8 @@ POP_WARNINGS #define LOG_FRAME2(log_name, x, y) +#define LOG_WARNING2(log_name, x, level) + #endif @@ -1400,6 +450,7 @@ POP_WARNINGS //#define LOGWIN_PLATFORM_ERROR(err_no) LOGWINDWOS_PLATFORM_ERROR2(LOG_DEFAULT_TARGET, err_no) #define LOG_SOCKET_ERROR(err_no) LOG_SOCKET_ERROR2(LOG_DEFAULT_TARGET, err_no) //#define LOGWIN_PLATFORM_ERROR_UNCRITICAL(mess) LOGWINDWOS_PLATFORM_ERROR_UNCRITICAL2(LOG_DEFAULT_TARGET, mess) +#define LOG_WARNING(mess, level) LOG_WARNING2(LOG_DEFAULT_TARGET, mess, level) #define ENDL std::endl diff --git a/contrib/epee/include/misc_os_dependent.cpp b/contrib/epee/include/misc_os_dependent.cpp new file mode 100644 index 00000000..3bff6585 --- /dev/null +++ b/contrib/epee/include/misc_os_dependent.cpp @@ -0,0 +1,97 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "misc_os_dependent.h" + +#include + +#include + +#ifdef __MACH__ +#include +#include +#endif + +namespace epee +{ +namespace misc_utils +{ + + uint64_t get_tick_count() + { +#if defined(_MSC_VER) + return ::GetTickCount64(); +#elif defined(__MACH__) + clock_serv_t cclock; + mach_timespec_t mts; + + host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &cclock); + clock_get_time(cclock, &mts); + mach_port_deallocate(mach_task_self(), cclock); + + return (mts.tv_sec * 1000) + (mts.tv_nsec/1000000); +#else + struct timespec ts; + if(clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + return 0; + } + return (ts.tv_sec * 1000) + (ts.tv_nsec/1000000); +#endif + } + + int call_sys_cmd(const std::string& cmd) + { + std::cout << "# " << cmd << std::endl; + + FILE * fp ; + //char tstCommand[] ="ls *"; + char path[1000] = {0}; +#if !defined(__GNUC__) + fp = _popen(cmd.c_str(), "r"); +#else + fp = popen(cmd.c_str(), "r"); +#endif + while ( fgets( path, 1000, fp ) != NULL ) + std::cout << path; + +#if !defined(__GNUC__) + _pclose(fp); +#else + pclose(fp); +#endif + return 0; + } + + std::string get_thread_string_id() + { +#if defined(_MSC_VER) + return boost::lexical_cast(GetCurrentThreadId()); +#elif defined(__GNUC__) + return boost::lexical_cast(pthread_self()); +#endif + } +} +} diff --git a/contrib/epee/include/misc_os_dependent.h b/contrib/epee/include/misc_os_dependent.h index 4d9c991e..eb4a3a8a 100644 --- a/contrib/epee/include/misc_os_dependent.h +++ b/contrib/epee/include/misc_os_dependent.h @@ -23,86 +23,30 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // + +#pragma once + +#include +#include + #ifdef WIN32 #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif - //#ifdef _WIN32_WINNT - // #undef _WIN32_WINNT - // #define _WIN32_WINNT 0x0600 - //#endif - - -#include + #if !defined(NOMINMAX) + #define NOMINMAX 1 + #endif // !defined(NOMINMAX) + + #include #endif -#ifdef __MACH__ -#include -#include -#endif - -#pragma once namespace epee { namespace misc_utils { - - inline uint64_t get_tick_count() - { -#if defined(_MSC_VER) - return ::GetTickCount64(); -#elif defined(__MACH__) - clock_serv_t cclock; - mach_timespec_t mts; - - host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &cclock); - clock_get_time(cclock, &mts); - mach_port_deallocate(mach_task_self(), cclock); - - return (mts.tv_sec * 1000) + (mts.tv_nsec/1000000); -#else - struct timespec ts; - if(clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { - return 0; - } - return (ts.tv_sec * 1000) + (ts.tv_nsec/1000000); -#endif - } - - - inline int call_sys_cmd(const std::string& cmd) - { - std::cout << "# " << cmd << std::endl; - - FILE * fp ; - //char tstCommand[] ="ls *"; - char path[1000] = {0}; -#if !defined(__GNUC__) - fp = _popen(cmd.c_str(), "r"); -#else - fp = popen(cmd.c_str(), "r"); -#endif - while ( fgets( path, 1000, fp ) != NULL ) - std::cout << path; - -#if !defined(__GNUC__) - _pclose(fp); -#else - pclose(fp); -#endif - return 0; - - } - - - inline std::string get_thread_string_id() - { -#if defined(_MSC_VER) - return boost::lexical_cast(GetCurrentThreadId()); -#elif defined(__GNUC__) - return boost::lexical_cast(pthread_self()); -#endif - } + uint64_t get_tick_count(); + int call_sys_cmd(const std::string& cmd); + std::string get_thread_string_id(); } } diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 74f1e5c4..0265d57e 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -30,10 +30,12 @@ #include #include #include +#include #include #include #include #include +#include "include_base_utils.h" #include "misc_language.h" #include "pragma_comp_defs.h" @@ -304,7 +306,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) if(m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT) { send_guard.unlock(); -// LOG_ERROR("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); + LOG_WARNING("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection", LOG_LEVEL_2); close(); return false; } diff --git a/contrib/epee/include/net/http_protocol_handler.h b/contrib/epee/include/net/http_protocol_handler.h index aed90977..72b4b575 100644 --- a/contrib/epee/include/net/http_protocol_handler.h +++ b/contrib/epee/include/net/http_protocol_handler.h @@ -31,6 +31,7 @@ #define _HTTP_SERVER_H_ #include +#include "include_base_utils.h" #include "net_utils_base.h" #include "to_nonconst_iterator.h" #include "http_base.h" diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index 78b46427..fc091a21 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -28,8 +28,10 @@ #include #include #include "http_protocol_handler.h" +#include "include_base_utils.h" #include "reg_exp_definer.h" #include "string_tools.h" +#include "time_helper.h" #include "file_io_utils.h" #include "net_parse_helpers.h" diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 20146013..12ad9d97 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -28,6 +28,7 @@ #pragma once #include "http_base.h" #include "jsonrpc_structs.h" +#include "misc_os_dependent.h" #include "storages/portable_storage.h" #include "storages/portable_storage_template_helper.h" @@ -59,7 +60,7 @@ else if(query_info.m_URI == s_pattern) \ { \ handled = true; \ - uint64_t ticks = misc_utils::get_tick_count(); \ + uint64_t ticks = epee::misc_utils::get_tick_count(); \ boost::value_initialized req; \ bool parse_res = epee::serialization::load_t_from_json(static_cast(req), query_info.m_body); \ CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse json: \r\n" << query_info.m_body); \ @@ -84,11 +85,11 @@ else if(query_info.m_URI == s_pattern) \ { \ handled = true; \ - uint64_t ticks = misc_utils::get_tick_count(); \ + uint64_t ticks = epee::misc_utils::get_tick_count(); \ boost::value_initialized req; \ bool parse_res = epee::serialization::load_t_from_binary(static_cast(req), query_info.m_body); \ CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse bin body data, body size=" << query_info.m_body.size()); \ - uint64_t ticks1 = misc_utils::get_tick_count(); \ + uint64_t ticks1 = epee::misc_utils::get_tick_count(); \ boost::value_initialized resp;\ if(!callback_f(static_cast(req), static_cast(resp), m_conn_context)) \ { \ @@ -97,7 +98,7 @@ response_info.m_response_comment = "Internal Server Error"; \ return true; \ } \ - uint64_t ticks2 = misc_utils::get_tick_count(); \ + uint64_t ticks2 = epee::misc_utils::get_tick_count(); \ epee::serialization::store_t_to_binary(static_cast(resp), response_info.m_body); \ uint64_t ticks3 = epee::misc_utils::get_tick_count(); \ response_info.m_mime_tipe = " application/octet-stream"; \ diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index a286c472..406d92b2 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -25,6 +25,7 @@ // #pragma once +#include #include #include #include diff --git a/contrib/epee/include/net/net_parse_helpers.h b/contrib/epee/include/net/net_parse_helpers.h index 586dac98..40c3d935 100644 --- a/contrib/epee/include/net/net_parse_helpers.h +++ b/contrib/epee/include/net/net_parse_helpers.h @@ -29,6 +29,7 @@ #pragma once #include "http_base.h" +#include "include_base_utils.h" #include "reg_exp_definer.h" diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index b5619bab..b0e44f5d 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -36,6 +36,11 @@ #define MAKE_IP( a1, a2, a3, a4 ) (a1|(a2<<8)|(a3<<16)|(a4<<24)) #endif +namespace boost { + namespace asio { + class io_service; + } +} namespace epee { diff --git a/contrib/epee/include/profile_tools.h b/contrib/epee/include/profile_tools.h index 0e1646f6..49180c6a 100644 --- a/contrib/epee/include/profile_tools.h +++ b/contrib/epee/include/profile_tools.h @@ -28,6 +28,8 @@ #ifndef _PROFILE_TOOLS_H_ #define _PROFILE_TOOLS_H_ +#include + namespace epee { diff --git a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h index 33486d9e..d53f167f 100644 --- a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h +++ b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h @@ -26,6 +26,12 @@ #pragma once +#include +#include + +#include +#include + namespace epee { namespace serialization diff --git a/contrib/epee/include/static_initializer.h b/contrib/epee/include/static_initializer.h index 3463a560..1510805c 100644 --- a/contrib/epee/include/static_initializer.h +++ b/contrib/epee/include/static_initializer.h @@ -44,38 +44,11 @@ public: initializer() { to_initialize::init(); - //get_set_is_initialized(true, true); } ~initializer() { to_initialize::un_init(); - //get_set_is_uninitialized(true, true); } - - /*static inline bool is_initialized() - { - return get_set_is_initialized(); - } - static inline bool is_uninitialized() - { - return get_set_is_uninitialized(); - } - -private: - static inline bool get_set_is_initialized(bool need_to_set = false, bool val_to_set= false) - { - static bool val_is_initialized = false; - if(need_to_set) - val_is_initialized = val_to_set; - return val_is_initialized; - } - static inline bool get_set_is_uninitialized(bool need_to_set = false, bool val_to_set = false) - { - static bool val_is_uninitialized = false; - if(need_to_set) - val_is_uninitialized = val_to_set; - return val_is_uninitialized; - }*/ }; } diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 1b32c51d..da12c106 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -114,6 +114,7 @@ namespace epee const_cast(out_struct).store(stg);//TODO: add true const support to searilzation std::string buff_to_send, buff_to_recv; stg.store_to_binary(buff_to_send); + int res = transport.invoke_async(command, buff_to_send, conn_id, [cb, command](int code, const std::string& buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); @@ -131,6 +132,7 @@ namespace epee return false; } result_struct.load(stg_ret); + cb(code, result_struct, context); return true; }, inv_timeout); diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index bbfe5f85..87c6edb2 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -150,7 +150,8 @@ namespace epee m_root.m_entries.clear(); if(source.size() < sizeof(storage_block_header)) { - LOG_ERROR("portable_storage: wrong binary format, packet size = " << source.size() << " less than expected sizeof(storage_block_header)=" << sizeof(storage_block_header)); + LOG_WARNING("portable_storage: wrong binary format, packet size = " << source.size() << " less than expected sizeof(storage_block_header)=" + << sizeof(storage_block_header), LOG_LEVEL_2) return false; } storage_block_header* pbuff = (storage_block_header*)source.data(); @@ -158,12 +159,12 @@ namespace epee pbuff->m_signature_b != PORTABLE_STORAGE_SIGNATUREB ) { - LOG_ERROR("portable_storage: wrong binary format - signature missmatch"); + LOG_WARNING("portable_storage: wrong binary format - signature missmatch", LOG_LEVEL_2); return false; } if(pbuff->m_ver != PORTABLE_STORAGE_FORMAT_VER) { - LOG_ERROR("portable_storage: wrong binary format - unknown format ver = " << pbuff->m_ver); + LOG_WARNING("portable_storage: wrong binary format - unknown format ver = " << pbuff->m_ver, LOG_LEVEL_2); return false; } TRY_ENTRY(); diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index 4e74fb7a..3ca61c02 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -25,6 +25,10 @@ // #pragma once + +#include +#include + #include "parserse_base_utils.h" #include "file_io_utils.h" @@ -365,6 +369,7 @@ namespace epee } catch(const std::exception& ex) { + (void)(ex); LOG_PRINT_RED_L0("Failed to parse json, what: " << ex.what()); return false; } diff --git a/contrib/epee/include/storages/portable_storage_to_bin.h b/contrib/epee/include/storages/portable_storage_to_bin.h index baf90290..6743a608 100644 --- a/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/contrib/epee/include/storages/portable_storage_to_bin.h @@ -30,6 +30,7 @@ #include "misc_language.h" #include "portable_storage_base.h" +#include "pragma_comp_defs.h" namespace epee { diff --git a/contrib/epee/include/storages/portable_storage_val_converters.h b/contrib/epee/include/storages/portable_storage_val_converters.h index 6ea50588..73d339f5 100644 --- a/contrib/epee/include/storages/portable_storage_val_converters.h +++ b/contrib/epee/include/storages/portable_storage_val_converters.h @@ -24,10 +24,10 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - #pragma once +#include + #include "misc_language.h" #include "portable_storage_base.h" #include "warnings.h" diff --git a/contrib/epee/include/string_tools.cpp b/contrib/epee/include/string_tools.cpp new file mode 100644 index 00000000..8181bc10 --- /dev/null +++ b/contrib/epee/include/string_tools.cpp @@ -0,0 +1,487 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "string_tools.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + + +#ifdef WINDOWS_PLATFORM +#pragma comment (lib, "Rpcrt4.lib") +#endif + +namespace epee +{ +namespace string_tools +{ + std::wstring get_str_from_guid(const boost::uuids::uuid& rid) + { + return boost::lexical_cast(rid); + } + //---------------------------------------------------------------------------- + std::string get_str_from_guid_a(const boost::uuids::uuid& rid) + { + return boost::lexical_cast(rid); + } + //---------------------------------------------------------------------------- + bool get_guid_from_string( boost::uuids::uuid& inetifer, std::wstring str_id) + { + if(str_id.size() < 36) + return false; + + if('{' == *str_id.begin()) + str_id.erase(0, 1); + + if('}' == *(--str_id.end())) + str_id.erase(--str_id.end()); + + try + { + inetifer = boost::lexical_cast(str_id); + return true; + } + catch(...) + { + return false; + } + } + //---------------------------------------------------------------------------- + bool get_guid_from_string(OUT boost::uuids::uuid& inetifer, const std::string& str_id) + { + std::string local_str_id = str_id; + if(local_str_id.size() < 36) + return false; + + if('{' == *local_str_id.begin()) + local_str_id.erase(0, 1); + + if('}' == *(--local_str_id.end())) + local_str_id.erase(--local_str_id.end()); + + try + { + inetifer = boost::lexical_cast(local_str_id); + return true; + } + catch(...) + { + return false; + } + } + //---------------------------------------------------------------------------- +//#ifdef _WINSOCK2API_ + std::string get_ip_string_from_int32(uint32_t ip) + { + in_addr adr; + adr.s_addr = ip; + const char* pbuf = inet_ntoa(adr); + if(pbuf) + return pbuf; + else + return "[failed]"; + } + //---------------------------------------------------------------------------- + bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str) + { + ip = inet_addr(ip_str.c_str()); + if(INADDR_NONE == ip) + return false; + + return true; + } + //---------------------------------------------------------------------------- + bool parse_peer_from_string(uint32_t& ip, uint32_t& port, const std::string& addres) + { + //parse ip and address + std::string::size_type p = addres.find(':'); + if(p == std::string::npos) + { + return false; + } + std::string ip_str = addres.substr(0, p); + std::string port_str = addres.substr(p+1, addres.size()); + + if(!get_ip_int32_from_string(ip, ip_str)) + { + return false; + } + + if(!get_xtype_from_string(port, port_str)) + { + return false; + } + return true; + } + + //---------------------------------------------------------------------------- + std::string num_to_string_fast(int64_t val) + { + /* + char buff[30] = {0}; + i64toa_s(val, buff, sizeof(buff)-1, 10); + return buff;*/ + return boost::lexical_cast(val); + } + //---------------------------------------------------------------------------- + bool string_to_num_fast(const std::string& buff, int64_t& val) + { + //return get_xtype_from_string(val, buff); +#if (defined _MSC_VER) + val = _atoi64(buff.c_str()); +#else + val = atoll(buff.c_str()); +#endif + /* + * val = atoi64(buff.c_str()); + */ + if(buff != "0" && val == 0) + return false; + return true; + } + //---------------------------------------------------------------------------- + bool string_to_num_fast(const std::string& buff, int& val) + { + val = atoi(buff.c_str()); + if(buff != "0" && val == 0) + return false; + + return true; + } + //---------------------------------------------------------------------------- +#ifdef WINDOWS_PLATFORM + std::string system_time_to_string(const SYSTEMTIME& st) + { + + /* + TIME_ZONE_INFORMATION tzi; + GetTimeZoneInformation(&tzi); + SystemTimeToTzSpecificLocalTime(&tzi, &stUTC, &stLocal); + */ + + char szTime[25], szDate[25]; + ::GetTimeFormatA( + LOCALE_USER_DEFAULT, // locale + TIME_FORCE24HOURFORMAT, // options + &st, // time + NULL, // time format string + szTime, // formatted string buffer + 25 // size of string buffer + ); + + ::GetDateFormatA( + LOCALE_USER_DEFAULT, // locale + NULL, // options + &st, // date + NULL, // date format + szDate, // formatted string buffer + 25 // size of buffer + ); + szTime[24] = szDate[24] = 0; //be happy :) + + std::string res = szDate; + (res += " " )+= szTime; + return res; + + } +#endif + //---------------------------------------------------------------------------- + + bool compare_no_case(const std::string& str1, const std::string& str2) + { + + return !boost::iequals(str1, str2); + } + //---------------------------------------------------------------------------- + bool compare_no_case(const std::wstring& str1, const std::wstring& str2) + { + return !boost::iequals(str1, str2); + } + //---------------------------------------------------------------------------- + bool is_match_prefix(const std::wstring& str1, const std::wstring& prefix) + { + if(prefix.size()>str1.size()) + return false; + + if(!compare_no_case(str1.substr(0, prefix.size()), prefix)) + return true; + else + return false; + } + //---------------------------------------------------------------------------- + bool is_match_prefix(const std::string& str1, const std::string& prefix) + { + if(prefix.size()>str1.size()) + return false; + + if(!compare_no_case(str1.substr(0, prefix.size()), prefix)) + return true; + else + return false; + } + //---------------------------------------------------------------------------- + std::string& get_current_module_name() + { + static std::string module_name; + return module_name; + } + //---------------------------------------------------------------------------- + std::string& get_current_module_folder() + { + static std::string module_folder; + return module_folder; + } + //---------------------------------------------------------------------------- +#ifdef _WIN32 + std::string get_current_module_path() + { + char pname [5000] = {0}; + GetModuleFileNameA( NULL, pname, sizeof(pname)); + pname[sizeof(pname)-1] = 0; //be happy ;) + return pname; + } +#endif + //---------------------------------------------------------------------------- + bool set_module_name_and_folder(const std::string& path_to_process_) + { + std::string path_to_process = path_to_process_; +#ifdef _WIN32 + path_to_process = get_current_module_path(); +#endif + std::string::size_type a = path_to_process.rfind( '\\' ); + if(a == std::string::npos ) + { + a = path_to_process.rfind( '/' ); + } + if ( a != std::string::npos ) + { + get_current_module_name() = path_to_process.substr(a+1, path_to_process.size()); + get_current_module_folder() = path_to_process.substr(0, a); + return true; + }else + return false; + + } + + //---------------------------------------------------------------------------- + bool trim_left(std::string& str) + { + for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast(*it));) + str.erase(str.begin()); + + return true; + } + //---------------------------------------------------------------------------- + bool trim_right(std::string& str) + { + + for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast(*it));) + str.erase( --((it++).base())); + + return true; + } + //---------------------------------------------------------------------------- + std::string& trim(std::string& str) + { + + trim_left(str); + trim_right(str); + return str; + } + //---------------------------------------------------------------------------- + std::string trim(const std::string& str_) + { + std::string str = str_; + trim_left(str); + trim_right(str); + return str; + } + //---------------------------------------------------------------------------- + std::string get_extension(const std::string& str) + { + std::string res; + std::string::size_type pos = str.rfind('.'); + if(std::string::npos == pos) + return res; + + res = str.substr(pos+1, str.size()-pos); + return res; + } + //---------------------------------------------------------------------------- + std::string get_filename_from_path(const std::string& str) + { + std::string res; + std::string::size_type pos = str.rfind('\\'); + if(std::string::npos == pos) + return str; + + res = str.substr(pos+1, str.size()-pos); + return res; + } + //---------------------------------------------------------------------------- + + + + std::string cut_off_extension(const std::string& str) + { + std::string res; + std::string::size_type pos = str.rfind('.'); + if(std::string::npos == pos) + return str; + + res = str.substr(0, pos); + return res; + } + + //---------------------------------------------------------------------------- +#ifdef _WININET_ + std::string get_string_from_systemtime(const SYSTEMTIME& sys_time) + { + std::string result_string; + + char buff[100] = {0}; + BOOL res = ::InternetTimeFromSystemTimeA(&sys_time, INTERNET_RFC1123_FORMAT, buff, 99); + if(!res) + { + LOG_ERROR("Failed to load SytemTime to string"); + } + + result_string = buff; + return result_string; + + } + //------------------------------------------------------------------------------------- + SYSTEMTIME get_systemtime_from_string(const std::string& buff) + { + SYSTEMTIME result_time = {0}; + + BOOL res = ::InternetTimeToSystemTimeA(buff.c_str(), &result_time, NULL); + if(!res) + { + LOG_ERROR("Failed to load SytemTime from string " << buff << "interval set to 15 minutes"); + } + + return result_time; + } +#endif + +#ifdef WINDOWS_PLATFORM + const wchar_t* get_pc_name() + { + static wchar_t info[INFO_BUFFER_SIZE]; + static DWORD bufCharCount = INFO_BUFFER_SIZE; + static bool init = false; + + if (!init) { + if (!GetComputerNameW( info, &bufCharCount )) + info[0] = 0; + else + init = true; + } + + return info; + } + + const wchar_t* get_user_name() + { + static wchar_t info[INFO_BUFFER_SIZE]; + static DWORD bufCharCount = INFO_BUFFER_SIZE; + static bool init = false; + + if (!init) { + if (!GetUserNameW( info, &bufCharCount )) + info[0] = 0; + else + init = true; + } + + return info; + } +#endif + +#ifdef _LM_ + const wchar_t* get_domain_name() + { + static wchar_t info[INFO_BUFFER_SIZE]; + static DWORD bufCharCount = 0; + static bool init = false; + + if (!init) { + LPWSTR domain( NULL ); + NETSETUP_JOIN_STATUS status; + info[0] = 0; + + if (NET_API_STATUS result = NetGetJoinInformation( NULL, &domain, &status )) + { + LOG_ERROR("get_domain_name error: " << log_space::get_win32_err_descr(result)); + } else + { + StringCchCopyW( info, sizeof(info)/sizeof( info[0] ), domain ); + NetApiBufferFree((void*)domain); + init = true; + } + } + + return info; + } +#endif +#ifdef WINDOWS_PLATFORM + inline + std::string load_resource_string_a(int id, const char* pmodule_name = NULL) + { + //slow realization + HMODULE h = ::GetModuleHandleA( pmodule_name ); + + char buff[2000] = {0}; + + ::LoadStringA( h, id, buff, sizeof(buff)); + buff[sizeof(buff)-1] = 0; //be happy :) + return buff; + } + inline + std::wstring load_resource_string_w(int id, const char* pmodule_name = NULL) + { + //slow realization + HMODULE h = ::GetModuleHandleA( pmodule_name ); + + wchar_t buff[2000] = {0}; + + ::LoadStringW( h, id, buff, sizeof(buff) / sizeof( buff[0] ) ); + buff[(sizeof(buff)/sizeof(buff[0]))-1] = 0; //be happy :) + return buff; + } +#endif +} +} diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 8289ee0b..c5d09421 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -24,24 +24,23 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - +#pragma once #ifndef _STRING_TOOLS_H_ #define _STRING_TOOLS_H_ -//#include -#include -#include +#include +#include #include -//#include -#include -#include -#include -#include -#include -#include -#include "warnings.h" +#include +#include +#include +#include +#include +#include + +#include "warnings.h" #ifndef OUT #define OUT @@ -51,71 +50,32 @@ #pragma comment (lib, "Rpcrt4.lib") #endif +// Don't include lexical_cast.hpp, to reduce compilation time +//#include + +namespace boost { + namespace uuids { + struct uuid; + } + + template + inline Target lexical_cast(const Source &arg); +} + namespace epee { namespace string_tools { - inline std::wstring get_str_from_guid(const boost::uuids::uuid& rid) - { - return boost::lexical_cast(rid); - } - //---------------------------------------------------------------------------- - inline std::string get_str_from_guid_a(const boost::uuids::uuid& rid) - { - return boost::lexical_cast(rid); - } - //---------------------------------------------------------------------------- - inline bool get_guid_from_string( boost::uuids::uuid& inetifer, std::wstring str_id) - { - if(str_id.size() < 36) - return false; - - if('{' == *str_id.begin()) - str_id.erase(0, 1); - - if('}' == *(--str_id.end())) - str_id.erase(--str_id.end()); - - try - { - inetifer = boost::lexical_cast(str_id); - return true; - } - catch(...) - { - return false; - } - } - //---------------------------------------------------------------------------- - inline bool get_guid_from_string(OUT boost::uuids::uuid& inetifer, const std::string& str_id) - { - std::string local_str_id = str_id; - if(local_str_id.size() < 36) - return false; - - if('{' == *local_str_id.begin()) - local_str_id.erase(0, 1); - - if('}' == *(--local_str_id.end())) - local_str_id.erase(--local_str_id.end()); - - try - { - inetifer = boost::lexical_cast(local_str_id); - return true; - } - catch(...) - { - return false; - } - } - //---------------------------------------------------------------------------- + std::wstring get_str_from_guid(const boost::uuids::uuid& rid); + std::string get_str_from_guid_a(const boost::uuids::uuid& rid); + bool get_guid_from_string( boost::uuids::uuid& inetifer, std::wstring str_id); + bool get_guid_from_string(OUT boost::uuids::uuid& inetifer, const std::string& str_id); + //---------------------------------------------------------------------------- template std::basic_string buff_to_hex(const std::basic_string& s) { - using namespace std; - basic_stringstream hexStream; - hexStream << hex << noshowbase << setw(2); + std::basic_stringstream hexStream; + hexStream << std::hex << std::noshowbase << std::setw(2); for(typename std::basic_string::const_iterator it = s.begin(); it != s.end(); it++) { @@ -127,13 +87,12 @@ namespace string_tools template std::basic_string buff_to_hex_nodelimer(const std::basic_string& s) { - using namespace std; - basic_stringstream hexStream; - hexStream << hex << noshowbase; + std::basic_stringstream hexStream; + hexStream << std::hex << std::noshowbase; for(typename std::basic_string::const_iterator it = s.begin(); it != s.end(); it++) { - hexStream << setw(2) << setfill('0') << static_cast(static_cast(*it)); + hexStream << std::setw(2) << std::setfill('0') << static_cast(static_cast(*it)); } return hexStream.str(); } @@ -273,13 +232,6 @@ POP_WARNINGS return true; } -/* template - bool get_xparam_from_command_line(const std::map& res, const std::basic_string & key, t_type& val) - { - - } - */ - template bool get_xparam_from_command_line(const std::map& res, const t_string & key, t_type& val) { @@ -295,22 +247,22 @@ POP_WARNINGS return true; } - template - t_type get_xparam_from_command_line(const std::map& res, const t_string & key, const t_type& default_value) - { - typename std::map::const_iterator it = res.find(key); - if(it == res.end()) - return default_value; + template + t_type get_xparam_from_command_line(const std::map& res, const t_string & key, const t_type& default_value) + { + typename std::map::const_iterator it = res.find(key); + if(it == res.end()) + return default_value; - if(it->second.size()) - { - t_type s; - get_xtype_from_string(s, it->second); - return s; - } + if(it->second.size()) + { + t_type s; + get_xtype_from_string(s, it->second); + return s; + } - return default_value; - } + return default_value; + } template bool have_in_command_line(const std::map& res, const std::basic_string& key) @@ -322,243 +274,40 @@ POP_WARNINGS return true; } - //---------------------------------------------------------------------------- -//#ifdef _WINSOCK2API_ - inline std::string get_ip_string_from_int32(uint32_t ip) - { - in_addr adr; - adr.s_addr = ip; - const char* pbuf = inet_ntoa(adr); - if(pbuf) - return pbuf; - else - return "[failed]"; - } - //---------------------------------------------------------------------------- - inline bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str) - { - ip = inet_addr(ip_str.c_str()); - if(INADDR_NONE == ip) - return false; - - return true; - } //---------------------------------------------------------------------------- - inline bool parse_peer_from_string(uint32_t& ip, uint32_t& port, const std::string& addres) - { - //parse ip and address - std::string::size_type p = addres.find(':'); - if(p == std::string::npos) - { - return false; - } - std::string ip_str = addres.substr(0, p); - std::string port_str = addres.substr(p+1, addres.size()); - - if(!get_ip_int32_from_string(ip, ip_str)) - { - return false; - } - - if(!get_xtype_from_string(port, port_str)) - { - return false; - } - return true; - } - -//#endif - //---------------------------------------------------------------------------- + std::string get_ip_string_from_int32(uint32_t ip); + bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str); + bool parse_peer_from_string(uint32_t& ip, uint32_t& port, const std::string& addres); + //---------------------------------------------------------------------------- template - inline std::string get_t_as_hex_nwidth(const t& v, std::streamsize w = 8) + inline std::string get_t_as_hex_nwidth(const t& v, std::streamsize w = 8) { std::stringstream ss; ss << std::setfill ('0') << std::setw (w) << std::hex << std::noshowbase; ss << v; return ss.str(); } - - inline std::string num_to_string_fast(int64_t val) - { - /* - char buff[30] = {0}; - i64toa_s(val, buff, sizeof(buff)-1, 10); - return buff;*/ - return boost::lexical_cast(val); - } - //---------------------------------------------------------------------------- - inline bool string_to_num_fast(const std::string& buff, int64_t& val) - { - //return get_xtype_from_string(val, buff); -#if (defined _MSC_VER) - val = _atoi64(buff.c_str()); -#else - val = atoll(buff.c_str()); -#endif - /* - * val = atoi64(buff.c_str()); - */ - if(buff != "0" && val == 0) - return false; - return true; - } - //---------------------------------------------------------------------------- - inline bool string_to_num_fast(const std::string& buff, int& val) - { - val = atoi(buff.c_str()); - if(buff != "0" && val == 0) - return false; - - return true; - } - //---------------------------------------------------------------------------- + //---------------------------------------------------------------------------- + std::string num_to_string_fast(int64_t val); + bool string_to_num_fast(const std::string& buff, int64_t& val); + bool string_to_num_fast(const std::string& buff, int& val); #ifdef WINDOWS_PLATFORM - inline std::string system_time_to_string(const SYSTEMTIME& st) - { - - /* - TIME_ZONE_INFORMATION tzi; - GetTimeZoneInformation(&tzi); - SystemTimeToTzSpecificLocalTime(&tzi, &stUTC, &stLocal); - */ - - char szTime[25], szDate[25]; - ::GetTimeFormatA( - LOCALE_USER_DEFAULT, // locale - TIME_FORCE24HOURFORMAT, // options - &st, // time - NULL, // time format string - szTime, // formatted string buffer - 25 // size of string buffer - ); - - ::GetDateFormatA( - LOCALE_USER_DEFAULT, // locale - NULL, // options - &st, // date - NULL, // date format - szDate, // formatted string buffer - 25 // size of buffer - ); - szTime[24] = szDate[24] = 0; //be happy :) - - std::string res = szDate; - (res += " " )+= szTime; - return res; - - } + std::string system_time_to_string(const SYSTEMTIME& st); #endif - //---------------------------------------------------------------------------- - - inline bool compare_no_case(const std::string& str1, const std::string& str2) - { - - return !boost::iequals(str1, str2); - } - //---------------------------------------------------------------------------- - inline bool compare_no_case(const std::wstring& str1, const std::wstring& str2) - { - return !boost::iequals(str1, str2); - } - //---------------------------------------------------------------------------- - inline bool is_match_prefix(const std::wstring& str1, const std::wstring& prefix) - { - if(prefix.size()>str1.size()) - return false; - - if(!compare_no_case(str1.substr(0, prefix.size()), prefix)) - return true; - else - return false; - } - //---------------------------------------------------------------------------- - inline bool is_match_prefix(const std::string& str1, const std::string& prefix) - { - if(prefix.size()>str1.size()) - return false; - - if(!compare_no_case(str1.substr(0, prefix.size()), prefix)) - return true; - else - return false; - } - //---------------------------------------------------------------------------- - inline std::string& get_current_module_name() - { - static std::string module_name; - return module_name; - } - //---------------------------------------------------------------------------- - inline std::string& get_current_module_folder() - { - static std::string module_folder; - return module_folder; - } - //---------------------------------------------------------------------------- + bool compare_no_case(const std::string& str1, const std::string& str2); + bool compare_no_case(const std::wstring& str1, const std::wstring& str2); + bool is_match_prefix(const std::wstring& str1, const std::wstring& prefix); + bool is_match_prefix(const std::string& str1, const std::string& prefix); + std::string& get_current_module_name(); + std::string& get_current_module_folder(); #ifdef _WIN32 - inline std::string get_current_module_path() - { - char pname [5000] = {0}; - GetModuleFileNameA( NULL, pname, sizeof(pname)); - pname[sizeof(pname)-1] = 0; //be happy ;) - return pname; - } + std::string get_current_module_path(); #endif - //---------------------------------------------------------------------------- - inline bool set_module_name_and_folder(const std::string& path_to_process_) - { - std::string path_to_process = path_to_process_; -#ifdef _WIN32 - path_to_process = get_current_module_path(); -#endif - std::string::size_type a = path_to_process.rfind( '\\' ); - if(a == std::string::npos ) - { - a = path_to_process.rfind( '/' ); - } - if ( a != std::string::npos ) - { - get_current_module_name() = path_to_process.substr(a+1, path_to_process.size()); - get_current_module_folder() = path_to_process.substr(0, a); - return true; - }else - return false; - - } - - //---------------------------------------------------------------------------- - inline bool trim_left(std::string& str) - { - for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast(*it));) - str.erase(str.begin()); - - return true; - } - //---------------------------------------------------------------------------- - inline bool trim_right(std::string& str) - { - - for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast(*it));) - str.erase( --((it++).base())); - - return true; - } - //---------------------------------------------------------------------------- - inline std::string& trim(std::string& str) - { - - trim_left(str); - trim_right(str); - return str; - } - //---------------------------------------------------------------------------- - inline std::string trim(const std::string& str_) - { - std::string str = str_; - trim_left(str); - trim_right(str); - return str; - } + bool set_module_name_and_folder(const std::string& path_to_process_); + bool trim_left(std::string& str); + bool trim_right(std::string& str); + std::string& trim(std::string& str); + std::string trim(const std::string& str_); //---------------------------------------------------------------------------- template std::string pod_to_hex(const t_pod_type& s) @@ -584,161 +333,27 @@ POP_WARNINGS return true; } //---------------------------------------------------------------------------- - inline std::string get_extension(const std::string& str) - { - std::string res; - std::string::size_type pos = str.rfind('.'); - if(std::string::npos == pos) - return res; - - res = str.substr(pos+1, str.size()-pos); - return res; - } - //---------------------------------------------------------------------------- - inline std::string get_filename_from_path(const std::string& str) - { - std::string res; - std::string::size_type pos = str.rfind('\\'); - if(std::string::npos == pos) - return str; - - res = str.substr(pos+1, str.size()-pos); - return res; - } - //---------------------------------------------------------------------------- - - - - inline std::string cut_off_extension(const std::string& str) - { - std::string res; - std::string::size_type pos = str.rfind('.'); - if(std::string::npos == pos) - return str; - - res = str.substr(0, pos); - return res; - } - - //---------------------------------------------------------------------------- + std::string get_extension(const std::string& str); + std::string get_filename_from_path(const std::string& str); + std::string cut_off_extension(const std::string& str); #ifdef _WININET_ - inline std::string get_string_from_systemtime(const SYSTEMTIME& sys_time) - { - std::string result_string; - - char buff[100] = {0}; - BOOL res = ::InternetTimeFromSystemTimeA(&sys_time, INTERNET_RFC1123_FORMAT, buff, 99); - if(!res) - { - LOG_ERROR("Failed to load SytemTime to string"); - } - - result_string = buff; - return result_string; - - } - //------------------------------------------------------------------------------------- - inline SYSTEMTIME get_systemtime_from_string(const std::string& buff) - { - SYSTEMTIME result_time = {0}; - - BOOL res = ::InternetTimeToSystemTimeA(buff.c_str(), &result_time, NULL); - if(!res) - { - LOG_ERROR("Failed to load SytemTime from string " << buff << "interval set to 15 minutes"); - } - - return result_time; - } + std::string get_string_from_systemtime(const SYSTEMTIME& sys_time); + SYSTEMTIME get_systemtime_from_string(const std::string& buff); #endif #ifdef WINDOWS_PLATFORM - static const DWORD INFO_BUFFER_SIZE = 10000; + const DWORD INFO_BUFFER_SIZE = 10000; - static const wchar_t* get_pc_name() - { - static wchar_t info[INFO_BUFFER_SIZE]; - static DWORD bufCharCount = INFO_BUFFER_SIZE; - static bool init = false; - - if (!init) { - if (!GetComputerNameW( info, &bufCharCount )) - info[0] = 0; - else - init = true; - } - - return info; - } - - static const wchar_t* get_user_name() - { - static wchar_t info[INFO_BUFFER_SIZE]; - static DWORD bufCharCount = INFO_BUFFER_SIZE; - static bool init = false; - - if (!init) { - if (!GetUserNameW( info, &bufCharCount )) - info[0] = 0; - else - init = true; - } - - return info; - } + const wchar_t* get_pc_name(); + const wchar_t* get_user_name(); #endif #ifdef _LM_ - static const wchar_t* get_domain_name() - { - static wchar_t info[INFO_BUFFER_SIZE]; - static DWORD bufCharCount = 0; - static bool init = false; - - if (!init) { - LPWSTR domain( NULL ); - NETSETUP_JOIN_STATUS status; - info[0] = 0; - - if (NET_API_STATUS result = NetGetJoinInformation( NULL, &domain, &status )) - { - LOG_ERROR("get_domain_name error: " << log_space::get_win32_err_descr(result)); - } else - { - StringCchCopyW( info, sizeof(info)/sizeof( info[0] ), domain ); - NetApiBufferFree((void*)domain); - init = true; - } - } - - return info; - } + const wchar_t* get_domain_name(); #endif #ifdef WINDOWS_PLATFORM - inline - std::string load_resource_string_a(int id, const char* pmodule_name = NULL) - { - //slow realization - HMODULE h = ::GetModuleHandleA( pmodule_name ); - - char buff[2000] = {0}; - - ::LoadStringA( h, id, buff, sizeof(buff)); - buff[sizeof(buff)-1] = 0; //be happy :) - return buff; - } - inline - std::wstring load_resource_string_w(int id, const char* pmodule_name = NULL) - { - //slow realization - HMODULE h = ::GetModuleHandleA( pmodule_name ); - - wchar_t buff[2000] = {0}; - - ::LoadStringW( h, id, buff, sizeof(buff) / sizeof( buff[0] ) ); - buff[(sizeof(buff)/sizeof(buff[0]))-1] = 0; //be happy :) - return buff; - } + std::string load_resource_string_a(int id, const char* pmodule_name = NULL); + std::wstring load_resource_string_w(int id, const char* pmodule_name = NULL); #endif } } diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index b7273da8..52cb70e6 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -32,13 +32,9 @@ #include #include -#include -#include -#include namespace epee { - struct simple_event { simple_event() : m_rised(false) @@ -70,7 +66,7 @@ namespace epee class critical_section { - boost::recursive_mutex m_section; + std::recursive_mutex m_section; public: //to make copy fake! @@ -89,7 +85,6 @@ namespace epee void lock() { m_section.lock(); - //EnterCriticalSection( &m_section ); } void unlock() diff --git a/external/google/dense_hash_map b/external/google/dense_hash_map new file mode 100644 index 00000000..74c796ba --- /dev/null +++ b/external/google/dense_hash_map @@ -0,0 +1,333 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ---- +// Author: Craig Silverstein +// +// This is just a very thin wrapper over densehashtable.h, just +// like sgi stl's stl_hash_map is a very thin wrapper over +// stl_hashtable. The major thing we define is operator[], because +// we have a concept of a data_type which stl_hashtable doesn't +// (it only has a key and a value). +// +// NOTE: this is exactly like sparse_hash_map.h, with the word +// "sparse" replaced by "dense", except for the addition of +// set_empty_key(). +// +// YOU MUST CALL SET_EMPTY_KEY() IMMEDIATELY AFTER CONSTRUCTION. +// +// Otherwise your program will die in mysterious ways. (Note if you +// use the constructor that takes an InputIterator range, you pass in +// the empty key in the constructor, rather than after. As a result, +// this constructor differs from the standard STL version.) +// +// In other respects, we adhere mostly to the STL semantics for +// hash-map. One important exception is that insert() may invalidate +// iterators entirely -- STL semantics are that insert() may reorder +// iterators, but they all still refer to something valid in the +// hashtable. Not so for us. Likewise, insert() may invalidate +// pointers into the hashtable. (Whether insert invalidates iterators +// and pointers depends on whether it results in a hashtable resize). +// On the plus side, delete() doesn't invalidate iterators or pointers +// at all, or even change the ordering of elements. +// +// Here are a few "power user" tips: +// +// 1) set_deleted_key(): +// If you want to use erase() you *must* call set_deleted_key(), +// in addition to set_empty_key(), after construction. +// The deleted and empty keys must differ. +// +// 2) resize(0): +// When an item is deleted, its memory isn't freed right +// away. This allows you to iterate over a hashtable, +// and call erase(), without invalidating the iterator. +// To force the memory to be freed, call resize(0). +// For tr1 compatibility, this can also be called as rehash(0). +// +// 3) min_load_factor(0.0) +// Setting the minimum load factor to 0.0 guarantees that +// the hash table will never shrink. +// +// Roughly speaking: +// (1) dense_hash_map: fastest, uses the most memory unless entries are small +// (2) sparse_hash_map: slowest, uses the least memory +// (3) hash_map / unordered_map (STL): in the middle +// +// Typically I use sparse_hash_map when I care about space and/or when +// I need to save the hashtable on disk. I use hash_map otherwise. I +// don't personally use dense_hash_set ever; some people use it for +// small sets with lots of lookups. +// +// - dense_hash_map has, typically, about 78% memory overhead (if your +// data takes up X bytes, the hash_map uses .78X more bytes in overhead). +// - sparse_hash_map has about 4 bits overhead per entry. +// - sparse_hash_map can be 3-7 times slower than the others for lookup and, +// especially, inserts. See time_hash_map.cc for details. +// +// See /usr/(local/)?doc/sparsehash-*/dense_hash_map.html +// for information about how to use this class. + +#ifndef _DENSE_HASH_MAP_H_ +#define _DENSE_HASH_MAP_H_ + +#include +#include // for FILE * in read()/write() +#include // for the default template args +#include // for equal_to +#include // for alloc<> +#include // for pair<> +#include HASH_FUN_H // defined in config.h +#include +#include + + +_START_GOOGLE_NAMESPACE_ + +using STL_NAMESPACE::pair; + +template , // defined in sparseconfig.h + class EqualKey = STL_NAMESPACE::equal_to, + class Alloc = libc_allocator_with_realloc > > +class dense_hash_map { + private: + // Apparently select1st is not stl-standard, so we define our own + struct SelectKey { + typedef const Key& result_type; + const Key& operator()(const pair& p) const { + return p.first; + } + }; + struct SetKey { + void operator()(pair* value, const Key& new_key) const { + *const_cast(&value->first) = new_key; + // It would be nice to clear the rest of value here as well, in + // case it's taking up a lot of memory. We do this by clearing + // the value. This assumes T has a zero-arg constructor! + value->second = T(); + } + }; + // For operator[]. + struct DefaultValue { + STL_NAMESPACE::pair operator()(const Key& key) { + return STL_NAMESPACE::make_pair(key, T()); + } + }; + + // The actual data + typedef dense_hashtable, Key, HashFcn, SelectKey, + SetKey, EqualKey, Alloc> ht; + ht rep; + + public: + typedef typename ht::key_type key_type; + typedef T data_type; + typedef T mapped_type; + typedef typename ht::value_type value_type; + typedef typename ht::hasher hasher; + typedef typename ht::key_equal key_equal; + typedef Alloc allocator_type; + + typedef typename ht::size_type size_type; + typedef typename ht::difference_type difference_type; + typedef typename ht::pointer pointer; + typedef typename ht::const_pointer const_pointer; + typedef typename ht::reference reference; + typedef typename ht::const_reference const_reference; + + typedef typename ht::iterator iterator; + typedef typename ht::const_iterator const_iterator; + typedef typename ht::local_iterator local_iterator; + typedef typename ht::const_local_iterator const_local_iterator; + + // Iterator functions + iterator begin() { return rep.begin(); } + iterator end() { return rep.end(); } + const_iterator begin() const { return rep.begin(); } + const_iterator end() const { return rep.end(); } + + + // These come from tr1's unordered_map. For us, a bucket has 0 or 1 elements. + local_iterator begin(size_type i) { return rep.begin(i); } + local_iterator end(size_type i) { return rep.end(i); } + const_local_iterator begin(size_type i) const { return rep.begin(i); } + const_local_iterator end(size_type i) const { return rep.end(i); } + + // Accessor functions + allocator_type get_allocator() const { return rep.get_allocator(); } + hasher hash_funct() const { return rep.hash_funct(); } + hasher hash_function() const { return hash_funct(); } + key_equal key_eq() const { return rep.key_eq(); } + + + // Constructors + explicit dense_hash_map(size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { + } + + template + dense_hash_map(InputIterator f, InputIterator l, + const key_type& empty_key_val, + size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { + set_empty_key(empty_key_val); + rep.insert(f, l); + } + // We use the default copy constructor + // We use the default operator=() + // We use the default destructor + + void clear() { rep.clear(); } + // This clears the hash map without resizing it down to the minimum + // bucket count, but rather keeps the number of buckets constant + void clear_no_resize() { rep.clear_no_resize(); } + void swap(dense_hash_map& hs) { rep.swap(hs.rep); } + + + // Functions concerning size + size_type size() const { return rep.size(); } + size_type max_size() const { return rep.max_size(); } + bool empty() const { return rep.empty(); } + size_type bucket_count() const { return rep.bucket_count(); } + size_type max_bucket_count() const { return rep.max_bucket_count(); } + + // These are tr1 methods. bucket() is the bucket the key is or would be in. + size_type bucket_size(size_type i) const { return rep.bucket_size(i); } + size_type bucket(const key_type& key) const { return rep.bucket(key); } + float load_factor() const { + return size() * 1.0f / bucket_count(); + } + float max_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return grow; + } + void max_load_factor(float new_grow) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(shrink, new_grow); + } + // These aren't tr1 methods but perhaps ought to be. + float min_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return shrink; + } + void min_load_factor(float new_shrink) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(new_shrink, grow); + } + // Deprecated; use min_load_factor() or max_load_factor() instead. + void set_resizing_parameters(float shrink, float grow) { + rep.set_resizing_parameters(shrink, grow); + } + + void resize(size_type hint) { rep.resize(hint); } + void rehash(size_type hint) { resize(hint); } // the tr1 name + + // Lookup routines + iterator find(const key_type& key) { return rep.find(key); } + const_iterator find(const key_type& key) const { return rep.find(key); } + + data_type& operator[](const key_type& key) { // This is our value-add! + // If key is in the hashtable, returns find(key)->second, + // otherwise returns insert(value_type(key, T()).first->second. + // Note it does not create an empty T unless the find fails. + return rep.template find_or_insert(key).second; + } + + size_type count(const key_type& key) const { return rep.count(key); } + + pair equal_range(const key_type& key) { + return rep.equal_range(key); + } + pair equal_range(const key_type& key) const { + return rep.equal_range(key); + } + + // Insertion routines + pair insert(const value_type& obj) { return rep.insert(obj); } + template + void insert(InputIterator f, InputIterator l) { rep.insert(f, l); } + void insert(const_iterator f, const_iterator l) { rep.insert(f, l); } + // required for std::insert_iterator; the passed-in iterator is ignored + iterator insert(iterator, const value_type& obj) { return insert(obj).first; } + + + // Deletion and empty routines + // THESE ARE NON-STANDARD! I make you specify an "impossible" key + // value to identify deleted and empty buckets. You can change the + // deleted key as time goes on, or get rid of it entirely to be insert-only. + void set_empty_key(const key_type& key) { // YOU MUST CALL THIS! + rep.set_empty_key(value_type(key, data_type())); // rep wants a value + } + key_type empty_key() const { + return rep.empty_key().first; // rep returns a value + } + + void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); } + void clear_deleted_key() { rep.clear_deleted_key(); } + key_type deleted_key() const { return rep.deleted_key(); } + + // These are standard + size_type erase(const key_type& key) { return rep.erase(key); } + void erase(iterator it) { rep.erase(it); } + void erase(iterator f, iterator l) { rep.erase(f, l); } + + + // Comparison + bool operator==(const dense_hash_map& hs) const { return rep == hs.rep; } + bool operator!=(const dense_hash_map& hs) const { return rep != hs.rep; } + + + // I/O -- this is an add-on for writing metainformation to disk + bool write_metadata(FILE *fp) { return rep.write_metadata(fp); } + bool read_metadata(FILE *fp) { return rep.read_metadata(fp); } + bool write_nopointer_data(FILE *fp) { return rep.write_nopointer_data(fp); } + bool read_nopointer_data(FILE *fp) { return rep.read_nopointer_data(fp); } +}; + +// We need a global swap as well +template +inline void swap(dense_hash_map& hm1, + dense_hash_map& hm2) { + hm1.swap(hm2); +} + +_END_GOOGLE_NAMESPACE_ + +#endif /* _DENSE_HASH_MAP_H_ */ diff --git a/external/google/dense_hash_set b/external/google/dense_hash_set new file mode 100644 index 00000000..fcf5db84 --- /dev/null +++ b/external/google/dense_hash_set @@ -0,0 +1,308 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein +// +// This is just a very thin wrapper over densehashtable.h, just +// like sgi stl's stl_hash_set is a very thin wrapper over +// stl_hashtable. The major thing we define is operator[], because +// we have a concept of a data_type which stl_hashtable doesn't +// (it only has a key and a value). +// +// This is more different from dense_hash_map than you might think, +// because all iterators for sets are const (you obviously can't +// change the key, and for sets there is no value). +// +// NOTE: this is exactly like sparse_hash_set.h, with the word +// "sparse" replaced by "dense", except for the addition of +// set_empty_key(). +// +// YOU MUST CALL SET_EMPTY_KEY() IMMEDIATELY AFTER CONSTRUCTION. +// +// Otherwise your program will die in mysterious ways. (Note if you +// use the constructor that takes an InputIterator range, you pass in +// the empty key in the constructor, rather than after. As a result, +// this constructor differs from the standard STL version.) +// +// In other respects, we adhere mostly to the STL semantics for +// hash-map. One important exception is that insert() may invalidate +// iterators entirely -- STL semantics are that insert() may reorder +// iterators, but they all still refer to something valid in the +// hashtable. Not so for us. Likewise, insert() may invalidate +// pointers into the hashtable. (Whether insert invalidates iterators +// and pointers depends on whether it results in a hashtable resize). +// On the plus side, delete() doesn't invalidate iterators or pointers +// at all, or even change the ordering of elements. +// +// Here are a few "power user" tips: +// +// 1) set_deleted_key(): +// If you want to use erase() you must call set_deleted_key(), +// in addition to set_empty_key(), after construction. +// The deleted and empty keys must differ. +// +// 2) resize(0): +// When an item is deleted, its memory isn't freed right +// away. This allows you to iterate over a hashtable, +// and call erase(), without invalidating the iterator. +// To force the memory to be freed, call resize(0). +// For tr1 compatibility, this can also be called as rehash(0). +// +// 3) min_load_factor(0.0) +// Setting the minimum load factor to 0.0 guarantees that +// the hash table will never shrink. +// +// Roughly speaking: +// (1) dense_hash_set: fastest, uses the most memory unless entries are small +// (2) sparse_hash_set: slowest, uses the least memory +// (3) hash_set / unordered_set (STL): in the middle +// +// Typically I use sparse_hash_set when I care about space and/or when +// I need to save the hashtable on disk. I use hash_set otherwise. I +// don't personally use dense_hash_set ever; some people use it for +// small sets with lots of lookups. +// +// - dense_hash_set has, typically, about 78% memory overhead (if your +// data takes up X bytes, the hash_set uses .78X more bytes in overhead). +// - sparse_hash_set has about 4 bits overhead per entry. +// - sparse_hash_set can be 3-7 times slower than the others for lookup and, +// especially, inserts. See time_hash_map.cc for details. +// +// See /usr/(local/)?doc/sparsehash-*/dense_hash_set.html +// for information about how to use this class. + +#ifndef _DENSE_HASH_SET_H_ +#define _DENSE_HASH_SET_H_ + +#include +#include // for FILE * in read()/write() +#include // for the default template args +#include // for equal_to +#include // for alloc<> +#include // for pair<> +#include HASH_FUN_H // defined in config.h +#include +#include + + +_START_GOOGLE_NAMESPACE_ + +using STL_NAMESPACE::pair; + +template , // defined in sparseconfig.h + class EqualKey = STL_NAMESPACE::equal_to, + class Alloc = libc_allocator_with_realloc > +class dense_hash_set { + private: + // Apparently identity is not stl-standard, so we define our own + struct Identity { + typedef const Value& result_type; + const Value& operator()(const Value& v) const { return v; } + }; + struct SetKey { + void operator()(Value* value, const Value& new_key) const { + *value = new_key; + } + }; + + // The actual data + typedef dense_hashtable ht; + ht rep; + + public: + typedef typename ht::key_type key_type; + typedef typename ht::value_type value_type; + typedef typename ht::hasher hasher; + typedef typename ht::key_equal key_equal; + typedef Alloc allocator_type; + + typedef typename ht::size_type size_type; + typedef typename ht::difference_type difference_type; + typedef typename ht::const_pointer pointer; + typedef typename ht::const_pointer const_pointer; + typedef typename ht::const_reference reference; + typedef typename ht::const_reference const_reference; + + typedef typename ht::const_iterator iterator; + typedef typename ht::const_iterator const_iterator; + typedef typename ht::const_local_iterator local_iterator; + typedef typename ht::const_local_iterator const_local_iterator; + + + // Iterator functions -- recall all iterators are const + iterator begin() const { return rep.begin(); } + iterator end() const { return rep.end(); } + + // These come from tr1's unordered_set. For us, a bucket has 0 or 1 elements. + local_iterator begin(size_type i) const { return rep.begin(i); } + local_iterator end(size_type i) const { return rep.end(i); } + + + // Accessor functions + allocator_type get_allocator() const { return rep.get_allocator(); } + hasher hash_funct() const { return rep.hash_funct(); } + hasher hash_function() const { return hash_funct(); } // tr1 name + key_equal key_eq() const { return rep.key_eq(); } + + + // Constructors + explicit dense_hash_set(size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { + } + + template + dense_hash_set(InputIterator f, InputIterator l, + const key_type& empty_key_val, + size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { + set_empty_key(empty_key_val); + rep.insert(f, l); + } + // We use the default copy constructor + // We use the default operator=() + // We use the default destructor + + void clear() { rep.clear(); } + // This clears the hash set without resizing it down to the minimum + // bucket count, but rather keeps the number of buckets constant + void clear_no_resize() { rep.clear_no_resize(); } + void swap(dense_hash_set& hs) { rep.swap(hs.rep); } + + + // Functions concerning size + size_type size() const { return rep.size(); } + size_type max_size() const { return rep.max_size(); } + bool empty() const { return rep.empty(); } + size_type bucket_count() const { return rep.bucket_count(); } + size_type max_bucket_count() const { return rep.max_bucket_count(); } + + // These are tr1 methods. bucket() is the bucket the key is or would be in. + size_type bucket_size(size_type i) const { return rep.bucket_size(i); } + size_type bucket(const key_type& key) const { return rep.bucket(key); } + float load_factor() const { + return size() * 1.0f / bucket_count(); + } + float max_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return grow; + } + void max_load_factor(float new_grow) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(shrink, new_grow); + } + // These aren't tr1 methods but perhaps ought to be. + float min_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return shrink; + } + void min_load_factor(float new_shrink) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(new_shrink, grow); + } + // Deprecated; use min_load_factor() or max_load_factor() instead. + void set_resizing_parameters(float shrink, float grow) { + rep.set_resizing_parameters(shrink, grow); + } + + void resize(size_type hint) { rep.resize(hint); } + void rehash(size_type hint) { resize(hint); } // the tr1 name + + // Lookup routines + iterator find(const key_type& key) const { return rep.find(key); } + + size_type count(const key_type& key) const { return rep.count(key); } + + pair equal_range(const key_type& key) const { + return rep.equal_range(key); + } + + + // Insertion routines + pair insert(const value_type& obj) { + pair p = rep.insert(obj); + return pair(p.first, p.second); // const to non-const + } + template + void insert(InputIterator f, InputIterator l) { rep.insert(f, l); } + void insert(const_iterator f, const_iterator l) { rep.insert(f, l); } + // required for std::insert_iterator; the passed-in iterator is ignored + iterator insert(iterator, const value_type& obj) { return insert(obj).first; } + + + // Deletion and empty routines + // THESE ARE NON-STANDARD! I make you specify an "impossible" key + // value to identify deleted and empty buckets. You can change the + // deleted key as time goes on, or get rid of it entirely to be insert-only. + void set_empty_key(const key_type& key) { rep.set_empty_key(key); } + key_type empty_key() const { return rep.empty_key(); } + + void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); } + void clear_deleted_key() { rep.clear_deleted_key(); } + key_type deleted_key() const { return rep.deleted_key(); } + + // These are standard + size_type erase(const key_type& key) { return rep.erase(key); } + void erase(iterator it) { rep.erase(it); } + void erase(iterator f, iterator l) { rep.erase(f, l); } + + + // Comparison + bool operator==(const dense_hash_set& hs) const { return rep == hs.rep; } + bool operator!=(const dense_hash_set& hs) const { return rep != hs.rep; } + + + // I/O -- this is an add-on for writing metainformation to disk + bool write_metadata(FILE *fp) { return rep.write_metadata(fp); } + bool read_metadata(FILE *fp) { return rep.read_metadata(fp); } + bool write_nopointer_data(FILE *fp) { return rep.write_nopointer_data(fp); } + bool read_nopointer_data(FILE *fp) { return rep.read_nopointer_data(fp); } +}; + +template +inline void swap(dense_hash_set& hs1, + dense_hash_set& hs2) { + hs1.swap(hs2); +} + +_END_GOOGLE_NAMESPACE_ + +#endif /* _DENSE_HASH_SET_H_ */ diff --git a/external/google/sparse_hash_map b/external/google/sparse_hash_map new file mode 100644 index 00000000..07e52d8b --- /dev/null +++ b/external/google/sparse_hash_map @@ -0,0 +1,310 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein +// +// This is just a very thin wrapper over sparsehashtable.h, just +// like sgi stl's stl_hash_map is a very thin wrapper over +// stl_hashtable. The major thing we define is operator[], because +// we have a concept of a data_type which stl_hashtable doesn't +// (it only has a key and a value). +// +// We adhere mostly to the STL semantics for hash-map. One important +// exception is that insert() may invalidate iterators entirely -- STL +// semantics are that insert() may reorder iterators, but they all +// still refer to something valid in the hashtable. Not so for us. +// Likewise, insert() may invalidate pointers into the hashtable. +// (Whether insert invalidates iterators and pointers depends on +// whether it results in a hashtable resize). On the plus side, +// delete() doesn't invalidate iterators or pointers at all, or even +// change the ordering of elements. +// +// Here are a few "power user" tips: +// +// 1) set_deleted_key(): +// Unlike STL's hash_map, if you want to use erase() you +// *must* call set_deleted_key() after construction. +// +// 2) resize(0): +// When an item is deleted, its memory isn't freed right +// away. This is what allows you to iterate over a hashtable +// and call erase() without invalidating the iterator. +// To force the memory to be freed, call resize(0). +// For tr1 compatibility, this can also be called as rehash(0). +// +// 3) min_load_factor(0.0) +// Setting the minimum load factor to 0.0 guarantees that +// the hash table will never shrink. +// +// Roughly speaking: +// (1) dense_hash_map: fastest, uses the most memory unless entries are small +// (2) sparse_hash_map: slowest, uses the least memory +// (3) hash_map / unordered_map (STL): in the middle +// +// Typically I use sparse_hash_map when I care about space and/or when +// I need to save the hashtable on disk. I use hash_map otherwise. I +// don't personally use dense_hash_map ever; some people use it for +// small maps with lots of lookups. +// +// - dense_hash_map has, typically, about 78% memory overhead (if your +// data takes up X bytes, the hash_map uses .78X more bytes in overhead). +// - sparse_hash_map has about 4 bits overhead per entry. +// - sparse_hash_map can be 3-7 times slower than the others for lookup and, +// especially, inserts. See time_hash_map.cc for details. +// +// See /usr/(local/)?doc/sparsehash-*/sparse_hash_map.html +// for information about how to use this class. + +#ifndef _SPARSE_HASH_MAP_H_ +#define _SPARSE_HASH_MAP_H_ + +#include +#include // for FILE * in read()/write() +#include // for the default template args +#include // for equal_to +#include // for alloc<> +#include // for pair<> +#include HASH_FUN_H // defined in config.h +#include +#include + + +_START_GOOGLE_NAMESPACE_ + +using STL_NAMESPACE::pair; + +template , // defined in sparseconfig.h + class EqualKey = STL_NAMESPACE::equal_to, + class Alloc = libc_allocator_with_realloc > > +class sparse_hash_map { + private: + // Apparently select1st is not stl-standard, so we define our own + struct SelectKey { + typedef const Key& result_type; + const Key& operator()(const pair& p) const { + return p.first; + } + }; + struct SetKey { + void operator()(pair* value, const Key& new_key) const { + *const_cast(&value->first) = new_key; + // It would be nice to clear the rest of value here as well, in + // case it's taking up a lot of memory. We do this by clearing + // the value. This assumes T has a zero-arg constructor! + value->second = T(); + } + }; + // For operator[]. + struct DefaultValue { + STL_NAMESPACE::pair operator()(const Key& key) { + return STL_NAMESPACE::make_pair(key, T()); + } + }; + + // The actual data + typedef sparse_hashtable, Key, HashFcn, SelectKey, + SetKey, EqualKey, Alloc> ht; + ht rep; + + public: + typedef typename ht::key_type key_type; + typedef T data_type; + typedef T mapped_type; + typedef typename ht::value_type value_type; + typedef typename ht::hasher hasher; + typedef typename ht::key_equal key_equal; + typedef Alloc allocator_type; + + typedef typename ht::size_type size_type; + typedef typename ht::difference_type difference_type; + typedef typename ht::pointer pointer; + typedef typename ht::const_pointer const_pointer; + typedef typename ht::reference reference; + typedef typename ht::const_reference const_reference; + + typedef typename ht::iterator iterator; + typedef typename ht::const_iterator const_iterator; + typedef typename ht::local_iterator local_iterator; + typedef typename ht::const_local_iterator const_local_iterator; + + // Iterator functions + iterator begin() { return rep.begin(); } + iterator end() { return rep.end(); } + const_iterator begin() const { return rep.begin(); } + const_iterator end() const { return rep.end(); } + + // These come from tr1's unordered_map. For us, a bucket has 0 or 1 elements. + local_iterator begin(size_type i) { return rep.begin(i); } + local_iterator end(size_type i) { return rep.end(i); } + const_local_iterator begin(size_type i) const { return rep.begin(i); } + const_local_iterator end(size_type i) const { return rep.end(i); } + + // Accessor functions + allocator_type get_allocator() const { return rep.get_allocator(); } + hasher hash_funct() const { return rep.hash_funct(); } + hasher hash_function() const { return hash_funct(); } + key_equal key_eq() const { return rep.key_eq(); } + + + // Constructors + explicit sparse_hash_map(size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { + } + + template + sparse_hash_map(InputIterator f, InputIterator l, + size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { + rep.insert(f, l); + } + // We use the default copy constructor + // We use the default operator=() + // We use the default destructor + + void clear() { rep.clear(); } + void swap(sparse_hash_map& hs) { rep.swap(hs.rep); } + + + // Functions concerning size + size_type size() const { return rep.size(); } + size_type max_size() const { return rep.max_size(); } + bool empty() const { return rep.empty(); } + size_type bucket_count() const { return rep.bucket_count(); } + size_type max_bucket_count() const { return rep.max_bucket_count(); } + + // These are tr1 methods. bucket() is the bucket the key is or would be in. + size_type bucket_size(size_type i) const { return rep.bucket_size(i); } + size_type bucket(const key_type& key) const { return rep.bucket(key); } + float load_factor() const { + return size() * 1.0f / bucket_count(); + } + float max_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return grow; + } + void max_load_factor(float new_grow) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(shrink, new_grow); + } + // These aren't tr1 methods but perhaps ought to be. + float min_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return shrink; + } + void min_load_factor(float new_shrink) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(new_shrink, grow); + } + // Deprecated; use min_load_factor() or max_load_factor() instead. + void set_resizing_parameters(float shrink, float grow) { + rep.set_resizing_parameters(shrink, grow); + } + + void resize(size_type hint) { rep.resize(hint); } + void rehash(size_type hint) { resize(hint); } // the tr1 name + + // Lookup routines + iterator find(const key_type& key) { return rep.find(key); } + const_iterator find(const key_type& key) const { return rep.find(key); } + + data_type& operator[](const key_type& key) { // This is our value-add! + // If key is in the hashtable, returns find(key)->second, + // otherwise returns insert(value_type(key, T()).first->second. + // Note it does not create an empty T unless the find fails. + return rep.template find_or_insert(key).second; + } + + size_type count(const key_type& key) const { return rep.count(key); } + + pair equal_range(const key_type& key) { + return rep.equal_range(key); + } + pair equal_range(const key_type& key) const { + return rep.equal_range(key); + } + + // Insertion routines + pair insert(const value_type& obj) { return rep.insert(obj); } + template + void insert(InputIterator f, InputIterator l) { rep.insert(f, l); } + void insert(const_iterator f, const_iterator l) { rep.insert(f, l); } + // required for std::insert_iterator; the passed-in iterator is ignored + iterator insert(iterator, const value_type& obj) { return insert(obj).first; } + + + // Deletion routines + // THESE ARE NON-STANDARD! I make you specify an "impossible" key + // value to identify deleted buckets. You can change the key as + // time goes on, or get rid of it entirely to be insert-only. + void set_deleted_key(const key_type& key) { + rep.set_deleted_key(key); + } + void clear_deleted_key() { rep.clear_deleted_key(); } + key_type deleted_key() const { return rep.deleted_key(); } + + // These are standard + size_type erase(const key_type& key) { return rep.erase(key); } + void erase(iterator it) { rep.erase(it); } + void erase(iterator f, iterator l) { rep.erase(f, l); } + + + // Comparison + bool operator==(const sparse_hash_map& hs) const { return rep == hs.rep; } + bool operator!=(const sparse_hash_map& hs) const { return rep != hs.rep; } + + + // I/O -- this is an add-on for writing metainformation to disk + bool write_metadata(FILE *fp) { return rep.write_metadata(fp); } + bool read_metadata(FILE *fp) { return rep.read_metadata(fp); } + bool write_nopointer_data(FILE *fp) { return rep.write_nopointer_data(fp); } + bool read_nopointer_data(FILE *fp) { return rep.read_nopointer_data(fp); } +}; + +// We need a global swap as well +template +inline void swap(sparse_hash_map& hm1, + sparse_hash_map& hm2) { + hm1.swap(hm2); +} + +_END_GOOGLE_NAMESPACE_ + +#endif /* _SPARSE_HASH_MAP_H_ */ diff --git a/external/google/sparse_hash_set b/external/google/sparse_hash_set new file mode 100644 index 00000000..6be16e9a --- /dev/null +++ b/external/google/sparse_hash_set @@ -0,0 +1,285 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein +// +// This is just a very thin wrapper over sparsehashtable.h, just +// like sgi stl's stl_hash_set is a very thin wrapper over +// stl_hashtable. The major thing we define is operator[], because +// we have a concept of a data_type which stl_hashtable doesn't +// (it only has a key and a value). +// +// This is more different from sparse_hash_map than you might think, +// because all iterators for sets are const (you obviously can't +// change the key, and for sets there is no value). +// +// We adhere mostly to the STL semantics for hash-map. One important +// exception is that insert() may invalidate iterators entirely -- STL +// semantics are that insert() may reorder iterators, but they all +// still refer to something valid in the hashtable. Not so for us. +// Likewise, insert() may invalidate pointers into the hashtable. +// (Whether insert invalidates iterators and pointers depends on +// whether it results in a hashtable resize). On the plus side, +// delete() doesn't invalidate iterators or pointers at all, or even +// change the ordering of elements. +// +// Here are a few "power user" tips: +// +// 1) set_deleted_key(): +// Unlike STL's hash_map, if you want to use erase() you +// *must* call set_deleted_key() after construction. +// +// 2) resize(0): +// When an item is deleted, its memory isn't freed right +// away. This allows you to iterate over a hashtable, +// and call erase(), without invalidating the iterator. +// To force the memory to be freed, call resize(0). +// For tr1 compatibility, this can also be called as rehash(0). +// +// 3) min_load_factor(0.0) +// Setting the minimum load factor to 0.0 guarantees that +// the hash table will never shrink. +// +// Roughly speaking: +// (1) dense_hash_set: fastest, uses the most memory unless entries are small +// (2) sparse_hash_set: slowest, uses the least memory +// (3) hash_set / unordered_set (STL): in the middle +// +// Typically I use sparse_hash_set when I care about space and/or when +// I need to save the hashtable on disk. I use hash_set otherwise. I +// don't personally use dense_hash_set ever; some people use it for +// small sets with lots of lookups. +// +// - dense_hash_set has, typically, about 78% memory overhead (if your +// data takes up X bytes, the hash_set uses .78X more bytes in overhead). +// - sparse_hash_set has about 4 bits overhead per entry. +// - sparse_hash_set can be 3-7 times slower than the others for lookup and, +// especially, inserts. See time_hash_map.cc for details. +// +// See /usr/(local/)?doc/sparsehash-*/sparse_hash_set.html +// for information about how to use this class. + +#ifndef _SPARSE_HASH_SET_H_ +#define _SPARSE_HASH_SET_H_ + +#include +#include // for FILE * in read()/write() +#include // for the default template args +#include // for equal_to +#include // for alloc<> +#include // for pair<> +#include HASH_FUN_H // defined in config.h +#include +#include + +_START_GOOGLE_NAMESPACE_ + +using STL_NAMESPACE::pair; + +template , // defined in sparseconfig.h + class EqualKey = STL_NAMESPACE::equal_to, + class Alloc = libc_allocator_with_realloc > +class sparse_hash_set { + private: + // Apparently identity is not stl-standard, so we define our own + struct Identity { + typedef const Value& result_type; + const Value& operator()(const Value& v) const { return v; } + }; + struct SetKey { + void operator()(Value* value, const Value& new_key) const { + *value = new_key; + } + }; + + typedef sparse_hashtable ht; + ht rep; + + public: + typedef typename ht::key_type key_type; + typedef typename ht::value_type value_type; + typedef typename ht::hasher hasher; + typedef typename ht::key_equal key_equal; + typedef Alloc allocator_type; + + typedef typename ht::size_type size_type; + typedef typename ht::difference_type difference_type; + typedef typename ht::const_pointer pointer; + typedef typename ht::const_pointer const_pointer; + typedef typename ht::const_reference reference; + typedef typename ht::const_reference const_reference; + + typedef typename ht::const_iterator iterator; + typedef typename ht::const_iterator const_iterator; + typedef typename ht::const_local_iterator local_iterator; + typedef typename ht::const_local_iterator const_local_iterator; + + + // Iterator functions -- recall all iterators are const + iterator begin() const { return rep.begin(); } + iterator end() const { return rep.end(); } + + // These come from tr1's unordered_set. For us, a bucket has 0 or 1 elements. + local_iterator begin(size_type i) const { return rep.begin(i); } + local_iterator end(size_type i) const { return rep.end(i); } + + + // Accessor functions + allocator_type get_allocator() const { return rep.get_allocator(); } + hasher hash_funct() const { return rep.hash_funct(); } + hasher hash_function() const { return hash_funct(); } // tr1 name + key_equal key_eq() const { return rep.key_eq(); } + + + // Constructors + explicit sparse_hash_set(size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { + } + + template + sparse_hash_set(InputIterator f, InputIterator l, + size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { + rep.insert(f, l); + } + // We use the default copy constructor + // We use the default operator=() + // We use the default destructor + + void clear() { rep.clear(); } + void swap(sparse_hash_set& hs) { rep.swap(hs.rep); } + + + // Functions concerning size + size_type size() const { return rep.size(); } + size_type max_size() const { return rep.max_size(); } + bool empty() const { return rep.empty(); } + size_type bucket_count() const { return rep.bucket_count(); } + size_type max_bucket_count() const { return rep.max_bucket_count(); } + + // These are tr1 methods. bucket() is the bucket the key is or would be in. + size_type bucket_size(size_type i) const { return rep.bucket_size(i); } + size_type bucket(const key_type& key) const { return rep.bucket(key); } + float load_factor() const { + return size() * 1.0f / bucket_count(); + } + float max_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return grow; + } + void max_load_factor(float new_grow) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(shrink, new_grow); + } + // These aren't tr1 methods but perhaps ought to be. + float min_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return shrink; + } + void min_load_factor(float new_shrink) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(new_shrink, grow); + } + // Deprecated; use min_load_factor() or max_load_factor() instead. + void set_resizing_parameters(float shrink, float grow) { + rep.set_resizing_parameters(shrink, grow); + } + + void resize(size_type hint) { rep.resize(hint); } + void rehash(size_type hint) { resize(hint); } // the tr1 name + + // Lookup routines + iterator find(const key_type& key) const { return rep.find(key); } + + size_type count(const key_type& key) const { return rep.count(key); } + + pair equal_range(const key_type& key) const { + return rep.equal_range(key); + } + + // Insertion routines + pair insert(const value_type& obj) { + pair p = rep.insert(obj); + return pair(p.first, p.second); // const to non-const + } + template + void insert(InputIterator f, InputIterator l) { rep.insert(f, l); } + void insert(const_iterator f, const_iterator l) { rep.insert(f, l); } + // required for std::insert_iterator; the passed-in iterator is ignored + iterator insert(iterator, const value_type& obj) { return insert(obj).first; } + + + // Deletion routines + // THESE ARE NON-STANDARD! I make you specify an "impossible" key + // value to identify deleted buckets. You can change the key as + // time goes on, or get rid of it entirely to be insert-only. + void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); } + void clear_deleted_key() { rep.clear_deleted_key(); } + key_type deleted_key() const { return rep.deleted_key(); } + + // These are standard + size_type erase(const key_type& key) { return rep.erase(key); } + void erase(iterator it) { rep.erase(it); } + void erase(iterator f, iterator l) { rep.erase(f, l); } + + + // Comparison + bool operator==(const sparse_hash_set& hs) const { return rep == hs.rep; } + bool operator!=(const sparse_hash_set& hs) const { return rep != hs.rep; } + + + // I/O -- this is an add-on for writing metainformation to disk + bool write_metadata(FILE *fp) { return rep.write_metadata(fp); } + bool read_metadata(FILE *fp) { return rep.read_metadata(fp); } + bool write_nopointer_data(FILE *fp) { return rep.write_nopointer_data(fp); } + bool read_nopointer_data(FILE *fp) { return rep.read_nopointer_data(fp); } +}; + +template +inline void swap(sparse_hash_set& hs1, + sparse_hash_set& hs2) { + hs1.swap(hs2); +} + +_END_GOOGLE_NAMESPACE_ + +#endif /* _SPARSE_HASH_SET_H_ */ diff --git a/external/google/sparsehash/densehashtable.h b/external/google/sparsehash/densehashtable.h new file mode 100644 index 00000000..8a3126e3 --- /dev/null +++ b/external/google/sparsehash/densehashtable.h @@ -0,0 +1,1268 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein +// +// A dense hashtable is a particular implementation of +// a hashtable: one that is meant to minimize memory allocation. +// It does this by using an array to store all the data. We +// steal a value from the key space to indicate "empty" array +// elements (ie indices where no item lives) and another to indicate +// "deleted" elements. +// +// (Note it is possible to change the value of the delete key +// on the fly; you can even remove it, though after that point +// the hashtable is insert_only until you set it again. The empty +// value however can't be changed.) +// +// To minimize allocation and pointer overhead, we use internal +// probing, in which the hashtable is a single table, and collisions +// are resolved by trying to insert again in another bucket. The +// most cache-efficient internal probing schemes are linear probing +// (which suffers, alas, from clumping) and quadratic probing, which +// is what we implement by default. +// +// Type requirements: value_type is required to be Copy Constructible +// and Default Constructible. It is not required to be (and commonly +// isn't) Assignable. +// +// You probably shouldn't use this code directly. Use +// or instead. + +// You can change the following below: +// HT_OCCUPANCY_PCT -- how full before we double size +// HT_EMPTY_PCT -- how empty before we halve size +// HT_MIN_BUCKETS -- default smallest bucket size +// +// You can also change enlarge_factor (which defaults to +// HT_OCCUPANCY_PCT), and shrink_factor (which defaults to +// HT_EMPTY_PCT) with set_resizing_parameters(). +// +// How to decide what values to use? +// shrink_factor's default of .4 * OCCUPANCY_PCT, is probably good. +// HT_MIN_BUCKETS is probably unnecessary since you can specify +// (indirectly) the starting number of buckets at construct-time. +// For enlarge_factor, you can use this chart to try to trade-off +// expected lookup time to the space taken up. By default, this +// code uses quadratic probing, though you can change it to linear +// via _JUMP below if you really want to. +// +// From http://www.augustana.ca/~mohrj/courses/1999.fall/csc210/lecture_notes/hashing.html +// NUMBER OF PROBES / LOOKUP Successful Unsuccessful +// Quadratic collision resolution 1 - ln(1-L) - L/2 1/(1-L) - L - ln(1-L) +// Linear collision resolution [1+1/(1-L)]/2 [1+1/(1-L)2]/2 +// +// -- enlarge_factor -- 0.10 0.50 0.60 0.75 0.80 0.90 0.99 +// QUADRATIC COLLISION RES. +// probes/successful lookup 1.05 1.44 1.62 2.01 2.21 2.85 5.11 +// probes/unsuccessful lookup 1.11 2.19 2.82 4.64 5.81 11.4 103.6 +// LINEAR COLLISION RES. +// probes/successful lookup 1.06 1.5 1.75 2.5 3.0 5.5 50.5 +// probes/unsuccessful lookup 1.12 2.5 3.6 8.5 13.0 50.0 5000.0 + +#ifndef _DENSEHASHTABLE_H_ +#define _DENSEHASHTABLE_H_ + +// The probing method +// Linear probing +// #define JUMP_(key, num_probes) ( 1 ) +// Quadratic probing +#define JUMP_(key, num_probes) ( num_probes ) + + +#include +#include +#include +#include // for abort() +#include // For swap(), eg +#include // For length_error +#include // For cerr +#include // For uninitialized_fill, uninitialized_copy +#include // for pair<> +#include // for facts about iterator tags +#include // for numeric_limits<> +#include +#include +#include // for true_type, integral_constant, etc. + +_START_GOOGLE_NAMESPACE_ + +using STL_NAMESPACE::pair; + +// Hashtable class, used to implement the hashed associative containers +// hash_set and hash_map. + +// Value: what is stored in the table (each bucket is a Value). +// Key: something in a 1-to-1 correspondence to a Value, that can be used +// to search for a Value in the table (find() takes a Key). +// HashFcn: Takes a Key and returns an integer, the more unique the better. +// ExtractKey: given a Value, returns the unique Key associated with it. +// Must inherit from unary_function, or at least have a +// result_type enum indicating the return type of operator(). +// SetKey: given a Value* and a Key, modifies the value such that +// ExtractKey(value) == key. We guarantee this is only called +// with key == deleted_key or key == empty_key. +// EqualKey: Given two Keys, says whether they are the same (that is, +// if they are both associated with the same Value). +// Alloc: STL allocator to use to allocate memory. + +template +class dense_hashtable; + +template +struct dense_hashtable_iterator; + +template +struct dense_hashtable_const_iterator; + +// We're just an array, but we need to skip over empty and deleted elements +template +struct dense_hashtable_iterator { + private: + typedef typename A::template rebind::other value_alloc_type; + + public: + typedef dense_hashtable_iterator iterator; + typedef dense_hashtable_const_iterator const_iterator; + + typedef STL_NAMESPACE::forward_iterator_tag iterator_category; + typedef V value_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::pointer pointer; + + // "Real" constructor and default constructor + dense_hashtable_iterator(const dense_hashtable *h, + pointer it, pointer it_end, bool advance) + : ht(h), pos(it), end(it_end) { + if (advance) advance_past_empty_and_deleted(); + } + dense_hashtable_iterator() { } + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *pos; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic. The only hard part is making sure that + // we're not on an empty or marked-deleted array element + void advance_past_empty_and_deleted() { + while ( pos != end && (ht->test_empty(*this) || ht->test_deleted(*this)) ) + ++pos; + } + iterator& operator++() { + assert(pos != end); ++pos; advance_past_empty_and_deleted(); return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + + // Comparison. + bool operator==(const iterator& it) const { return pos == it.pos; } + bool operator!=(const iterator& it) const { return pos != it.pos; } + + + // The actual data + const dense_hashtable *ht; + pointer pos, end; +}; + + +// Now do it all again, but with const-ness! +template +struct dense_hashtable_const_iterator { + private: + typedef typename A::template rebind::other value_alloc_type; + + public: + typedef dense_hashtable_iterator iterator; + typedef dense_hashtable_const_iterator const_iterator; + + typedef STL_NAMESPACE::forward_iterator_tag iterator_category; + typedef V value_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::const_reference reference; + typedef typename value_alloc_type::const_pointer pointer; + + // "Real" constructor and default constructor + dense_hashtable_const_iterator( + const dense_hashtable *h, + pointer it, pointer it_end, bool advance) + : ht(h), pos(it), end(it_end) { + if (advance) advance_past_empty_and_deleted(); + } + dense_hashtable_const_iterator() + : ht(NULL), pos(pointer()), end(pointer()) { } + // This lets us convert regular iterators to const iterators + dense_hashtable_const_iterator(const iterator &it) + : ht(it.ht), pos(it.pos), end(it.end) { } + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *pos; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic. The only hard part is making sure that + // we're not on an empty or marked-deleted array element + void advance_past_empty_and_deleted() { + while ( pos != end && (ht->test_empty(*this) || ht->test_deleted(*this)) ) + ++pos; + } + const_iterator& operator++() { + assert(pos != end); ++pos; advance_past_empty_and_deleted(); return *this; + } + const_iterator operator++(int) { const_iterator tmp(*this); ++*this; return tmp; } + + // Comparison. + bool operator==(const const_iterator& it) const { return pos == it.pos; } + bool operator!=(const const_iterator& it) const { return pos != it.pos; } + + + // The actual data + const dense_hashtable *ht; + pointer pos, end; +}; + +template +class dense_hashtable { + private: + typedef typename Alloc::template rebind::other value_alloc_type; + + public: + typedef Key key_type; + typedef Value value_type; + typedef HashFcn hasher; + typedef EqualKey key_equal; + typedef Alloc allocator_type; + + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::const_reference const_reference; + typedef typename value_alloc_type::pointer pointer; + typedef typename value_alloc_type::const_pointer const_pointer; + typedef dense_hashtable_iterator + iterator; + + typedef dense_hashtable_const_iterator + const_iterator; + + // These come from tr1. For us they're the same as regular iterators. + typedef iterator local_iterator; + typedef const_iterator const_local_iterator; + + // How full we let the table get before we resize, by default. + // Knuth says .8 is good -- higher causes us to probe too much, + // though it saves memory. + static const int HT_OCCUPANCY_PCT; // = 50 (out of 100) + + // How empty we let the table get before we resize lower, by default. + // (0.0 means never resize lower.) + // It should be less than OCCUPANCY_PCT / 2 or we thrash resizing + static const int HT_EMPTY_PCT; // = 0.4 * HT_OCCUPANCY_PCT; + + // Minimum size we're willing to let hashtables be. + // Must be a power of two, and at least 4. + // Note, however, that for a given hashtable, the initial size is a + // function of the first constructor arg, and may be >HT_MIN_BUCKETS. + static const size_type HT_MIN_BUCKETS = 4; + + // By default, if you don't specify a hashtable size at + // construction-time, we use this size. Must be a power of two, and + // at least HT_MIN_BUCKETS. + static const size_type HT_DEFAULT_STARTING_BUCKETS = 32; + + // ITERATOR FUNCTIONS + iterator begin() { return iterator(this, table, + table + num_buckets, true); } + iterator end() { return iterator(this, table + num_buckets, + table + num_buckets, true); } + const_iterator begin() const { return const_iterator(this, table, + table+num_buckets,true);} + const_iterator end() const { return const_iterator(this, table + num_buckets, + table+num_buckets,true);} + + // These come from tr1 unordered_map. They iterate over 'bucket' n. + // We'll just consider bucket n to be the n-th element of the table. + local_iterator begin(size_type i) { + return local_iterator(this, table + i, table + i+1, false); + } + local_iterator end(size_type i) { + local_iterator it = begin(i); + if (!test_empty(i) && !test_deleted(i)) + ++it; + return it; + } + const_local_iterator begin(size_type i) const { + return const_local_iterator(this, table + i, table + i+1, false); + } + const_local_iterator end(size_type i) const { + const_local_iterator it = begin(i); + if (!test_empty(i) && !test_deleted(i)) + ++it; + return it; + } + + // ACCESSOR FUNCTIONS for the things we templatize on, basically + hasher hash_funct() const { return settings; } + key_equal key_eq() const { return key_info; } + allocator_type get_allocator() const { + return allocator_type(val_info); + } + + // Accessor function for statistics gathering. + int num_table_copies() const { return settings.num_ht_copies(); } + + private: + // Annoyingly, we can't copy values around, because they might have + // const components (they're probably pair). We use + // explicit destructor invocation and placement new to get around + // this. Arg. + void set_value(pointer dst, const_reference src) { + dst->~value_type(); // delete the old value, if any + new(dst) value_type(src); + } + + void destroy_buckets(size_type first, size_type last) { + for ( ; first != last; ++first) + table[first].~value_type(); + } + + // DELETE HELPER FUNCTIONS + // This lets the user describe a key that will indicate deleted + // table entries. This key should be an "impossible" entry -- + // if you try to insert it for real, you won't be able to retrieve it! + // (NB: while you pass in an entire value, only the key part is looked + // at. This is just because I don't know how to assign just a key.) + private: + void squash_deleted() { // gets rid of any deleted entries we have + if ( num_deleted ) { // get rid of deleted before writing + dense_hashtable tmp(*this); // copying will get rid of deleted + swap(tmp); // now we are tmp + } + assert(num_deleted == 0); + } + + bool test_deleted_key(const key_type& key) const { + // The num_deleted test is crucial for read(): after read(), the ht values + // are garbage, and we don't want to think some of them are deleted. + // Invariant: !use_deleted implies num_deleted is 0. + assert(settings.use_deleted() || num_deleted == 0); + return num_deleted > 0 && equals(key_info.delkey, key); + } + + public: + void set_deleted_key(const key_type &key) { + // the empty indicator (if specified) and the deleted indicator + // must be different + assert((!settings.use_empty() || !equals(key, get_key(val_info.emptyval))) + && "Passed the empty-key to set_deleted_key"); + // It's only safe to change what "deleted" means if we purge deleted guys + squash_deleted(); + settings.set_use_deleted(true); + key_info.delkey = key; + } + void clear_deleted_key() { + squash_deleted(); + settings.set_use_deleted(false); + } + key_type deleted_key() const { + assert(settings.use_deleted() + && "Must set deleted key before calling deleted_key"); + return key_info.delkey; + } + + // These are public so the iterators can use them + // True if the item at position bucknum is "deleted" marker + bool test_deleted(size_type bucknum) const { + return test_deleted_key(get_key(table[bucknum])); + } + bool test_deleted(const iterator &it) const { + return test_deleted_key(get_key(*it)); + } + bool test_deleted(const const_iterator &it) const { + return test_deleted_key(get_key(*it)); + } + + private: + // Set it so test_deleted is true. true if object didn't used to be deleted. + bool set_deleted(iterator &it) { + assert(settings.use_deleted()); + bool retval = !test_deleted(it); + // &* converts from iterator to value-type. + set_key(&(*it), key_info.delkey); + return retval; + } + // Set it so test_deleted is false. true if object used to be deleted. + bool clear_deleted(iterator &it) { + assert(settings.use_deleted()); + // Happens automatically when we assign something else in its place. + return test_deleted(it); + } + + // We also allow to set/clear the deleted bit on a const iterator. + // We allow a const_iterator for the same reason you can delete a + // const pointer: it's convenient, and semantically you can't use + // 'it' after it's been deleted anyway, so its const-ness doesn't + // really matter. + bool set_deleted(const_iterator &it) { + assert(settings.use_deleted()); + bool retval = !test_deleted(it); + set_key(const_cast(&(*it)), key_info.delkey); + return retval; + } + // Set it so test_deleted is false. true if object used to be deleted. + bool clear_deleted(const_iterator &it) { + assert(settings.use_deleted()); + return test_deleted(it); + } + + // EMPTY HELPER FUNCTIONS + // This lets the user describe a key that will indicate empty (unused) + // table entries. This key should be an "impossible" entry -- + // if you try to insert it for real, you won't be able to retrieve it! + // (NB: while you pass in an entire value, only the key part is looked + // at. This is just because I don't know how to assign just a key.) + public: + // These are public so the iterators can use them + // True if the item at position bucknum is "empty" marker + bool test_empty(size_type bucknum) const { + assert(settings.use_empty()); // we always need to know what's empty! + return equals(get_key(val_info.emptyval), get_key(table[bucknum])); + } + bool test_empty(const iterator &it) const { + assert(settings.use_empty()); // we always need to know what's empty! + return equals(get_key(val_info.emptyval), get_key(*it)); + } + bool test_empty(const const_iterator &it) const { + assert(settings.use_empty()); // we always need to know what's empty! + return equals(get_key(val_info.emptyval), get_key(*it)); + } + + private: + void fill_range_with_empty(pointer table_start, pointer table_end) { + STL_NAMESPACE::uninitialized_fill(table_start, table_end, val_info.emptyval); + } + + public: + // TODO(csilvers): change all callers of this to pass in a key instead, + // and take a const key_type instead of const value_type. + void set_empty_key(const_reference val) { + // Once you set the empty key, you can't change it + assert(!settings.use_empty() && "Calling set_empty_key multiple times"); + // The deleted indicator (if specified) and the empty indicator + // must be different. + assert((!settings.use_deleted() || !equals(get_key(val), key_info.delkey)) + && "Setting the empty key the same as the deleted key"); + settings.set_use_empty(true); + set_value(&val_info.emptyval, val); + + assert(!table); // must set before first use + // num_buckets was set in constructor even though table was NULL + table = val_info.allocate(num_buckets); + assert(table); + fill_range_with_empty(table, table + num_buckets); + } + // TODO(sjackman): return a key_type rather than a value_type + value_type empty_key() const { + assert(settings.use_empty()); + return val_info.emptyval; + } + + // FUNCTIONS CONCERNING SIZE + public: + size_type size() const { return num_elements - num_deleted; } + size_type max_size() const { return val_info.max_size(); } + bool empty() const { return size() == 0; } + size_type bucket_count() const { return num_buckets; } + size_type max_bucket_count() const { return max_size(); } + size_type nonempty_bucket_count() const { return num_elements; } + // These are tr1 methods. Their idea of 'bucket' doesn't map well to + // what we do. We just say every bucket has 0 or 1 items in it. + size_type bucket_size(size_type i) const { + return begin(i) == end(i) ? 0 : 1; + } + + private: + // Because of the above, size_type(-1) is never legal; use it for errors + static const size_type ILLEGAL_BUCKET = size_type(-1); + + // Used after a string of deletes. Returns true if we actually shrunk. + // TODO(csilvers): take a delta so we can take into account inserts + // done after shrinking. Maybe make part of the Settings class? + bool maybe_shrink() { + assert(num_elements >= num_deleted); + assert((bucket_count() & (bucket_count()-1)) == 0); // is a power of two + assert(bucket_count() >= HT_MIN_BUCKETS); + bool retval = false; + + // If you construct a hashtable with < HT_DEFAULT_STARTING_BUCKETS, + // we'll never shrink until you get relatively big, and we'll never + // shrink below HT_DEFAULT_STARTING_BUCKETS. Otherwise, something + // like "dense_hash_set x; x.insert(4); x.erase(4);" will + // shrink us down to HT_MIN_BUCKETS buckets, which is too small. + const size_type num_remain = num_elements - num_deleted; + const size_type shrink_threshold = settings.shrink_threshold(); + if (shrink_threshold > 0 && num_remain < shrink_threshold && + bucket_count() > HT_DEFAULT_STARTING_BUCKETS) { + const float shrink_factor = settings.shrink_factor(); + size_type sz = bucket_count() / 2; // find how much we should shrink + while (sz > HT_DEFAULT_STARTING_BUCKETS && + num_remain < sz * shrink_factor) { + sz /= 2; // stay a power of 2 + } + dense_hashtable tmp(*this, sz); // Do the actual resizing + swap(tmp); // now we are tmp + retval = true; + } + settings.set_consider_shrink(false); // because we just considered it + return retval; + } + + // We'll let you resize a hashtable -- though this makes us copy all! + // When you resize, you say, "make it big enough for this many more elements" + // Returns true if we actually resized, false if size was already ok. + bool resize_delta(size_type delta) { + bool did_resize = false; + if ( settings.consider_shrink() ) { // see if lots of deletes happened + if ( maybe_shrink() ) + did_resize = true; + } + if (num_elements >= (STL_NAMESPACE::numeric_limits::max)() - delta) + throw std::length_error("resize overflow"); + if ( bucket_count() >= HT_MIN_BUCKETS && + (num_elements + delta) <= settings.enlarge_threshold() ) + return did_resize; // we're ok as we are + + // Sometimes, we need to resize just to get rid of all the + // "deleted" buckets that are clogging up the hashtable. So when + // deciding whether to resize, count the deleted buckets (which + // are currently taking up room). But later, when we decide what + // size to resize to, *don't* count deleted buckets, since they + // get discarded during the resize. + const size_type needed_size = settings.min_buckets(num_elements + delta, 0); + if ( needed_size <= bucket_count() ) // we have enough buckets + return did_resize; + + size_type resize_to = + settings.min_buckets(num_elements - num_deleted + delta, bucket_count()); + + if (resize_to < needed_size && // may double resize_to + resize_to < (STL_NAMESPACE::numeric_limits::max)() / 2) { + // This situation means that we have enough deleted elements, + // that once we purge them, we won't actually have needed to + // grow. But we may want to grow anyway: if we just purge one + // element, say, we'll have to grow anyway next time we + // insert. Might as well grow now, since we're already going + // through the trouble of copying (in order to purge the + // deleted elements). + const size_type target = + static_cast(settings.shrink_size(resize_to*2)); + if (num_elements - num_deleted + delta >= target) { + // Good, we won't be below the shrink threshhold even if we double. + resize_to *= 2; + } + } + dense_hashtable tmp(*this, resize_to); + swap(tmp); // now we are tmp + return true; + } + + // We require table be not-NULL and empty before calling this. + void resize_table(size_type /*old_size*/, size_type new_size, + true_type) { + table = val_info.realloc_or_die(table, new_size); + } + + void resize_table(size_type old_size, size_type new_size, false_type) { + val_info.deallocate(table, old_size); + table = val_info.allocate(new_size); + } + + // Used to actually do the rehashing when we grow/shrink a hashtable + void copy_from(const dense_hashtable &ht, size_type min_buckets_wanted) { + clear_to_size(settings.min_buckets(ht.size(), min_buckets_wanted)); + + // We use a normal iterator to get non-deleted bcks from ht + // We could use insert() here, but since we know there are + // no duplicates and no deleted items, we can be more efficient + assert((bucket_count() & (bucket_count()-1)) == 0); // a power of two + for ( const_iterator it = ht.begin(); it != ht.end(); ++it ) { + size_type num_probes = 0; // how many times we've probed + size_type bucknum; + const size_type bucket_count_minus_one = bucket_count() - 1; + for (bucknum = hash(get_key(*it)) & bucket_count_minus_one; + !test_empty(bucknum); // not empty + bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one) { + ++num_probes; + assert(num_probes < bucket_count() + && "Hashtable is full: an error in key_equal<> or hash<>"); + } + set_value(&table[bucknum], *it); // copies the value to here + num_elements++; + } + settings.inc_num_ht_copies(); + } + + // Required by the spec for hashed associative container + public: + // Though the docs say this should be num_buckets, I think it's much + // more useful as num_elements. As a special feature, calling with + // req_elements==0 will cause us to shrink if we can, saving space. + void resize(size_type req_elements) { // resize to this or larger + if ( settings.consider_shrink() || req_elements == 0 ) + maybe_shrink(); + if ( req_elements > num_elements ) + resize_delta(req_elements - num_elements); + } + + // Get and change the value of shrink_factor and enlarge_factor. The + // description at the beginning of this file explains how to choose + // the values. Setting the shrink parameter to 0.0 ensures that the + // table never shrinks. + void get_resizing_parameters(float* shrink, float* grow) const { + *shrink = settings.shrink_factor(); + *grow = settings.enlarge_factor(); + } + void set_resizing_parameters(float shrink, float grow) { + settings.set_resizing_parameters(shrink, grow); + settings.reset_thresholds(bucket_count()); + } + + // CONSTRUCTORS -- as required by the specs, we take a size, + // but also let you specify a hashfunction, key comparator, + // and key extractor. We also define a copy constructor and =. + // DESTRUCTOR -- needs to free the table + explicit dense_hashtable(size_type expected_max_items_in_table = 0, + const HashFcn& hf = HashFcn(), + const EqualKey& eql = EqualKey(), + const ExtractKey& ext = ExtractKey(), + const SetKey& set = SetKey(), + const Alloc& alloc = Alloc()) + : settings(hf), + key_info(ext, set, eql), + num_deleted(0), + num_elements(0), + num_buckets(expected_max_items_in_table == 0 + ? HT_DEFAULT_STARTING_BUCKETS + : settings.min_buckets(expected_max_items_in_table, 0)), + val_info(alloc_impl(alloc)), + table(NULL) { + // table is NULL until emptyval is set. However, we set num_buckets + // here so we know how much space to allocate once emptyval is set + settings.reset_thresholds(bucket_count()); + } + + // As a convenience for resize(), we allow an optional second argument + // which lets you make this new hashtable a different size than ht + dense_hashtable(const dense_hashtable& ht, + size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS) + : settings(ht.settings), + key_info(ht.key_info), + num_deleted(0), + num_elements(0), + num_buckets(0), + val_info(ht.val_info), + table(NULL) { + if (!ht.settings.use_empty()) { + // If use_empty isn't set, copy_from will crash, so we do our own copying. + assert(ht.empty()); + num_buckets = settings.min_buckets(ht.size(), min_buckets_wanted); + settings.reset_thresholds(bucket_count()); + return; + } + settings.reset_thresholds(bucket_count()); + copy_from(ht, min_buckets_wanted); // copy_from() ignores deleted entries + } + + dense_hashtable& operator= (const dense_hashtable& ht) { + if (&ht == this) return *this; // don't copy onto ourselves + if (!ht.settings.use_empty()) { + assert(ht.empty()); + dense_hashtable empty_table(ht); // empty table with ht's thresholds + this->swap(empty_table); + return *this; + } + settings = ht.settings; + key_info = ht.key_info; + set_value(&val_info.emptyval, ht.val_info.emptyval); + // copy_from() calls clear and sets num_deleted to 0 too + copy_from(ht, HT_MIN_BUCKETS); + // we purposefully don't copy the allocator, which may not be copyable + return *this; + } + + ~dense_hashtable() { + if (table) { + destroy_buckets(0, num_buckets); + val_info.deallocate(table, num_buckets); + } + } + + // Many STL algorithms use swap instead of copy constructors + void swap(dense_hashtable& ht) { + STL_NAMESPACE::swap(settings, ht.settings); + STL_NAMESPACE::swap(key_info, ht.key_info); + STL_NAMESPACE::swap(num_deleted, ht.num_deleted); + STL_NAMESPACE::swap(num_elements, ht.num_elements); + STL_NAMESPACE::swap(num_buckets, ht.num_buckets); + { value_type tmp; // for annoying reasons, swap() doesn't work + set_value(&tmp, val_info.emptyval); + set_value(&val_info.emptyval, ht.val_info.emptyval); + set_value(&ht.val_info.emptyval, tmp); + } + STL_NAMESPACE::swap(table, ht.table); + settings.reset_thresholds(bucket_count()); // this also resets consider_shrink + ht.settings.reset_thresholds(bucket_count()); + // we purposefully don't swap the allocator, which may not be swap-able + } + + private: + void clear_to_size(size_type new_num_buckets) { + if (!table) { + table = val_info.allocate(new_num_buckets); + } else { + destroy_buckets(0, num_buckets); + if (new_num_buckets != num_buckets) { // resize, if necessary + typedef integral_constant >::value> + realloc_ok; + resize_table(num_buckets, new_num_buckets, realloc_ok()); + } + } + assert(table); + fill_range_with_empty(table, table + new_num_buckets); + num_elements = 0; + num_deleted = 0; + num_buckets = new_num_buckets; // our new size + settings.reset_thresholds(bucket_count()); + } + + public: + // It's always nice to be able to clear a table without deallocating it + void clear() { + // If the table is already empty, and the number of buckets is + // already as we desire, there's nothing to do. + const size_type new_num_buckets = settings.min_buckets(0, 0); + if (num_elements == 0 && new_num_buckets == num_buckets) { + return; + } + clear_to_size(new_num_buckets); + } + + // Clear the table without resizing it. + // Mimicks the stl_hashtable's behaviour when clear()-ing in that it + // does not modify the bucket count + void clear_no_resize() { + if (num_elements > 0) { + assert(table); + destroy_buckets(0, num_buckets); + fill_range_with_empty(table, table + num_buckets); + } + // don't consider to shrink before another erase() + settings.reset_thresholds(bucket_count()); + num_elements = 0; + num_deleted = 0; + } + + // LOOKUP ROUTINES + private: + // Returns a pair of positions: 1st where the object is, 2nd where + // it would go if you wanted to insert it. 1st is ILLEGAL_BUCKET + // if object is not found; 2nd is ILLEGAL_BUCKET if it is. + // Note: because of deletions where-to-insert is not trivial: it's the + // first deleted bucket we see, as long as we don't find the key later + pair find_position(const key_type &key) const { + size_type num_probes = 0; // how many times we've probed + const size_type bucket_count_minus_one = bucket_count() - 1; + size_type bucknum = hash(key) & bucket_count_minus_one; + size_type insert_pos = ILLEGAL_BUCKET; // where we would insert + while ( 1 ) { // probe until something happens + if ( test_empty(bucknum) ) { // bucket is empty + if ( insert_pos == ILLEGAL_BUCKET ) // found no prior place to insert + return pair(ILLEGAL_BUCKET, bucknum); + else + return pair(ILLEGAL_BUCKET, insert_pos); + + } else if ( test_deleted(bucknum) ) {// keep searching, but mark to insert + if ( insert_pos == ILLEGAL_BUCKET ) + insert_pos = bucknum; + + } else if ( equals(key, get_key(table[bucknum])) ) { + return pair(bucknum, ILLEGAL_BUCKET); + } + ++num_probes; // we're doing another probe + bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one; + assert(num_probes < bucket_count() + && "Hashtable is full: an error in key_equal<> or hash<>"); + } + } + + public: + iterator find(const key_type& key) { + if ( size() == 0 ) return end(); + pair pos = find_position(key); + if ( pos.first == ILLEGAL_BUCKET ) // alas, not there + return end(); + else + return iterator(this, table + pos.first, table + num_buckets, false); + } + + const_iterator find(const key_type& key) const { + if ( size() == 0 ) return end(); + pair pos = find_position(key); + if ( pos.first == ILLEGAL_BUCKET ) // alas, not there + return end(); + else + return const_iterator(this, table + pos.first, table+num_buckets, false); + } + + // This is a tr1 method: the bucket a given key is in, or what bucket + // it would be put in, if it were to be inserted. Shrug. + size_type bucket(const key_type& key) const { + pair pos = find_position(key); + return pos.first == ILLEGAL_BUCKET ? pos.second : pos.first; + } + + // Counts how many elements have key key. For maps, it's either 0 or 1. + size_type count(const key_type &key) const { + pair pos = find_position(key); + return pos.first == ILLEGAL_BUCKET ? 0 : 1; + } + + // Likewise, equal_range doesn't really make sense for us. Oh well. + pair equal_range(const key_type& key) { + iterator pos = find(key); // either an iterator or end + if (pos == end()) { + return pair(pos, pos); + } else { + const iterator startpos = pos++; + return pair(startpos, pos); + } + } + pair equal_range(const key_type& key) const { + const_iterator pos = find(key); // either an iterator or end + if (pos == end()) { + return pair(pos, pos); + } else { + const const_iterator startpos = pos++; + return pair(startpos, pos); + } + } + + + // INSERTION ROUTINES + private: + // Private method used by insert_noresize and find_or_insert. + iterator insert_at(const_reference obj, size_type pos) { + if (size() >= max_size()) + throw std::length_error("insert overflow"); + if ( test_deleted(pos) ) { // just replace if it's been del. + // shrug: shouldn't need to be const. + const_iterator delpos(this, table + pos, table + num_buckets, false); + clear_deleted(delpos); + assert( num_deleted > 0); + --num_deleted; // used to be, now it isn't + } else { + ++num_elements; // replacing an empty bucket + } + set_value(&table[pos], obj); + return iterator(this, table + pos, table + num_buckets, false); + } + + // If you know *this is big enough to hold obj, use this routine + pair insert_noresize(const_reference obj) { + // First, double-check we're not inserting delkey or emptyval + assert((!settings.use_empty() || !equals(get_key(obj), + get_key(val_info.emptyval))) + && "Inserting the empty key"); + assert((!settings.use_deleted() || !equals(get_key(obj), key_info.delkey)) + && "Inserting the deleted key"); + const pair pos = find_position(get_key(obj)); + if ( pos.first != ILLEGAL_BUCKET) { // object was already there + return pair(iterator(this, table + pos.first, + table + num_buckets, false), + false); // false: we didn't insert + } else { // pos.second says where to put it + return pair(insert_at(obj, pos.second), true); + } + } + + // Specializations of insert(it, it) depending on the power of the iterator: + // (1) Iterator supports operator-, resize before inserting + template + void insert(ForwardIterator f, ForwardIterator l, STL_NAMESPACE::forward_iterator_tag) { + size_t dist = STL_NAMESPACE::distance(f, l); + if (dist >= (std::numeric_limits::max)()) + throw std::length_error("insert-range overflow"); + resize_delta(static_cast(dist)); + for ( ; dist > 0; --dist, ++f) { + insert_noresize(*f); + } + } + + // (2) Arbitrary iterator, can't tell how much to resize + template + void insert(InputIterator f, InputIterator l, STL_NAMESPACE::input_iterator_tag) { + for ( ; f != l; ++f) + insert(*f); + } + + public: + // This is the normal insert routine, used by the outside world + pair insert(const_reference obj) { + resize_delta(1); // adding an object, grow if need be + return insert_noresize(obj); + } + + // When inserting a lot at a time, we specialize on the type of iterator + template + void insert(InputIterator f, InputIterator l) { + // specializes on iterator type + insert(f, l, typename STL_NAMESPACE::iterator_traits::iterator_category()); + } + + // DefaultValue is a functor that takes a key and returns a value_type + // representing the default value to be inserted if none is found. + template + value_type& find_or_insert(const key_type& key) { + // First, double-check we're not inserting emptykey or delkey + assert((!settings.use_empty() || !equals(key, get_key(val_info.emptyval))) + && "Inserting the empty key"); + assert((!settings.use_deleted() || !equals(key, key_info.delkey)) + && "Inserting the deleted key"); + const pair pos = find_position(key); + DefaultValue default_value; + if ( pos.first != ILLEGAL_BUCKET) { // object was already there + return table[pos.first]; + } else if (resize_delta(1)) { // needed to rehash to make room + // Since we resized, we can't use pos, so recalculate where to insert. + return *insert_noresize(default_value(key)).first; + } else { // no need to rehash, insert right here + return *insert_at(default_value(key), pos.second); + } + } + + // DELETION ROUTINES + size_type erase(const key_type& key) { + // First, double-check we're not trying to erase delkey or emptyval. + assert((!settings.use_empty() || !equals(key, get_key(val_info.emptyval))) + && "Erasing the empty key"); + assert((!settings.use_deleted() || !equals(key, key_info.delkey)) + && "Erasing the deleted key"); + const_iterator pos = find(key); // shrug: shouldn't need to be const + if ( pos != end() ) { + assert(!test_deleted(pos)); // or find() shouldn't have returned it + set_deleted(pos); + ++num_deleted; + settings.set_consider_shrink(true); // will think about shrink after next insert + return 1; // because we deleted one thing + } else { + return 0; // because we deleted nothing + } + } + + // We return the iterator past the deleted item. + void erase(iterator pos) { + if ( pos == end() ) return; // sanity check + if ( set_deleted(pos) ) { // true if object has been newly deleted + ++num_deleted; + settings.set_consider_shrink(true); // will think about shrink after next insert + } + } + + void erase(iterator f, iterator l) { + for ( ; f != l; ++f) { + if ( set_deleted(f) ) // should always be true + ++num_deleted; + } + settings.set_consider_shrink(true); // will think about shrink after next insert + } + + // We allow you to erase a const_iterator just like we allow you to + // erase an iterator. This is in parallel to 'delete': you can delete + // a const pointer just like a non-const pointer. The logic is that + // you can't use the object after it's erased anyway, so it doesn't matter + // if it's const or not. + void erase(const_iterator pos) { + if ( pos == end() ) return; // sanity check + if ( set_deleted(pos) ) { // true if object has been newly deleted + ++num_deleted; + settings.set_consider_shrink(true); // will think about shrink after next insert + } + } + void erase(const_iterator f, const_iterator l) { + for ( ; f != l; ++f) { + if ( set_deleted(f) ) // should always be true + ++num_deleted; + } + settings.set_consider_shrink(true); // will think about shrink after next insert + } + + + // COMPARISON + bool operator==(const dense_hashtable& ht) const { + if (size() != ht.size()) { + return false; + } else if (this == &ht) { + return true; + } else { + // Iterate through the elements in "this" and see if the + // corresponding element is in ht + for ( const_iterator it = begin(); it != end(); ++it ) { + const_iterator it2 = ht.find(get_key(*it)); + if ((it2 == ht.end()) || (*it != *it2)) { + return false; + } + } + return true; + } + } + bool operator!=(const dense_hashtable& ht) const { + return !(*this == ht); + } + + + // I/O + // We support reading and writing hashtables to disk. Alas, since + // I don't know how to write a hasher or key_equal, you have to make + // sure everything but the table is the same. We compact before writing + // + // NOTE: These functions are currently TODO. They've not been implemented. + bool write_metadata(FILE * /*fp*/) { + squash_deleted(); // so we don't have to worry about delkey + return false; // TODO + } + + bool read_metadata(FILE* /*fp*/) { + num_deleted = 0; // since we got rid before writing + assert(settings.use_empty() && "empty_key not set for read_metadata"); + if (table) val_info.deallocate(table, num_buckets); // we'll make our own + // TODO: read magic number + // TODO: read num_buckets + settings.reset_thresholds(bucket_count()); + table = val_info.allocate(num_buckets); + assert(table); + fill_range_with_empty(table, table + num_buckets); + // TODO: read num_elements + for ( size_type i = 0; i < num_elements; ++i ) { + // TODO: read bucket_num + // TODO: set with non-empty, non-deleted value + } + return false; // TODO + } + + // If your keys and values are simple enough, we can write them to + // disk for you. "simple enough" means value_type is a POD type + // that contains no pointers. However, we don't try to normalize + // endianness + bool write_nopointer_data(FILE *fp) const { + for ( const_iterator it = begin(); it != end(); ++it ) { + // TODO: skip empty/deleted values + if ( !fwrite(&*it, sizeof(*it), 1, fp) ) return false; + } + return false; + } + + // When reading, we have to override the potential const-ness of *it + bool read_nopointer_data(FILE *fp) { + for ( iterator it = begin(); it != end(); ++it ) { + // TODO: skip empty/deleted values + if ( !fread(reinterpret_cast(&(*it)), sizeof(*it), 1, fp) ) + return false; + } + return false; + } + + private: + template + class alloc_impl : public A { + public: + typedef typename A::pointer pointer; + typedef typename A::size_type size_type; + + // Convert a normal allocator to one that has realloc_or_die() + alloc_impl(const A& a) : A(a) { } + + // realloc_or_die should only be used when using the default + // allocator (libc_allocator_with_realloc). + pointer realloc_or_die(pointer /*ptr*/, size_type /*n*/) { + fprintf(stderr, "realloc_or_die is only supported for " + "libc_allocator_with_realloc"); + exit(1); + return NULL; + } + }; + + // A template specialization of alloc_impl for + // libc_allocator_with_realloc that can handle realloc_or_die. + template + class alloc_impl > + : public libc_allocator_with_realloc { + public: + typedef typename libc_allocator_with_realloc::pointer pointer; + typedef typename libc_allocator_with_realloc::size_type size_type; + + alloc_impl(const libc_allocator_with_realloc& a) + : libc_allocator_with_realloc(a) { } + + pointer realloc_or_die(pointer ptr, size_type n) { + pointer retval = this->reallocate(ptr, n); + if (retval == NULL) { + // We really should use PRIuS here, but I don't want to have to add + // a whole new configure option, with concomitant macro namespace + // pollution, just to print this (unlikely) error message. So I cast. + fprintf(stderr, "sparsehash: FATAL ERROR: failed to reallocate " + "%lu elements for ptr %p", + static_cast(n), ptr); + exit(1); + } + return retval; + } + }; + + // Package allocator with emptyval to eliminate memory needed for + // the zero-size allocator. + // If new fields are added to this class, we should add them to + // operator= and swap. + class ValInfo : public alloc_impl { + public: + typedef typename alloc_impl::value_type value_type; + + ValInfo(const alloc_impl& a) + : alloc_impl(a), emptyval() { } + ValInfo(const ValInfo& v) + : alloc_impl(v), emptyval(v.emptyval) { } + + value_type emptyval; // which key marks unused entries + }; + + + // Package functors with another class to eliminate memory needed for + // zero-size functors. Since ExtractKey and hasher's operator() might + // have the same function signature, they must be packaged in + // different classes. + struct Settings : + sh_hashtable_settings { + explicit Settings(const hasher& hf) + : sh_hashtable_settings( + hf, HT_OCCUPANCY_PCT / 100.0f, HT_EMPTY_PCT / 100.0f) {} + }; + + // Packages ExtractKey and SetKey functors. + class KeyInfo : public ExtractKey, public SetKey, public key_equal { + public: + KeyInfo(const ExtractKey& ek, const SetKey& sk, const key_equal& eq) + : ExtractKey(ek), + SetKey(sk), + key_equal(eq) { + } + + // We want to return the exact same type as ExtractKey: Key or const Key& + typename ExtractKey::result_type get_key(const_reference v) const { + return ExtractKey::operator()(v); + } + void set_key(pointer v, const key_type& k) const { + SetKey::operator()(v, k); + } + bool equals(const key_type& a, const key_type& b) const { + return key_equal::operator()(a, b); + } + + // Which key marks deleted entries. + // TODO(csilvers): make a pointer, and get rid of use_deleted (benchmark!) + typename remove_const::type delkey; + }; + + // Utility functions to access the templated operators + size_type hash(const key_type& v) const { + return settings.hash(v); + } + bool equals(const key_type& a, const key_type& b) const { + return key_info.equals(a, b); + } + typename ExtractKey::result_type get_key(const_reference v) const { + return key_info.get_key(v); + } + void set_key(pointer v, const key_type& k) const { + key_info.set_key(v, k); + } + + private: + // Actual data + Settings settings; + KeyInfo key_info; + + size_type num_deleted; // how many occupied buckets are marked deleted + size_type num_elements; + size_type num_buckets; + ValInfo val_info; // holds emptyval, and also the allocator + pointer table; +}; + + +// We need a global swap as well +template +inline void swap(dense_hashtable &x, + dense_hashtable &y) { + x.swap(y); +} + +#undef JUMP_ + +template +const typename dense_hashtable::size_type + dense_hashtable::ILLEGAL_BUCKET; + +// How full we let the table get before we resize. Knuth says .8 is +// good -- higher causes us to probe too much, though saves memory. +// However, we go with .5, getting better performance at the cost of +// more space (a trade-off densehashtable explicitly chooses to make). +// Feel free to play around with different values, though. +template +const int dense_hashtable::HT_OCCUPANCY_PCT = 50; + +// How empty we let the table get before we resize lower. +// It should be less than OCCUPANCY_PCT / 2 or we thrash resizing +template +const int dense_hashtable::HT_EMPTY_PCT + = static_cast(0.4 * + dense_hashtable::HT_OCCUPANCY_PCT); + +_END_GOOGLE_NAMESPACE_ + +#endif /* _DENSEHASHTABLE_H_ */ diff --git a/external/google/sparsehash/hashtable-common.h b/external/google/sparsehash/hashtable-common.h new file mode 100644 index 00000000..e823b124 --- /dev/null +++ b/external/google/sparsehash/hashtable-common.h @@ -0,0 +1,178 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Giao Nguyen + +#ifndef UTIL_GTL_HASHTABLE_COMMON_H_ +#define UTIL_GTL_HASHTABLE_COMMON_H_ + +#include + +// Settings contains parameters for growing and shrinking the table. +// It also packages zero-size functor (ie. hasher). + +template +class sh_hashtable_settings : public HashFunc { + public: + typedef Key key_type; + typedef HashFunc hasher; + typedef SizeType size_type; + + public: + sh_hashtable_settings(const hasher& hf, + const float ht_occupancy_flt, + const float ht_empty_flt) + : hasher(hf), + enlarge_threshold_(0), + shrink_threshold_(0), + consider_shrink_(false), + use_empty_(false), + use_deleted_(false), + num_ht_copies_(0) { + set_enlarge_factor(ht_occupancy_flt); + set_shrink_factor(ht_empty_flt); + } + + size_type hash(const key_type& v) const { + return hasher::operator()(v); + } + + float enlarge_factor() const { + return enlarge_factor_; + } + void set_enlarge_factor(float f) { + enlarge_factor_ = f; + } + float shrink_factor() const { + return shrink_factor_; + } + void set_shrink_factor(float f) { + shrink_factor_ = f; + } + + size_type enlarge_threshold() const { + return enlarge_threshold_; + } + void set_enlarge_threshold(size_type t) { + enlarge_threshold_ = t; + } + size_type shrink_threshold() const { + return shrink_threshold_; + } + void set_shrink_threshold(size_type t) { + shrink_threshold_ = t; + } + + size_type enlarge_size(size_type x) const { + return static_cast(x * enlarge_factor_); + } + size_type shrink_size(size_type x) const { + return static_cast(x * shrink_factor_); + } + + bool consider_shrink() const { + return consider_shrink_; + } + void set_consider_shrink(bool t) { + consider_shrink_ = t; + } + + bool use_empty() const { + return use_empty_; + } + void set_use_empty(bool t) { + use_empty_ = t; + } + + bool use_deleted() const { + return use_deleted_; + } + void set_use_deleted(bool t) { + use_deleted_ = t; + } + + size_type num_ht_copies() const { + return static_cast(num_ht_copies_); + } + void inc_num_ht_copies() { + ++num_ht_copies_; + } + + // Reset the enlarge and shrink thresholds + void reset_thresholds(size_type num_buckets) { + set_enlarge_threshold(enlarge_size(num_buckets)); + set_shrink_threshold(shrink_size(num_buckets)); + // whatever caused us to reset already considered + set_consider_shrink(false); + } + + // Caller is resposible for calling reset_threshold right after + // set_resizing_parameters. + void set_resizing_parameters(float shrink, float grow) { + assert(shrink >= 0.0); + assert(grow <= 1.0); + if (shrink > grow/2.0f) + shrink = grow / 2.0f; // otherwise we thrash hashtable size + set_shrink_factor(shrink); + set_enlarge_factor(grow); + } + + // This is the smallest size a hashtable can be without being too crowded + // If you like, you can give a min #buckets as well as a min #elts + size_type min_buckets(size_type num_elts, size_type min_buckets_wanted) { + float enlarge = enlarge_factor(); + size_type sz = HT_MIN_BUCKETS; // min buckets allowed + while ( sz < min_buckets_wanted || + num_elts >= static_cast(sz * enlarge) ) { + // This just prevents overflowing size_type, since sz can exceed + // max_size() here. + if (static_cast(sz * 2) < sz) { + throw std::length_error("resize overflow"); // protect against overflow + } + sz *= 2; + } + return sz; + } + + private: + size_type enlarge_threshold_; // table.size() * enlarge_factor + size_type shrink_threshold_; // table.size() * shrink_factor + float enlarge_factor_; // how full before resize + float shrink_factor_; // how empty before resize + // consider_shrink=true if we should try to shrink before next insert + bool consider_shrink_; + bool use_empty_; // used only by densehashtable, not sparsehashtable + bool use_deleted_; // false until delkey has been set + // num_ht_copies is a counter incremented every Copy/Move + unsigned int num_ht_copies_; +}; + +#endif // UTIL_GTL_HASHTABLE_COMMON_H_ diff --git a/external/google/sparsehash/libc_allocator_with_realloc.h b/external/google/sparsehash/libc_allocator_with_realloc.h new file mode 100644 index 00000000..4ba1db48 --- /dev/null +++ b/external/google/sparsehash/libc_allocator_with_realloc.h @@ -0,0 +1,121 @@ +// Copyright (c) 2010, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Guilin Chen + +#ifndef UTIL_GTL_LIBC_ALLOCATOR_WITH_REALLOC_H_ +#define UTIL_GTL_LIBC_ALLOCATOR_WITH_REALLOC_H_ + +#include + +#include // for malloc/realloc/free +#include // for ptrdiff_t + + +_START_GOOGLE_NAMESPACE_ + +template +class libc_allocator_with_realloc { + public: + typedef T value_type; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + + typedef T* pointer; + typedef const T* const_pointer; + typedef T& reference; + typedef const T& const_reference; + + libc_allocator_with_realloc() {} + libc_allocator_with_realloc(const libc_allocator_with_realloc&) {} + ~libc_allocator_with_realloc() {} + + pointer address(reference r) const { return &r; } + const_pointer address(const_reference r) const { return &r; } + + pointer allocate(size_type n, const_pointer = 0) { + return static_cast(malloc(n * sizeof(value_type))); + } + void deallocate(pointer p, size_type) { + free(p); + } + pointer reallocate(pointer p, size_type n) { + return static_cast(realloc(p, n * sizeof(value_type))); + } + + size_type max_size() const { + return static_cast(-1) / sizeof(value_type); + } + + void construct(pointer p, const value_type& val) { + new(p) value_type(val); + } + void destroy(pointer p) { p->~value_type(); } + + template + libc_allocator_with_realloc(const libc_allocator_with_realloc&) {} + + template + struct rebind { + typedef libc_allocator_with_realloc other; + }; +}; + +// libc_allocator_with_realloc specialization. +template<> +class libc_allocator_with_realloc { + public: + typedef void value_type; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef void* pointer; + typedef const void* const_pointer; + + template + struct rebind { + typedef libc_allocator_with_realloc other; + }; +}; + +template +inline bool operator==(const libc_allocator_with_realloc&, + const libc_allocator_with_realloc&) { + return true; +} + +template +inline bool operator!=(const libc_allocator_with_realloc&, + const libc_allocator_with_realloc&) { + return false; +} + +_END_GOOGLE_NAMESPACE_ + +#endif // UTIL_GTL_LIBC_ALLOCATOR_WITH_REALLOC_H_ diff --git a/external/google/sparsehash/os_config.h b/external/google/sparsehash/os_config.h new file mode 100644 index 00000000..0a397192 --- /dev/null +++ b/external/google/sparsehash/os_config.h @@ -0,0 +1,37 @@ +#ifndef _MSC_VER +//non-win version + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if the system has the type `uint16_t'. */ +#define HAVE_UINT16_T 1 + +/* Define to 1 if the system has the type `u_int16_t'. */ +#define HAVE_U_INT16_T 1 + +/* Define to 1 if the system has the type `__uint16'. */ +/* #undef HAVE___UINT16 */ + +#else +//win version + +/* Define to 1 if you have the header file. */ +#undef HAVE_INTTYPES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDINT_H + +/* Define to 1 if the system has the type `uint16_t'. */ +#undef HAVE_UINT16_T + +/* Define to 1 if the system has the type `u_int16_t'. */ +#undef HAVE_U_INT16_T + +/* Define to 1 if the system has the type `__uint16'. */ +#define HAVE___UINT16 1 + +#endif diff --git a/external/google/sparsehash/sparseconfig.h b/external/google/sparsehash/sparseconfig.h new file mode 100644 index 00000000..fbfa328a --- /dev/null +++ b/external/google/sparsehash/sparseconfig.h @@ -0,0 +1,39 @@ +/* + * NOTE: This file is for internal use only. + * Do not use these #defines in your own program! + */ + +/* Namespace for Google classes */ +#define GOOGLE_NAMESPACE ::google + +/* the location of the header defining hash functions */ +#define HASH_FUN_H + +/* the namespace of the hash<> function */ +#define HASH_NAMESPACE std + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the `memcpy' function. */ +#define HAVE_MEMCPY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* The system-provided hash function including the namespace. */ +#define SPARSEHASH_HASH HASH_NAMESPACE::hash + +/* The system-provided hash function, in namespace HASH_NAMESPACE. */ +#define SPARSEHASH_HASH_NO_NAMESPACE hash + +/* the namespace where STL code like vector<> is defined */ +#define STL_NAMESPACE std + +/* Stops putting the code inside the Google namespace */ +#define _END_GOOGLE_NAMESPACE_ } + +/* Puts following code inside the Google namespace */ +#define _START_GOOGLE_NAMESPACE_ namespace google { + +#include "os_config.h" diff --git a/external/google/sparsehash/sparseconfig_win.h b/external/google/sparsehash/sparseconfig_win.h new file mode 100644 index 00000000..a4607139 --- /dev/null +++ b/external/google/sparsehash/sparseconfig_win.h @@ -0,0 +1,37 @@ +/* + * NOTE: This file is for internal use only. + * Do not use these #defines in your own program! + */ + +/* Namespace for Google classes */ +#define GOOGLE_NAMESPACE ::google + +/* the location of the header defining hash functions */ +#define HASH_FUN_H + +/* the namespace of the hash<> function */ +#define HASH_NAMESPACE std + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the `memcpy' function. */ +#define HAVE_MEMCPY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* The system-provided hash function including the namespace. */ +#define SPARSEHASH_HASH HASH_NAMESPACE::hash + +/* The system-provided hash function, in namespace HASH_NAMESPACE. */ +#define SPARSEHASH_HASH_NO_NAMESPACE hash + +/* the namespace where STL code like vector<> is defined */ +#define STL_NAMESPACE std + +/* Stops putting the code inside the Google namespace */ +#define _END_GOOGLE_NAMESPACE_ } + +/* Puts following code inside the Google namespace */ +#define _START_GOOGLE_NAMESPACE_ namespace google { diff --git a/external/google/sparsehash/sparsehashtable.h b/external/google/sparsehash/sparsehashtable.h new file mode 100644 index 00000000..9f2d1443 --- /dev/null +++ b/external/google/sparsehash/sparsehashtable.h @@ -0,0 +1,1190 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein +// +// A sparse hashtable is a particular implementation of +// a hashtable: one that is meant to minimize memory use. +// It does this by using a *sparse table* (cf sparsetable.h), +// which uses between 1 and 2 bits to store empty buckets +// (we may need another bit for hashtables that support deletion). +// +// When empty buckets are so cheap, an appealing hashtable +// implementation is internal probing, in which the hashtable +// is a single table, and collisions are resolved by trying +// to insert again in another bucket. The most cache-efficient +// internal probing schemes are linear probing (which suffers, +// alas, from clumping) and quadratic probing, which is what +// we implement by default. +// +// Deleted buckets are a bit of a pain. We have to somehow mark +// deleted buckets (the probing must distinguish them from empty +// buckets). The most principled way is to have another bitmap, +// but that's annoying and takes up space. Instead we let the +// user specify an "impossible" key. We set deleted buckets +// to have the impossible key. +// +// Note it is possible to change the value of the delete key +// on the fly; you can even remove it, though after that point +// the hashtable is insert_only until you set it again. +// +// You probably shouldn't use this code directly. Use +// or instead. +// +// You can modify the following, below: +// HT_OCCUPANCY_PCT -- how full before we double size +// HT_EMPTY_PCT -- how empty before we halve size +// HT_MIN_BUCKETS -- smallest bucket size +// HT_DEFAULT_STARTING_BUCKETS -- default bucket size at construct-time +// +// You can also change enlarge_factor (which defaults to +// HT_OCCUPANCY_PCT), and shrink_factor (which defaults to +// HT_EMPTY_PCT) with set_resizing_parameters(). +// +// How to decide what values to use? +// shrink_factor's default of .4 * OCCUPANCY_PCT, is probably good. +// HT_MIN_BUCKETS is probably unnecessary since you can specify +// (indirectly) the starting number of buckets at construct-time. +// For enlarge_factor, you can use this chart to try to trade-off +// expected lookup time to the space taken up. By default, this +// code uses quadratic probing, though you can change it to linear +// via _JUMP below if you really want to. +// +// From http://www.augustana.ca/~mohrj/courses/1999.fall/csc210/lecture_notes/hashing.html +// NUMBER OF PROBES / LOOKUP Successful Unsuccessful +// Quadratic collision resolution 1 - ln(1-L) - L/2 1/(1-L) - L - ln(1-L) +// Linear collision resolution [1+1/(1-L)]/2 [1+1/(1-L)2]/2 +// +// -- enlarge_factor -- 0.10 0.50 0.60 0.75 0.80 0.90 0.99 +// QUADRATIC COLLISION RES. +// probes/successful lookup 1.05 1.44 1.62 2.01 2.21 2.85 5.11 +// probes/unsuccessful lookup 1.11 2.19 2.82 4.64 5.81 11.4 103.6 +// LINEAR COLLISION RES. +// probes/successful lookup 1.06 1.5 1.75 2.5 3.0 5.5 50.5 +// probes/unsuccessful lookup 1.12 2.5 3.6 8.5 13.0 50.0 5000.0 +// +// The value type is required to be copy constructible and default +// constructible, but it need not be (and commonly isn't) assignable. + +#ifndef _SPARSEHASHTABLE_H_ +#define _SPARSEHASHTABLE_H_ + +#ifndef SPARSEHASH_STAT_UPDATE +#define SPARSEHASH_STAT_UPDATE(x) ((void) 0) +#endif + +// The probing method +// Linear probing +// #define JUMP_(key, num_probes) ( 1 ) +// Quadratic probing +#define JUMP_(key, num_probes) ( num_probes ) + +#include +#include +#include // For swap(), eg +#include // For length_error +#include // for facts about iterator tags +#include // for numeric_limits<> +#include // for pair<> +#include +#include // Since that's basically what we are + +_START_GOOGLE_NAMESPACE_ + +using STL_NAMESPACE::pair; + +// The smaller this is, the faster lookup is (because the group bitmap is +// smaller) and the faster insert is, because there's less to move. +// On the other hand, there are more groups. Since group::size_type is +// a short, this number should be of the form 32*x + 16 to avoid waste. +static const u_int16_t DEFAULT_GROUP_SIZE = 48; // fits in 1.5 words + +// Hashtable class, used to implement the hashed associative containers +// hash_set and hash_map. +// +// Value: what is stored in the table (each bucket is a Value). +// Key: something in a 1-to-1 correspondence to a Value, that can be used +// to search for a Value in the table (find() takes a Key). +// HashFcn: Takes a Key and returns an integer, the more unique the better. +// ExtractKey: given a Value, returns the unique Key associated with it. +// Must inherit from unary_function, or at least have a +// result_type enum indicating the return type of operator(). +// SetKey: given a Value* and a Key, modifies the value such that +// ExtractKey(value) == key. We guarantee this is only called +// with key == deleted_key. +// EqualKey: Given two Keys, says whether they are the same (that is, +// if they are both associated with the same Value). +// Alloc: STL allocator to use to allocate memory. + +template +class sparse_hashtable; + +template +struct sparse_hashtable_iterator; + +template +struct sparse_hashtable_const_iterator; + +// As far as iterating, we're basically just a sparsetable +// that skips over deleted elements. +template +struct sparse_hashtable_iterator { + private: + typedef typename A::template rebind::other value_alloc_type; + + public: + typedef sparse_hashtable_iterator iterator; + typedef sparse_hashtable_const_iterator const_iterator; + typedef typename sparsetable::nonempty_iterator + st_iterator; + + typedef STL_NAMESPACE::forward_iterator_tag iterator_category; + typedef V value_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::pointer pointer; + + // "Real" constructor and default constructor + sparse_hashtable_iterator(const sparse_hashtable *h, + st_iterator it, st_iterator it_end) + : ht(h), pos(it), end(it_end) { advance_past_deleted(); } + sparse_hashtable_iterator() { } // not ever used internally + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *pos; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic. The only hard part is making sure that + // we're not on a marked-deleted array element + void advance_past_deleted() { + while ( pos != end && ht->test_deleted(*this) ) + ++pos; + } + iterator& operator++() { + assert(pos != end); ++pos; advance_past_deleted(); return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + + // Comparison. + bool operator==(const iterator& it) const { return pos == it.pos; } + bool operator!=(const iterator& it) const { return pos != it.pos; } + + + // The actual data + const sparse_hashtable *ht; + st_iterator pos, end; +}; + +// Now do it all again, but with const-ness! +template +struct sparse_hashtable_const_iterator { + private: + typedef typename A::template rebind::other value_alloc_type; + + public: + typedef sparse_hashtable_iterator iterator; + typedef sparse_hashtable_const_iterator const_iterator; + typedef typename sparsetable::const_nonempty_iterator + st_iterator; + + typedef STL_NAMESPACE::forward_iterator_tag iterator_category; + typedef V value_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::const_reference reference; + typedef typename value_alloc_type::const_pointer pointer; + + // "Real" constructor and default constructor + sparse_hashtable_const_iterator(const sparse_hashtable *h, + st_iterator it, st_iterator it_end) + : ht(h), pos(it), end(it_end) { advance_past_deleted(); } + // This lets us convert regular iterators to const iterators + sparse_hashtable_const_iterator() { } // never used internally + sparse_hashtable_const_iterator(const iterator &it) + : ht(it.ht), pos(it.pos), end(it.end) { } + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *pos; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic. The only hard part is making sure that + // we're not on a marked-deleted array element + void advance_past_deleted() { + while ( pos != end && ht->test_deleted(*this) ) + ++pos; + } + const_iterator& operator++() { + assert(pos != end); ++pos; advance_past_deleted(); return *this; + } + const_iterator operator++(int) { const_iterator tmp(*this); ++*this; return tmp; } + + // Comparison. + bool operator==(const const_iterator& it) const { return pos == it.pos; } + bool operator!=(const const_iterator& it) const { return pos != it.pos; } + + + // The actual data + const sparse_hashtable *ht; + st_iterator pos, end; +}; + +// And once again, but this time freeing up memory as we iterate +template +struct sparse_hashtable_destructive_iterator { + private: + typedef typename A::template rebind::other value_alloc_type; + + public: + typedef sparse_hashtable_destructive_iterator iterator; + typedef typename sparsetable::destructive_iterator + st_iterator; + + typedef STL_NAMESPACE::forward_iterator_tag iterator_category; + typedef V value_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::pointer pointer; + + // "Real" constructor and default constructor + sparse_hashtable_destructive_iterator(const + sparse_hashtable *h, + st_iterator it, st_iterator it_end) + : ht(h), pos(it), end(it_end) { advance_past_deleted(); } + sparse_hashtable_destructive_iterator() { } // never used internally + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *pos; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic. The only hard part is making sure that + // we're not on a marked-deleted array element + void advance_past_deleted() { + while ( pos != end && ht->test_deleted(*this) ) + ++pos; + } + iterator& operator++() { + assert(pos != end); ++pos; advance_past_deleted(); return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + + // Comparison. + bool operator==(const iterator& it) const { return pos == it.pos; } + bool operator!=(const iterator& it) const { return pos != it.pos; } + + + // The actual data + const sparse_hashtable *ht; + st_iterator pos, end; +}; + + +template +class sparse_hashtable { + private: + typedef typename Alloc::template rebind::other value_alloc_type; + + public: + typedef Key key_type; + typedef Value value_type; + typedef HashFcn hasher; + typedef EqualKey key_equal; + typedef Alloc allocator_type; + + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::const_reference const_reference; + typedef typename value_alloc_type::pointer pointer; + typedef typename value_alloc_type::const_pointer const_pointer; + typedef sparse_hashtable_iterator + iterator; + + typedef sparse_hashtable_const_iterator + const_iterator; + + typedef sparse_hashtable_destructive_iterator + destructive_iterator; + + // These come from tr1. For us they're the same as regular iterators. + typedef iterator local_iterator; + typedef const_iterator const_local_iterator; + + // How full we let the table get before we resize, by default. + // Knuth says .8 is good -- higher causes us to probe too much, + // though it saves memory. + static const int HT_OCCUPANCY_PCT; // = 80 (out of 100); + + // How empty we let the table get before we resize lower, by default. + // (0.0 means never resize lower.) + // It should be less than OCCUPANCY_PCT / 2 or we thrash resizing + static const int HT_EMPTY_PCT; // = 0.4 * HT_OCCUPANCY_PCT; + + // Minimum size we're willing to let hashtables be. + // Must be a power of two, and at least 4. + // Note, however, that for a given hashtable, the initial size is a + // function of the first constructor arg, and may be >HT_MIN_BUCKETS. + static const size_type HT_MIN_BUCKETS = 4; + + // By default, if you don't specify a hashtable size at + // construction-time, we use this size. Must be a power of two, and + // at least HT_MIN_BUCKETS. + static const size_type HT_DEFAULT_STARTING_BUCKETS = 32; + + // ITERATOR FUNCTIONS + iterator begin() { return iterator(this, table.nonempty_begin(), + table.nonempty_end()); } + iterator end() { return iterator(this, table.nonempty_end(), + table.nonempty_end()); } + const_iterator begin() const { return const_iterator(this, + table.nonempty_begin(), + table.nonempty_end()); } + const_iterator end() const { return const_iterator(this, + table.nonempty_end(), + table.nonempty_end()); } + + // These come from tr1 unordered_map. They iterate over 'bucket' n. + // For sparsehashtable, we could consider each 'group' to be a bucket, + // I guess, but I don't really see the point. We'll just consider + // bucket n to be the n-th element of the sparsetable, if it's occupied, + // or some empty element, otherwise. + local_iterator begin(size_type i) { + if (table.test(i)) + return local_iterator(this, table.get_iter(i), table.nonempty_end()); + else + return local_iterator(this, table.nonempty_end(), table.nonempty_end()); + } + local_iterator end(size_type i) { + local_iterator it = begin(i); + if (table.test(i) && !test_deleted(i)) + ++it; + return it; + } + const_local_iterator begin(size_type i) const { + if (table.test(i)) + return const_local_iterator(this, table.get_iter(i), + table.nonempty_end()); + else + return const_local_iterator(this, table.nonempty_end(), + table.nonempty_end()); + } + const_local_iterator end(size_type i) const { + const_local_iterator it = begin(i); + if (table.test(i) && !test_deleted(i)) + ++it; + return it; + } + + // This is used when resizing + destructive_iterator destructive_begin() { + return destructive_iterator(this, table.destructive_begin(), + table.destructive_end()); + } + destructive_iterator destructive_end() { + return destructive_iterator(this, table.destructive_end(), + table.destructive_end()); + } + + + // ACCESSOR FUNCTIONS for the things we templatize on, basically + hasher hash_funct() const { return settings; } + key_equal key_eq() const { return key_info; } + allocator_type get_allocator() const { return table.get_allocator(); } + + // Accessor function for statistics gathering. + int num_table_copies() const { return settings.num_ht_copies(); } + + private: + // We need to copy values when we set the special marker for deleted + // elements, but, annoyingly, we can't just use the copy assignment + // operator because value_type might not be assignable (it's often + // pair). We use explicit destructor invocation and + // placement new to get around this. Arg. + void set_value(pointer dst, const_reference src) { + dst->~value_type(); // delete the old value, if any + new(dst) value_type(src); + } + + // This is used as a tag for the copy constructor, saying to destroy its + // arg We have two ways of destructively copying: with potentially growing + // the hashtable as we copy, and without. To make sure the outside world + // can't do a destructive copy, we make the typename private. + enum MoveDontCopyT {MoveDontCopy, MoveDontGrow}; + + // DELETE HELPER FUNCTIONS + // This lets the user describe a key that will indicate deleted + // table entries. This key should be an "impossible" entry -- + // if you try to insert it for real, you won't be able to retrieve it! + // (NB: while you pass in an entire value, only the key part is looked + // at. This is just because I don't know how to assign just a key.) + private: + void squash_deleted() { // gets rid of any deleted entries we have + if ( num_deleted ) { // get rid of deleted before writing + sparse_hashtable tmp(MoveDontGrow, *this); + swap(tmp); // now we are tmp + } + assert(num_deleted == 0); + } + + bool test_deleted_key(const key_type& key) const { + // The num_deleted test is crucial for read(): after read(), the ht values + // are garbage, and we don't want to think some of them are deleted. + // Invariant: !use_deleted implies num_deleted is 0. + assert(settings.use_deleted() || num_deleted == 0); + return num_deleted > 0 && equals(key_info.delkey, key); + } + + public: + void set_deleted_key(const key_type &key) { + // It's only safe to change what "deleted" means if we purge deleted guys + squash_deleted(); + settings.set_use_deleted(true); + key_info.delkey = key; + } + void clear_deleted_key() { + squash_deleted(); + settings.set_use_deleted(false); + } + key_type deleted_key() const { + assert(settings.use_deleted() + && "Must set deleted key before calling deleted_key"); + return key_info.delkey; + } + + // These are public so the iterators can use them + // True if the item at position bucknum is "deleted" marker + bool test_deleted(size_type bucknum) const { + if (num_deleted == 0 || !table.test(bucknum)) return false; + return test_deleted_key(get_key(table.unsafe_get(bucknum))); + } + bool test_deleted(const iterator &it) const { + if (!settings.use_deleted()) return false; + return test_deleted_key(get_key(*it)); + } + bool test_deleted(const const_iterator &it) const { + if (!settings.use_deleted()) return false; + return test_deleted_key(get_key(*it)); + } + bool test_deleted(const destructive_iterator &it) const { + if (!settings.use_deleted()) return false; + return test_deleted_key(get_key(*it)); + } + + private: + // Set it so test_deleted is true. true if object didn't used to be deleted. + // TODO(csilvers): make these private (also in densehashtable.h) + bool set_deleted(iterator &it) { + assert(settings.use_deleted()); + bool retval = !test_deleted(it); + // &* converts from iterator to value-type. + set_key(&(*it), key_info.delkey); + return retval; + } + // Set it so test_deleted is false. true if object used to be deleted. + bool clear_deleted(iterator &it) { + assert(settings.use_deleted()); + // Happens automatically when we assign something else in its place. + return test_deleted(it); + } + + // We also allow to set/clear the deleted bit on a const iterator. + // We allow a const_iterator for the same reason you can delete a + // const pointer: it's convenient, and semantically you can't use + // 'it' after it's been deleted anyway, so its const-ness doesn't + // really matter. + bool set_deleted(const_iterator &it) { + assert(settings.use_deleted()); // bad if set_deleted_key() wasn't called + bool retval = !test_deleted(it); + set_key(const_cast(&(*it)), key_info.delkey); + return retval; + } + // Set it so test_deleted is false. true if object used to be deleted. + bool clear_deleted(const_iterator &it) { + assert(settings.use_deleted()); // bad if set_deleted_key() wasn't called + return test_deleted(it); + } + + // FUNCTIONS CONCERNING SIZE + public: + size_type size() const { return table.num_nonempty() - num_deleted; } + size_type max_size() const { return table.max_size(); } + bool empty() const { return size() == 0; } + size_type bucket_count() const { return table.size(); } + size_type max_bucket_count() const { return max_size(); } + // These are tr1 methods. Their idea of 'bucket' doesn't map well to + // what we do. We just say every bucket has 0 or 1 items in it. + size_type bucket_size(size_type i) const { + return begin(i) == end(i) ? 0 : 1; + } + + private: + // Because of the above, size_type(-1) is never legal; use it for errors + static const size_type ILLEGAL_BUCKET = size_type(-1); + + // Used after a string of deletes. Returns true if we actually shrunk. + // TODO(csilvers): take a delta so we can take into account inserts + // done after shrinking. Maybe make part of the Settings class? + bool maybe_shrink() { + assert(table.num_nonempty() >= num_deleted); + assert((bucket_count() & (bucket_count()-1)) == 0); // is a power of two + assert(bucket_count() >= HT_MIN_BUCKETS); + bool retval = false; + + // If you construct a hashtable with < HT_DEFAULT_STARTING_BUCKETS, + // we'll never shrink until you get relatively big, and we'll never + // shrink below HT_DEFAULT_STARTING_BUCKETS. Otherwise, something + // like "dense_hash_set x; x.insert(4); x.erase(4);" will + // shrink us down to HT_MIN_BUCKETS buckets, which is too small. + const size_type num_remain = table.num_nonempty() - num_deleted; + const size_type shrink_threshold = settings.shrink_threshold(); + if (shrink_threshold > 0 && num_remain < shrink_threshold && + bucket_count() > HT_DEFAULT_STARTING_BUCKETS) { + const float shrink_factor = settings.shrink_factor(); + size_type sz = bucket_count() / 2; // find how much we should shrink + while (sz > HT_DEFAULT_STARTING_BUCKETS && + num_remain < static_cast(sz * shrink_factor)) { + sz /= 2; // stay a power of 2 + } + sparse_hashtable tmp(MoveDontCopy, *this, sz); + swap(tmp); // now we are tmp + retval = true; + } + settings.set_consider_shrink(false); // because we just considered it + return retval; + } + + // We'll let you resize a hashtable -- though this makes us copy all! + // When you resize, you say, "make it big enough for this many more elements" + // Returns true if we actually resized, false if size was already ok. + bool resize_delta(size_type delta) { + bool did_resize = false; + if ( settings.consider_shrink() ) { // see if lots of deletes happened + if ( maybe_shrink() ) + did_resize = true; + } + if (table.num_nonempty() >= + (STL_NAMESPACE::numeric_limits::max)() - delta) + throw std::length_error("resize overflow"); + if ( bucket_count() >= HT_MIN_BUCKETS && + (table.num_nonempty() + delta) <= settings.enlarge_threshold() ) + return did_resize; // we're ok as we are + + // Sometimes, we need to resize just to get rid of all the + // "deleted" buckets that are clogging up the hashtable. So when + // deciding whether to resize, count the deleted buckets (which + // are currently taking up room). But later, when we decide what + // size to resize to, *don't* count deleted buckets, since they + // get discarded during the resize. + const size_type needed_size = + settings.min_buckets(table.num_nonempty() + delta, 0); + if ( needed_size <= bucket_count() ) // we have enough buckets + return did_resize; + + size_type resize_to = + settings.min_buckets(table.num_nonempty() - num_deleted + delta, + bucket_count()); + if (resize_to < needed_size && // may double resize_to + resize_to < (STL_NAMESPACE::numeric_limits::max)() / 2) { + // This situation means that we have enough deleted elements, + // that once we purge them, we won't actually have needed to + // grow. But we may want to grow anyway: if we just purge one + // element, say, we'll have to grow anyway next time we + // insert. Might as well grow now, since we're already going + // through the trouble of copying (in order to purge the + // deleted elements). + const size_type target = + static_cast(settings.shrink_size(resize_to*2)); + if (table.num_nonempty() - num_deleted + delta >= target) { + // Good, we won't be below the shrink threshhold even if we double. + resize_to *= 2; + } + } + + sparse_hashtable tmp(MoveDontCopy, *this, resize_to); + swap(tmp); // now we are tmp + return true; + } + + // Used to actually do the rehashing when we grow/shrink a hashtable + void copy_from(const sparse_hashtable &ht, size_type min_buckets_wanted) { + clear(); // clear table, set num_deleted to 0 + + // If we need to change the size of our table, do it now + const size_type resize_to = + settings.min_buckets(ht.size(), min_buckets_wanted); + if ( resize_to > bucket_count() ) { // we don't have enough buckets + table.resize(resize_to); // sets the number of buckets + settings.reset_thresholds(bucket_count()); + } + + // We use a normal iterator to get non-deleted bcks from ht + // We could use insert() here, but since we know there are + // no duplicates and no deleted items, we can be more efficient + assert((bucket_count() & (bucket_count()-1)) == 0); // a power of two + for ( const_iterator it = ht.begin(); it != ht.end(); ++it ) { + size_type num_probes = 0; // how many times we've probed + size_type bucknum; + const size_type bucket_count_minus_one = bucket_count() - 1; + for (bucknum = hash(get_key(*it)) & bucket_count_minus_one; + table.test(bucknum); // not empty + bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one) { + ++num_probes; + assert(num_probes < bucket_count() + && "Hashtable is full: an error in key_equal<> or hash<>"); + } + table.set(bucknum, *it); // copies the value to here + } + settings.inc_num_ht_copies(); + } + + // Implementation is like copy_from, but it destroys the table of the + // "from" guy by freeing sparsetable memory as we iterate. This is + // useful in resizing, since we're throwing away the "from" guy anyway. + void move_from(MoveDontCopyT mover, sparse_hashtable &ht, + size_type min_buckets_wanted) { + clear(); // clear table, set num_deleted to 0 + + // If we need to change the size of our table, do it now + size_type resize_to; + if ( mover == MoveDontGrow ) + resize_to = ht.bucket_count(); // keep same size as old ht + else // MoveDontCopy + resize_to = settings.min_buckets(ht.size(), min_buckets_wanted); + if ( resize_to > bucket_count() ) { // we don't have enough buckets + table.resize(resize_to); // sets the number of buckets + settings.reset_thresholds(bucket_count()); + } + + // We use a normal iterator to get non-deleted bcks from ht + // We could use insert() here, but since we know there are + // no duplicates and no deleted items, we can be more efficient + assert( (bucket_count() & (bucket_count()-1)) == 0); // a power of two + // THIS IS THE MAJOR LINE THAT DIFFERS FROM COPY_FROM(): + for ( destructive_iterator it = ht.destructive_begin(); + it != ht.destructive_end(); ++it ) { + size_type num_probes = 0; // how many times we've probed + size_type bucknum; + for ( bucknum = hash(get_key(*it)) & (bucket_count()-1); // h % buck_cnt + table.test(bucknum); // not empty + bucknum = (bucknum + JUMP_(key, num_probes)) & (bucket_count()-1) ) { + ++num_probes; + assert(num_probes < bucket_count() + && "Hashtable is full: an error in key_equal<> or hash<>"); + } + table.set(bucknum, *it); // copies the value to here + } + settings.inc_num_ht_copies(); + } + + + // Required by the spec for hashed associative container + public: + // Though the docs say this should be num_buckets, I think it's much + // more useful as num_elements. As a special feature, calling with + // req_elements==0 will cause us to shrink if we can, saving space. + void resize(size_type req_elements) { // resize to this or larger + if ( settings.consider_shrink() || req_elements == 0 ) + maybe_shrink(); + if ( req_elements > table.num_nonempty() ) // we only grow + resize_delta(req_elements - table.num_nonempty()); + } + + // Get and change the value of shrink_factor and enlarge_factor. The + // description at the beginning of this file explains how to choose + // the values. Setting the shrink parameter to 0.0 ensures that the + // table never shrinks. + void get_resizing_parameters(float* shrink, float* grow) const { + *shrink = settings.shrink_factor(); + *grow = settings.enlarge_factor(); + } + void set_resizing_parameters(float shrink, float grow) { + settings.set_resizing_parameters(shrink, grow); + settings.reset_thresholds(bucket_count()); + } + + // CONSTRUCTORS -- as required by the specs, we take a size, + // but also let you specify a hashfunction, key comparator, + // and key extractor. We also define a copy constructor and =. + // DESTRUCTOR -- the default is fine, surprisingly. + explicit sparse_hashtable(size_type expected_max_items_in_table = 0, + const HashFcn& hf = HashFcn(), + const EqualKey& eql = EqualKey(), + const ExtractKey& ext = ExtractKey(), + const SetKey& set = SetKey(), + const Alloc& alloc = Alloc()) + : settings(hf), + key_info(ext, set, eql), + num_deleted(0), + table((expected_max_items_in_table == 0 + ? HT_DEFAULT_STARTING_BUCKETS + : settings.min_buckets(expected_max_items_in_table, 0)), + alloc) { + settings.reset_thresholds(bucket_count()); + } + + // As a convenience for resize(), we allow an optional second argument + // which lets you make this new hashtable a different size than ht. + // We also provide a mechanism of saying you want to "move" the ht argument + // into us instead of copying. + sparse_hashtable(const sparse_hashtable& ht, + size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS) + : settings(ht.settings), + key_info(ht.key_info), + num_deleted(0), + table(0, ht.get_allocator()) { + settings.reset_thresholds(bucket_count()); + copy_from(ht, min_buckets_wanted); // copy_from() ignores deleted entries + } + sparse_hashtable(MoveDontCopyT mover, sparse_hashtable& ht, + size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS) + : settings(ht.settings), + key_info(ht.key_info), + num_deleted(0), + table(0, ht.get_allocator()) { + settings.reset_thresholds(bucket_count()); + move_from(mover, ht, min_buckets_wanted); // ignores deleted entries + } + + sparse_hashtable& operator= (const sparse_hashtable& ht) { + if (&ht == this) return *this; // don't copy onto ourselves + settings = ht.settings; + key_info = ht.key_info; + num_deleted = ht.num_deleted; + // copy_from() calls clear and sets num_deleted to 0 too + copy_from(ht, HT_MIN_BUCKETS); + // we purposefully don't copy the allocator, which may not be copyable + return *this; + } + + // Many STL algorithms use swap instead of copy constructors + void swap(sparse_hashtable& ht) { + STL_NAMESPACE::swap(settings, ht.settings); + STL_NAMESPACE::swap(key_info, ht.key_info); + STL_NAMESPACE::swap(num_deleted, ht.num_deleted); + table.swap(ht.table); + } + + // It's always nice to be able to clear a table without deallocating it + void clear() { + if (!empty() || (num_deleted != 0)) { + table.clear(); + } + settings.reset_thresholds(bucket_count()); + num_deleted = 0; + } + + // LOOKUP ROUTINES + private: + // Returns a pair of positions: 1st where the object is, 2nd where + // it would go if you wanted to insert it. 1st is ILLEGAL_BUCKET + // if object is not found; 2nd is ILLEGAL_BUCKET if it is. + // Note: because of deletions where-to-insert is not trivial: it's the + // first deleted bucket we see, as long as we don't find the key later + pair find_position(const key_type &key) const { + size_type num_probes = 0; // how many times we've probed + const size_type bucket_count_minus_one = bucket_count() - 1; + size_type bucknum = hash(key) & bucket_count_minus_one; + size_type insert_pos = ILLEGAL_BUCKET; // where we would insert + SPARSEHASH_STAT_UPDATE(total_lookups += 1); + while ( 1 ) { // probe until something happens + if ( !table.test(bucknum) ) { // bucket is empty + SPARSEHASH_STAT_UPDATE(total_probes += num_probes); + if ( insert_pos == ILLEGAL_BUCKET ) // found no prior place to insert + return pair(ILLEGAL_BUCKET, bucknum); + else + return pair(ILLEGAL_BUCKET, insert_pos); + + } else if ( test_deleted(bucknum) ) {// keep searching, but mark to insert + if ( insert_pos == ILLEGAL_BUCKET ) + insert_pos = bucknum; + + } else if ( equals(key, get_key(table.unsafe_get(bucknum))) ) { + SPARSEHASH_STAT_UPDATE(total_probes += num_probes); + return pair(bucknum, ILLEGAL_BUCKET); + } + ++num_probes; // we're doing another probe + bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one; + assert(num_probes < bucket_count() + && "Hashtable is full: an error in key_equal<> or hash<>"); + } + } + + public: + iterator find(const key_type& key) { + if ( size() == 0 ) return end(); + pair pos = find_position(key); + if ( pos.first == ILLEGAL_BUCKET ) // alas, not there + return end(); + else + return iterator(this, table.get_iter(pos.first), table.nonempty_end()); + } + + const_iterator find(const key_type& key) const { + if ( size() == 0 ) return end(); + pair pos = find_position(key); + if ( pos.first == ILLEGAL_BUCKET ) // alas, not there + return end(); + else + return const_iterator(this, + table.get_iter(pos.first), table.nonempty_end()); + } + + // This is a tr1 method: the bucket a given key is in, or what bucket + // it would be put in, if it were to be inserted. Shrug. + size_type bucket(const key_type& key) const { + pair pos = find_position(key); + return pos.first == ILLEGAL_BUCKET ? pos.second : pos.first; + } + + // Counts how many elements have key key. For maps, it's either 0 or 1. + size_type count(const key_type &key) const { + pair pos = find_position(key); + return pos.first == ILLEGAL_BUCKET ? 0 : 1; + } + + // Likewise, equal_range doesn't really make sense for us. Oh well. + pair equal_range(const key_type& key) { + iterator pos = find(key); // either an iterator or end + if (pos == end()) { + return pair(pos, pos); + } else { + const iterator startpos = pos++; + return pair(startpos, pos); + } + } + pair equal_range(const key_type& key) const { + const_iterator pos = find(key); // either an iterator or end + if (pos == end()) { + return pair(pos, pos); + } else { + const const_iterator startpos = pos++; + return pair(startpos, pos); + } + } + + + // INSERTION ROUTINES + private: + // Private method used by insert_noresize and find_or_insert. + iterator insert_at(const_reference obj, size_type pos) { + if (size() >= max_size()) + throw std::length_error("insert overflow"); + if ( test_deleted(pos) ) { // just replace if it's been deleted + // The set() below will undelete this object. We just worry about stats + assert(num_deleted > 0); + --num_deleted; // used to be, now it isn't + } + table.set(pos, obj); + return iterator(this, table.get_iter(pos), table.nonempty_end()); + } + + // If you know *this is big enough to hold obj, use this routine + pair insert_noresize(const_reference obj) { + // First, double-check we're not inserting delkey + assert((!settings.use_deleted() || !equals(get_key(obj), key_info.delkey)) + && "Inserting the deleted key"); + const pair pos = find_position(get_key(obj)); + if ( pos.first != ILLEGAL_BUCKET) { // object was already there + return pair(iterator(this, table.get_iter(pos.first), + table.nonempty_end()), + false); // false: we didn't insert + } else { // pos.second says where to put it + return pair(insert_at(obj, pos.second), true); + } + } + + // Specializations of insert(it, it) depending on the power of the iterator: + // (1) Iterator supports operator-, resize before inserting + template + void insert(ForwardIterator f, ForwardIterator l, STL_NAMESPACE::forward_iterator_tag) { + size_t dist = STL_NAMESPACE::distance(f, l); + if (dist >= (std::numeric_limits::max)()) + throw std::length_error("insert-range overflow"); + resize_delta(static_cast(dist)); + for ( ; dist > 0; --dist, ++f) { + insert_noresize(*f); + } + } + + // (2) Arbitrary iterator, can't tell how much to resize + template + void insert(InputIterator f, InputIterator l, STL_NAMESPACE::input_iterator_tag) { + for ( ; f != l; ++f) + insert(*f); + } + + public: + // This is the normal insert routine, used by the outside world + pair insert(const_reference obj) { + resize_delta(1); // adding an object, grow if need be + return insert_noresize(obj); + } + + // When inserting a lot at a time, we specialize on the type of iterator + template + void insert(InputIterator f, InputIterator l) { + // specializes on iterator type + insert(f, l, typename STL_NAMESPACE::iterator_traits::iterator_category()); + } + + // DefaultValue is a functor that takes a key and returns a value_type + // representing the default value to be inserted if none is found. + template + value_type& find_or_insert(const key_type& key) { + // First, double-check we're not inserting delkey + assert((!settings.use_deleted() || !equals(key, key_info.delkey)) + && "Inserting the deleted key"); + const pair pos = find_position(key); + DefaultValue default_value; + if ( pos.first != ILLEGAL_BUCKET) { // object was already there + return *table.get_iter(pos.first); + } else if (resize_delta(1)) { // needed to rehash to make room + // Since we resized, we can't use pos, so recalculate where to insert. + return *insert_noresize(default_value(key)).first; + } else { // no need to rehash, insert right here + return *insert_at(default_value(key), pos.second); + } + } + + // DELETION ROUTINES + size_type erase(const key_type& key) { + // First, double-check we're not erasing delkey. + assert((!settings.use_deleted() || !equals(key, key_info.delkey)) + && "Erasing the deleted key"); + assert(!settings.use_deleted() || !equals(key, key_info.delkey)); + const_iterator pos = find(key); // shrug: shouldn't need to be const + if ( pos != end() ) { + assert(!test_deleted(pos)); // or find() shouldn't have returned it + set_deleted(pos); + ++num_deleted; + // will think about shrink after next insert + settings.set_consider_shrink(true); + return 1; // because we deleted one thing + } else { + return 0; // because we deleted nothing + } + } + + // We return the iterator past the deleted item. + void erase(iterator pos) { + if ( pos == end() ) return; // sanity check + if ( set_deleted(pos) ) { // true if object has been newly deleted + ++num_deleted; + // will think about shrink after next insert + settings.set_consider_shrink(true); + } + } + + void erase(iterator f, iterator l) { + for ( ; f != l; ++f) { + if ( set_deleted(f) ) // should always be true + ++num_deleted; + } + // will think about shrink after next insert + settings.set_consider_shrink(true); + } + + // We allow you to erase a const_iterator just like we allow you to + // erase an iterator. This is in parallel to 'delete': you can delete + // a const pointer just like a non-const pointer. The logic is that + // you can't use the object after it's erased anyway, so it doesn't matter + // if it's const or not. + void erase(const_iterator pos) { + if ( pos == end() ) return; // sanity check + if ( set_deleted(pos) ) { // true if object has been newly deleted + ++num_deleted; + // will think about shrink after next insert + settings.set_consider_shrink(true); + } + } + void erase(const_iterator f, const_iterator l) { + for ( ; f != l; ++f) { + if ( set_deleted(f) ) // should always be true + ++num_deleted; + } + // will think about shrink after next insert + settings.set_consider_shrink(true); + } + + + // COMPARISON + bool operator==(const sparse_hashtable& ht) const { + if (size() != ht.size()) { + return false; + } else if (this == &ht) { + return true; + } else { + // Iterate through the elements in "this" and see if the + // corresponding element is in ht + for ( const_iterator it = begin(); it != end(); ++it ) { + const_iterator it2 = ht.find(get_key(*it)); + if ((it2 == ht.end()) || (*it != *it2)) { + return false; + } + } + return true; + } + } + bool operator!=(const sparse_hashtable& ht) const { + return !(*this == ht); + } + + + // I/O + // We support reading and writing hashtables to disk. NOTE that + // this only stores the hashtable metadata, not the stuff you've + // actually put in the hashtable! Alas, since I don't know how to + // write a hasher or key_equal, you have to make sure everything + // but the table is the same. We compact before writing. + bool write_metadata(FILE *fp) { + squash_deleted(); // so we don't have to worry about delkey + return table.write_metadata(fp); + } + + bool read_metadata(FILE *fp) { + num_deleted = 0; // since we got rid before writing + bool result = table.read_metadata(fp); + settings.reset_thresholds(bucket_count()); + return result; + } + + // Only meaningful if value_type is a POD. + bool write_nopointer_data(FILE *fp) { + return table.write_nopointer_data(fp); + } + + // Only meaningful if value_type is a POD. + bool read_nopointer_data(FILE *fp) { + return table.read_nopointer_data(fp); + } + + private: + // Table is the main storage class. + typedef sparsetable Table; + + // Package templated functors with the other types to eliminate memory + // needed for storing these zero-size operators. Since ExtractKey and + // hasher's operator() might have the same function signature, they + // must be packaged in different classes. + struct Settings : + sh_hashtable_settings { + explicit Settings(const hasher& hf) + : sh_hashtable_settings( + hf, HT_OCCUPANCY_PCT / 100.0f, HT_EMPTY_PCT / 100.0f) {} + }; + + // KeyInfo stores delete key and packages zero-size functors: + // ExtractKey and SetKey. + class KeyInfo : public ExtractKey, public SetKey, public key_equal { + public: + KeyInfo(const ExtractKey& ek, const SetKey& sk, const key_equal& eq) + : ExtractKey(ek), + SetKey(sk), + key_equal(eq) { + } + // We want to return the exact same type as ExtractKey: Key or const Key& + typename ExtractKey::result_type get_key(const_reference v) const { + return ExtractKey::operator()(v); + } + void set_key(pointer v, const key_type& k) const { + SetKey::operator()(v, k); + } + bool equals(const key_type& a, const key_type& b) const { + return key_equal::operator()(a, b); + } + + // Which key marks deleted entries. + // TODO(csilvers): make a pointer, and get rid of use_deleted (benchmark!) + typename remove_const::type delkey; + }; + + // Utility functions to access the templated operators + size_type hash(const key_type& v) const { + return settings.hash(v); + } + bool equals(const key_type& a, const key_type& b) const { + return key_info.equals(a, b); + } + typename ExtractKey::result_type get_key(const_reference v) const { + return key_info.get_key(v); + } + void set_key(pointer v, const key_type& k) const { + key_info.set_key(v, k); + } + + private: + // Actual data + Settings settings; + KeyInfo key_info; + size_type num_deleted; // how many occupied buckets are marked deleted + Table table; // holds num_buckets and num_elements too +}; + + +// We need a global swap as well +template +inline void swap(sparse_hashtable &x, + sparse_hashtable &y) { + x.swap(y); +} + +#undef JUMP_ + +template +const typename sparse_hashtable::size_type + sparse_hashtable::ILLEGAL_BUCKET; + +// How full we let the table get before we resize. Knuth says .8 is +// good -- higher causes us to probe too much, though saves memory +template +const int sparse_hashtable::HT_OCCUPANCY_PCT = 80; + +// How empty we let the table get before we resize lower. +// It should be less than OCCUPANCY_PCT / 2 or we thrash resizing +template +const int sparse_hashtable::HT_EMPTY_PCT + = static_cast(0.4 * + sparse_hashtable::HT_OCCUPANCY_PCT); + +_END_GOOGLE_NAMESPACE_ + +#endif /* _SPARSEHASHTABLE_H_ */ diff --git a/external/google/sparsetable b/external/google/sparsetable new file mode 100644 index 00000000..aa4ccab4 --- /dev/null +++ b/external/google/sparsetable @@ -0,0 +1,1598 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein +// +// A sparsetable is a random container that implements a sparse array, +// that is, an array that uses very little memory to store unassigned +// indices (in this case, between 1-2 bits per unassigned index). For +// instance, if you allocate an array of size 5 and assign a[2] = , then a[2] will take up a lot of memory but a[0], a[1], +// a[3], and a[4] will not. Array elements that have a value are +// called "assigned". Array elements that have no value yet, or have +// had their value cleared using erase() or clear(), are called +// "unassigned". +// +// Unassigned values seem to have the default value of T (see below). +// Nevertheless, there is a difference between an unassigned index and +// one explicitly assigned the value of T(). The latter is considered +// assigned. +// +// Access to an array element is constant time, as is insertion and +// deletion. Insertion and deletion may be fairly slow, however: +// because of this container's memory economy, each insert and delete +// causes a memory reallocation. +// +// See doc/sparsetable.html for information about how to use this class. + +#ifndef _SPARSETABLE_H_ +#define _SPARSETABLE_H_ + +#include +#include // for malloc/free +#include // to read/write tables +#ifdef HAVE_STDINT_H +#include // the normal place uint16_t is defined +#endif +#ifdef HAVE_SYS_TYPES_H +#include // the normal place u_int16_t is defined +#endif +#ifdef HAVE_INTTYPES_H +#include // a third place for uint16_t or u_int16_t +#endif +#include // for bounds checking +#include // to define reverse_iterator for me +#include // equal, lexicographical_compare, swap,... +#include // uninitialized_copy +#include // a sparsetable is a vector of groups +#include +#include // for true_type, integral_constant, etc. + +#if STDC_HEADERS +#include // for memcpy +#else +#if !HAVE_MEMCPY +#define memcpy(d, s, n) bcopy ((s), (d), (n)) +#endif +#endif + +#ifndef HAVE_U_INT16_T +# if defined HAVE_UINT16_T + typedef uint16_t u_int16_t; // true on solaris, possibly other C99 libc's +# elif defined HAVE___UINT16 + typedef __int16 int16_t; // true on vc++7 + typedef unsigned __int16 u_int16_t; +# else + // Cannot find a 16-bit integer type. Hoping for the best with "short"... + typedef short int int16_t; + typedef unsigned short int u_int16_t; +# endif +#endif + +_START_GOOGLE_NAMESPACE_ + +using STL_NAMESPACE::vector; +using STL_NAMESPACE::uninitialized_copy; + +// The smaller this is, the faster lookup is (because the group bitmap is +// smaller) and the faster insert is, because there's less to move. +// On the other hand, there are more groups. Since group::size_type is +// a short, this number should be of the form 32*x + 16 to avoid waste. +static const u_int16_t DEFAULT_SPARSEGROUP_SIZE = 48; // fits in 1.5 words + + +// A NOTE ON ASSIGNING: +// A sparse table does not actually allocate memory for entries +// that are not filled. Because of this, it becomes complicated +// to have a non-const iterator: we don't know, if the iterator points +// to a not-filled bucket, whether you plan to fill it with something +// or whether you plan to read its value (in which case you'll get +// the default bucket value). Therefore, while we can define const +// operations in a pretty 'normal' way, for non-const operations, we +// define something that returns a helper object with operator= and +// operator& that allocate a bucket lazily. We use this for table[] +// and also for regular table iterators. + +template +class table_element_adaptor { + public: + typedef typename tabletype::value_type value_type; + typedef typename tabletype::size_type size_type; + typedef typename tabletype::reference reference; + typedef typename tabletype::pointer pointer; + + table_element_adaptor(tabletype *tbl, size_type p) + : table(tbl), pos(p) { } + table_element_adaptor& operator= (const value_type &val) { + table->set(pos, val); + return *this; + } + operator value_type() { return table->get(pos); } // we look like a value + pointer operator& () { return &table->mutating_get(pos); } + + private: + tabletype* table; + size_type pos; +}; + +// Our iterator as simple as iterators can be: basically it's just +// the index into our table. Dereference, the only complicated +// thing, we punt to the table class. This just goes to show how +// much machinery STL requires to do even the most trivial tasks. +// +// By templatizing over tabletype, we have one iterator type which +// we can use for both sparsetables and sparsebins. In fact it +// works on any class that allows size() and operator[] (eg vector), +// as long as it does the standard STL typedefs too (eg value_type). + +template +class table_iterator { + public: + typedef table_iterator iterator; + + typedef STL_NAMESPACE::random_access_iterator_tag iterator_category; + typedef typename tabletype::value_type value_type; + typedef typename tabletype::difference_type difference_type; + typedef typename tabletype::size_type size_type; + typedef table_element_adaptor reference; + typedef table_element_adaptor* pointer; + + // The "real" constructor + table_iterator(tabletype *tbl, size_type p) + : table(tbl), pos(p) { } + // The default constructor, used when I define vars of type table::iterator + table_iterator() : table(NULL), pos(0) { } + // The copy constructor, for when I say table::iterator foo = tbl.begin() + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // The main thing our iterator does is dereference. If the table entry + // we point to is empty, we return the default value type. + // This is the big different function from the const iterator. + reference operator*() { + return table_element_adaptor(table, pos); + } + pointer operator->() { return &(operator*()); } + + // Helper function to assert things are ok; eg pos is still in range + void check() const { + assert(table); + assert(pos <= table->size()); + } + + // Arithmetic: we just do arithmetic on pos. We don't even need to + // do bounds checking, since STL doesn't consider that its job. :-) + iterator& operator+=(size_type t) { pos += t; check(); return *this; } + iterator& operator-=(size_type t) { pos -= t; check(); return *this; } + iterator& operator++() { ++pos; check(); return *this; } + iterator& operator--() { --pos; check(); return *this; } + iterator operator++(int) { iterator tmp(*this); // for x++ + ++pos; check(); return tmp; } + iterator operator--(int) { iterator tmp(*this); // for x-- + --pos; check(); return tmp; } + iterator operator+(difference_type i) const { iterator tmp(*this); + tmp += i; return tmp; } + iterator operator-(difference_type i) const { iterator tmp(*this); + tmp -= i; return tmp; } + difference_type operator-(iterator it) const { // for "x = it2 - it" + assert(table == it.table); + return pos - it.pos; + } + reference operator[](difference_type n) const { + return *(*this + n); // simple though not totally efficient + } + + // Comparisons. + bool operator==(const iterator& it) const { + return table == it.table && pos == it.pos; + } + bool operator<(const iterator& it) const { + assert(table == it.table); // life is bad bad bad otherwise + return pos < it.pos; + } + bool operator!=(const iterator& it) const { return !(*this == it); } + bool operator<=(const iterator& it) const { return !(it < *this); } + bool operator>(const iterator& it) const { return it < *this; } + bool operator>=(const iterator& it) const { return !(*this < it); } + + // Here's the info we actually need to be an iterator + tabletype *table; // so we can dereference and bounds-check + size_type pos; // index into the table +}; + +// support for "3 + iterator" has to be defined outside the class, alas +template +table_iterator operator+(typename table_iterator::difference_type i, + table_iterator it) { + return it + i; // so people can say it2 = 3 + it +} + +template +class const_table_iterator { + public: + typedef table_iterator iterator; + typedef const_table_iterator const_iterator; + + typedef STL_NAMESPACE::random_access_iterator_tag iterator_category; + typedef typename tabletype::value_type value_type; + typedef typename tabletype::difference_type difference_type; + typedef typename tabletype::size_type size_type; + typedef typename tabletype::const_reference reference; // we're const-only + typedef typename tabletype::const_pointer pointer; + + // The "real" constructor + const_table_iterator(const tabletype *tbl, size_type p) + : table(tbl), pos(p) { } + // The default constructor, used when I define vars of type table::iterator + const_table_iterator() : table(NULL), pos(0) { } + // The copy constructor, for when I say table::iterator foo = tbl.begin() + // Also converts normal iterators to const iterators + const_table_iterator(const iterator &from) + : table(from.table), pos(from.pos) { } + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // The main thing our iterator does is dereference. If the table entry + // we point to is empty, we return the default value type. + reference operator*() const { return (*table)[pos]; } + pointer operator->() const { return &(operator*()); } + + // Helper function to assert things are ok; eg pos is still in range + void check() const { + assert(table); + assert(pos <= table->size()); + } + + // Arithmetic: we just do arithmetic on pos. We don't even need to + // do bounds checking, since STL doesn't consider that its job. :-) + const_iterator& operator+=(size_type t) { pos += t; check(); return *this; } + const_iterator& operator-=(size_type t) { pos -= t; check(); return *this; } + const_iterator& operator++() { ++pos; check(); return *this; } + const_iterator& operator--() { --pos; check(); return *this; } + const_iterator operator++(int) { const_iterator tmp(*this); // for x++ + ++pos; check(); return tmp; } + const_iterator operator--(int) { const_iterator tmp(*this); // for x-- + --pos; check(); return tmp; } + const_iterator operator+(difference_type i) const { const_iterator tmp(*this); + tmp += i; return tmp; } + const_iterator operator-(difference_type i) const { const_iterator tmp(*this); + tmp -= i; return tmp; } + difference_type operator-(const_iterator it) const { // for "x = it2 - it" + assert(table == it.table); + return pos - it.pos; + } + reference operator[](difference_type n) const { + return *(*this + n); // simple though not totally efficient + } + + // Comparisons. + bool operator==(const const_iterator& it) const { + return table == it.table && pos == it.pos; + } + bool operator<(const const_iterator& it) const { + assert(table == it.table); // life is bad bad bad otherwise + return pos < it.pos; + } + bool operator!=(const const_iterator& it) const { return !(*this == it); } + bool operator<=(const const_iterator& it) const { return !(it < *this); } + bool operator>(const const_iterator& it) const { return it < *this; } + bool operator>=(const const_iterator& it) const { return !(*this < it); } + + // Here's the info we actually need to be an iterator + const tabletype *table; // so we can dereference and bounds-check + size_type pos; // index into the table +}; + +// support for "3 + iterator" has to be defined outside the class, alas +template +const_table_iterator operator+(typename + const_table_iterator::difference_type i, + const_table_iterator it) { + return it + i; // so people can say it2 = 3 + it +} + + +// --------------------------------------------------------------------------- + + +/* +// This is a 2-D iterator. You specify a begin and end over a list +// of *containers*. We iterate over each container by iterating over +// it. It's actually simple: +// VECTOR.begin() VECTOR[0].begin() --------> VECTOR[0].end() ---, +// | ________________________________________________/ +// | \_> VECTOR[1].begin() --------> VECTOR[1].end() -, +// | ___________________________________________________/ +// v \_> ...... +// VECTOR.end() +// +// It's impossible to do random access on one of these things in constant +// time, so it's just a bidirectional iterator. +// +// Unfortunately, because we need to use this for a non-empty iterator, +// we use nonempty_begin() and nonempty_end() instead of begin() and end() +// (though only going across, not down). +*/ + +#define TWOD_BEGIN_ nonempty_begin +#define TWOD_END_ nonempty_end +#define TWOD_ITER_ nonempty_iterator +#define TWOD_CONST_ITER_ const_nonempty_iterator + +template +class two_d_iterator { + public: + typedef two_d_iterator iterator; + + typedef STL_NAMESPACE::bidirectional_iterator_tag iterator_category; + // apparently some versions of VC++ have trouble with two ::'s in a typename + typedef typename containertype::value_type _tmp_vt; + typedef typename _tmp_vt::value_type value_type; + typedef typename _tmp_vt::difference_type difference_type; + typedef typename _tmp_vt::reference reference; + typedef typename _tmp_vt::pointer pointer; + + // The "real" constructor. begin and end specify how many rows we have + // (in the diagram above); we always iterate over each row completely. + two_d_iterator(typename containertype::iterator begin, + typename containertype::iterator end, + typename containertype::iterator curr) + : row_begin(begin), row_end(end), row_current(curr), col_current() { + if ( row_current != row_end ) { + col_current = row_current->TWOD_BEGIN_(); + advance_past_end(); // in case cur->begin() == cur->end() + } + } + // If you want to start at an arbitrary place, you can, I guess + two_d_iterator(typename containertype::iterator begin, + typename containertype::iterator end, + typename containertype::iterator curr, + typename containertype::value_type::TWOD_ITER_ col) + : row_begin(begin), row_end(end), row_current(curr), col_current(col) { + advance_past_end(); // in case cur->begin() == cur->end() + } + // The default constructor, used when I define vars of type table::iterator + two_d_iterator() : row_begin(), row_end(), row_current(), col_current() { } + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *col_current; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic: we just do arithmetic on pos. We don't even need to + // do bounds checking, since STL doesn't consider that its job. :-) + // NOTE: this is not amortized constant time! What do we do about it? + void advance_past_end() { // used when col_current points to end() + while ( col_current == row_current->TWOD_END_() ) { // end of current row + ++row_current; // go to beginning of next + if ( row_current != row_end ) // col is irrelevant at end + col_current = row_current->TWOD_BEGIN_(); + else + break; // don't go past row_end + } + } + + iterator& operator++() { + assert(row_current != row_end); // how to ++ from there? + ++col_current; + advance_past_end(); // in case col_current is at end() + return *this; + } + iterator& operator--() { + while ( row_current == row_end || + col_current == row_current->TWOD_BEGIN_() ) { + assert(row_current != row_begin); + --row_current; + col_current = row_current->TWOD_END_(); // this is 1 too far + } + --col_current; + return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + iterator operator--(int) { iterator tmp(*this); --*this; return tmp; } + + + // Comparisons. + bool operator==(const iterator& it) const { + return ( row_begin == it.row_begin && + row_end == it.row_end && + row_current == it.row_current && + (row_current == row_end || col_current == it.col_current) ); + } + bool operator!=(const iterator& it) const { return !(*this == it); } + + + // Here's the info we actually need to be an iterator + // These need to be public so we convert from iterator to const_iterator + typename containertype::iterator row_begin, row_end, row_current; + typename containertype::value_type::TWOD_ITER_ col_current; +}; + +// The same thing again, but this time const. :-( +template +class const_two_d_iterator { + public: + typedef const_two_d_iterator iterator; + + typedef STL_NAMESPACE::bidirectional_iterator_tag iterator_category; + // apparently some versions of VC++ have trouble with two ::'s in a typename + typedef typename containertype::value_type _tmp_vt; + typedef typename _tmp_vt::value_type value_type; + typedef typename _tmp_vt::difference_type difference_type; + typedef typename _tmp_vt::const_reference reference; + typedef typename _tmp_vt::const_pointer pointer; + + const_two_d_iterator(typename containertype::const_iterator begin, + typename containertype::const_iterator end, + typename containertype::const_iterator curr) + : row_begin(begin), row_end(end), row_current(curr), col_current() { + if ( curr != end ) { + col_current = curr->TWOD_BEGIN_(); + advance_past_end(); // in case cur->begin() == cur->end() + } + } + const_two_d_iterator(typename containertype::const_iterator begin, + typename containertype::const_iterator end, + typename containertype::const_iterator curr, + typename containertype::value_type::TWOD_CONST_ITER_ col) + : row_begin(begin), row_end(end), row_current(curr), col_current(col) { + advance_past_end(); // in case cur->begin() == cur->end() + } + const_two_d_iterator() + : row_begin(), row_end(), row_current(), col_current() { + } + // Need this explicitly so we can convert normal iterators to const iterators + const_two_d_iterator(const two_d_iterator& it) : + row_begin(it.row_begin), row_end(it.row_end), row_current(it.row_current), + col_current(it.col_current) { } + + typename containertype::const_iterator row_begin, row_end, row_current; + typename containertype::value_type::TWOD_CONST_ITER_ col_current; + + + // EVERYTHING FROM HERE DOWN IS THE SAME AS THE NON-CONST ITERATOR + reference operator*() const { return *col_current; } + pointer operator->() const { return &(operator*()); } + + void advance_past_end() { // used when col_current points to end() + while ( col_current == row_current->TWOD_END_() ) { // end of current row + ++row_current; // go to beginning of next + if ( row_current != row_end ) // col is irrelevant at end + col_current = row_current->TWOD_BEGIN_(); + else + break; // don't go past row_end + } + } + iterator& operator++() { + assert(row_current != row_end); // how to ++ from there? + ++col_current; + advance_past_end(); // in case col_current is at end() + return *this; + } + iterator& operator--() { + while ( row_current == row_end || + col_current == row_current->TWOD_BEGIN_() ) { + assert(row_current != row_begin); + --row_current; + col_current = row_current->TWOD_END_(); // this is 1 too far + } + --col_current; + return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + iterator operator--(int) { iterator tmp(*this); --*this; return tmp; } + + bool operator==(const iterator& it) const { + return ( row_begin == it.row_begin && + row_end == it.row_end && + row_current == it.row_current && + (row_current == row_end || col_current == it.col_current) ); + } + bool operator!=(const iterator& it) const { return !(*this == it); } +}; + +// We provide yet another version, to be as frugal with memory as +// possible. This one frees each block of memory as it finishes +// iterating over it. By the end, the entire table is freed. +// For understandable reasons, you can only iterate over it once, +// which is why it's an input iterator +template +class destructive_two_d_iterator { + public: + typedef destructive_two_d_iterator iterator; + + typedef STL_NAMESPACE::input_iterator_tag iterator_category; + // apparently some versions of VC++ have trouble with two ::'s in a typename + typedef typename containertype::value_type _tmp_vt; + typedef typename _tmp_vt::value_type value_type; + typedef typename _tmp_vt::difference_type difference_type; + typedef typename _tmp_vt::reference reference; + typedef typename _tmp_vt::pointer pointer; + + destructive_two_d_iterator(typename containertype::iterator begin, + typename containertype::iterator end, + typename containertype::iterator curr) + : row_begin(begin), row_end(end), row_current(curr), col_current() { + if ( curr != end ) { + col_current = curr->TWOD_BEGIN_(); + advance_past_end(); // in case cur->begin() == cur->end() + } + } + destructive_two_d_iterator(typename containertype::iterator begin, + typename containertype::iterator end, + typename containertype::iterator curr, + typename containertype::value_type::TWOD_ITER_ col) + : row_begin(begin), row_end(end), row_current(curr), col_current(col) { + advance_past_end(); // in case cur->begin() == cur->end() + } + destructive_two_d_iterator() + : row_begin(), row_end(), row_current(), col_current() { + } + + typename containertype::iterator row_begin, row_end, row_current; + typename containertype::value_type::TWOD_ITER_ col_current; + + // This is the part that destroys + void advance_past_end() { // used when col_current points to end() + while ( col_current == row_current->TWOD_END_() ) { // end of current row + row_current->clear(); // the destructive part + // It would be nice if we could decrement sparsetable->num_buckets here + ++row_current; // go to beginning of next + if ( row_current != row_end ) // col is irrelevant at end + col_current = row_current->TWOD_BEGIN_(); + else + break; // don't go past row_end + } + } + + // EVERYTHING FROM HERE DOWN IS THE SAME AS THE REGULAR ITERATOR + reference operator*() const { return *col_current; } + pointer operator->() const { return &(operator*()); } + + iterator& operator++() { + assert(row_current != row_end); // how to ++ from there? + ++col_current; + advance_past_end(); // in case col_current is at end() + return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + + bool operator==(const iterator& it) const { + return ( row_begin == it.row_begin && + row_end == it.row_end && + row_current == it.row_current && + (row_current == row_end || col_current == it.col_current) ); + } + bool operator!=(const iterator& it) const { return !(*this == it); } +}; + +#undef TWOD_BEGIN_ +#undef TWOD_END_ +#undef TWOD_ITER_ +#undef TWOD_CONST_ITER_ + + + + +// SPARSE-TABLE +// ------------ +// The idea is that a table with (logically) t buckets is divided +// into t/M *groups* of M buckets each. (M is a constant set in +// GROUP_SIZE for efficiency.) Each group is stored sparsely. +// Thus, inserting into the table causes some array to grow, which is +// slow but still constant time. Lookup involves doing a +// logical-position-to-sparse-position lookup, which is also slow but +// constant time. The larger M is, the slower these operations are +// but the less overhead (slightly). +// +// To store the sparse array, we store a bitmap B, where B[i] = 1 iff +// bucket i is non-empty. Then to look up bucket i we really look up +// array[# of 1s before i in B]. This is constant time for fixed M. +// +// Terminology: the position of an item in the overall table (from +// 1 .. t) is called its "location." The logical position in a group +// (from 1 .. M ) is called its "position." The actual location in +// the array (from 1 .. # of non-empty buckets in the group) is +// called its "offset." + +// The weird mod in the offset is entirely to quiet compiler warnings +// as is the cast to int after doing the "x mod 256" +#define PUT_(take_from, offset) do { \ + if (putc(static_cast(((take_from) >> ((offset) % (sizeof(take_from)*8)))\ + % 256), fp) \ + == EOF) \ + return false; \ +} while (0) + +#define GET_(add_to, offset) do { \ + if ((x=getc(fp)) == EOF) \ + return false; \ + else \ + add_to |= (static_cast(x) << ((offset) % (sizeof(add_to)*8))); \ +} while (0) + +template +class sparsegroup { + private: + typedef typename Alloc::template rebind::other value_alloc_type; + + public: + // Basic types + typedef T value_type; + typedef Alloc allocator_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::const_reference const_reference; + typedef typename value_alloc_type::pointer pointer; + typedef typename value_alloc_type::const_pointer const_pointer; + + typedef table_iterator > iterator; + typedef const_table_iterator > + const_iterator; + typedef table_element_adaptor > + element_adaptor; + typedef u_int16_t size_type; // max # of buckets + typedef int16_t difference_type; + typedef STL_NAMESPACE::reverse_iterator const_reverse_iterator; + typedef STL_NAMESPACE::reverse_iterator reverse_iterator; + + // These are our special iterators, that go over non-empty buckets in a + // group. These aren't const-only because you can change non-empty bcks. + typedef pointer nonempty_iterator; + typedef const_pointer const_nonempty_iterator; + typedef STL_NAMESPACE::reverse_iterator reverse_nonempty_iterator; + typedef STL_NAMESPACE::reverse_iterator const_reverse_nonempty_iterator; + + // Iterator functions + iterator begin() { return iterator(this, 0); } + const_iterator begin() const { return const_iterator(this, 0); } + iterator end() { return iterator(this, size()); } + const_iterator end() const { return const_iterator(this, size()); } + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } + + // We'll have versions for our special non-empty iterator too + nonempty_iterator nonempty_begin() { return group; } + const_nonempty_iterator nonempty_begin() const { return group; } + nonempty_iterator nonempty_end() { + return group + settings.num_buckets; + } + const_nonempty_iterator nonempty_end() const { + return group + settings.num_buckets; + } + reverse_nonempty_iterator nonempty_rbegin() { + return reverse_nonempty_iterator(nonempty_end()); + } + const_reverse_nonempty_iterator nonempty_rbegin() const { + return const_reverse_nonempty_iterator(nonempty_end()); + } + reverse_nonempty_iterator nonempty_rend() { + return reverse_nonempty_iterator(nonempty_begin()); + } + const_reverse_nonempty_iterator nonempty_rend() const { + return const_reverse_nonempty_iterator(nonempty_begin()); + } + + + // This gives us the "default" value to return for an empty bucket. + // We just use the default constructor on T, the template type + const_reference default_value() const { + static value_type defaultval = value_type(); + return defaultval; + } + + + private: + // We need to do all this bit manipulation, of course. ick + static size_type charbit(size_type i) { return i >> 3; } + static size_type modbit(size_type i) { return 1 << (i&7); } + int bmtest(size_type i) const { return bitmap[charbit(i)] & modbit(i); } + void bmset(size_type i) { bitmap[charbit(i)] |= modbit(i); } + void bmclear(size_type i) { bitmap[charbit(i)] &= ~modbit(i); } + + pointer allocate_group(size_type n) { + pointer retval = settings.allocate(n); + if (retval == NULL) { + // We really should use PRIuS here, but I don't want to have to add + // a whole new configure option, with concomitant macro namespace + // pollution, just to print this (unlikely) error message. So I cast. + fprintf(stderr, "sparsehash: FATAL ERROR: " + "failed to allocate %lu groups\n", + static_cast(n)); + exit(1); + } + return retval; + } + + void free_group() { + if (!group) return; + pointer end_it = group + settings.num_buckets; + for (pointer p = group; p != end_it; ++p) + p->~value_type(); + settings.deallocate(group, settings.num_buckets); + group = NULL; + } + + public: // get_iter() in sparsetable needs it + // We need a small function that tells us how many set bits there are + // in positions 0..i-1 of the bitmap. It uses a big table. + // We make it static so templates don't allocate lots of these tables. + // There are lots of ways to do this calculation (called 'popcount'). + // The 8-bit table lookup is one of the fastest, though this + // implementation suffers from not doing any loop unrolling. See, eg, + // http://www.dalkescientific.com/writings/diary/archive/2008/07/03/hakmem_and_other_popcounts.html + // http://gurmeetsingh.wordpress.com/2008/08/05/fast-bit-counting-routines/ + static size_type pos_to_offset(const unsigned char *bm, size_type pos) { + // We could make these ints. The tradeoff is size (eg does it overwhelm + // the cache?) vs efficiency in referencing sub-word-sized array elements + static const char bits_in[256] = { // # of bits set in one char + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8, + }; + size_type retval = 0; + + // [Note: condition pos > 8 is an optimization; convince yourself we + // give exactly the same result as if we had pos >= 8 here instead.] + for ( ; pos > 8; pos -= 8 ) // bm[0..pos/8-1] + retval += bits_in[*bm++]; // chars we want *all* bits in + return retval + bits_in[*bm & ((1 << pos)-1)]; // the char that includes pos + } + + size_type pos_to_offset(size_type pos) const { // not static but still const + return pos_to_offset(bitmap, pos); + } + + + public: + // Constructors -- default and copy -- and destructor + sparsegroup(allocator_type& a) : + group(0), settings(alloc_impl(a)) { + memset(bitmap, 0, sizeof(bitmap)); + } + sparsegroup(const sparsegroup& x) : group(0), settings(x.settings) { + if ( settings.num_buckets ) { + group = allocate_group(x.settings.num_buckets); + uninitialized_copy(x.group, x.group + x.settings.num_buckets, group); + } + memcpy(bitmap, x.bitmap, sizeof(bitmap)); + } + ~sparsegroup() { free_group(); } + + // Operator= is just like the copy constructor, I guess + // TODO(austern): Make this exception safe. Handle exceptions in value_type's + // copy constructor. + sparsegroup &operator=(const sparsegroup& x) { + if ( &x == this ) return *this; // x = x + if ( x.settings.num_buckets == 0 ) { + free_group(); + } else { + pointer p = allocate_group(x.settings.num_buckets); + uninitialized_copy(x.group, x.group + x.settings.num_buckets, p); + free_group(); + group = p; + } + memcpy(bitmap, x.bitmap, sizeof(bitmap)); + settings.num_buckets = x.settings.num_buckets; + return *this; + } + + // Many STL algorithms use swap instead of copy constructors + void swap(sparsegroup& x) { + STL_NAMESPACE::swap(group, x.group); + for ( int i = 0; i < sizeof(bitmap) / sizeof(*bitmap); ++i ) + STL_NAMESPACE::swap(bitmap[i], x.bitmap[i]); // swap not defined on arrays + STL_NAMESPACE::swap(settings.num_buckets, x.settings.num_buckets); + // we purposefully don't swap the allocator, which may not be swap-able + } + + // It's always nice to be able to clear a table without deallocating it + void clear() { + free_group(); + memset(bitmap, 0, sizeof(bitmap)); + settings.num_buckets = 0; + } + + // Functions that tell you about size. Alas, these aren't so useful + // because our table is always fixed size. + size_type size() const { return GROUP_SIZE; } + size_type max_size() const { return GROUP_SIZE; } + bool empty() const { return false; } + // We also may want to know how many *used* buckets there are + size_type num_nonempty() const { return settings.num_buckets; } + + + // get()/set() are explicitly const/non-const. You can use [] if + // you want something that can be either (potentially more expensive). + const_reference get(size_type i) const { + if ( bmtest(i) ) // bucket i is occupied + return group[pos_to_offset(bitmap, i)]; + else + return default_value(); // return the default reference + } + + // TODO(csilvers): make protected + friend + // This is used by sparse_hashtable to get an element from the table + // when we know it exists. + const_reference unsafe_get(size_type i) const { + assert(bmtest(i)); + return group[pos_to_offset(bitmap, i)]; + } + + // TODO(csilvers): make protected + friend + reference mutating_get(size_type i) { // fills bucket i before getting + if ( !bmtest(i) ) + set(i, default_value()); + return group[pos_to_offset(bitmap, i)]; + } + + // Syntactic sugar. It's easy to return a const reference. To + // return a non-const reference, we need to use the assigner adaptor. + const_reference operator[](size_type i) const { + return get(i); + } + + element_adaptor operator[](size_type i) { + return element_adaptor(this, i); + } + + private: + // Create space at group[offset], assuming value_type has trivial + // copy constructor and destructor, and the allocator_type is + // the default libc_allocator_with_alloc. (Really, we want it to have + // "trivial move", because that's what realloc and memmove both do. + // But there's no way to capture that using type_traits, so we + // pretend that move(x, y) is equivalent to "x.~T(); new(x) T(y);" + // which is pretty much correct, if a bit conservative.) + void set_aux(size_type offset, true_type) { + group = settings.realloc_or_die(group, settings.num_buckets+1); + // This is equivalent to memmove(), but faster on my Intel P4, + // at least with gcc4.1 -O2 / glibc 2.3.6. + for (size_type i = settings.num_buckets; i > offset; --i) + memcpy(group + i, group + i-1, sizeof(*group)); + } + + // Create space at group[offset], without special assumptions about value_type + // and allocator_type. + void set_aux(size_type offset, false_type) { + // This is valid because 0 <= offset <= num_buckets + pointer p = allocate_group(settings.num_buckets + 1); + uninitialized_copy(group, group + offset, p); + uninitialized_copy(group + offset, group + settings.num_buckets, + p + offset + 1); + free_group(); + group = p; + } + + public: + // This returns a reference to the inserted item (which is a copy of val). + // TODO(austern): Make this exception safe: handle exceptions from + // value_type's copy constructor. + reference set(size_type i, const_reference val) { + size_type offset = pos_to_offset(bitmap, i); // where we'll find (or insert) + if ( bmtest(i) ) { + // Delete the old value, which we're replacing with the new one + group[offset].~value_type(); + } else { + typedef integral_constant::value && + has_trivial_destructor::value && + is_same >::value)> + realloc_and_memmove_ok; // we pretend mv(x,y) == "x.~T(); new(x) T(y)" + set_aux(offset, realloc_and_memmove_ok()); + ++settings.num_buckets; + bmset(i); + } + // This does the actual inserting. Since we made the array using + // malloc, we use "placement new" to just call the constructor. + new(&group[offset]) value_type(val); + return group[offset]; + } + + // We let you see if a bucket is non-empty without retrieving it + bool test(size_type i) const { + return bmtest(i) != 0; + } + bool test(iterator pos) const { + return bmtest(pos.pos) != 0; + } + + private: + // Shrink the array, assuming value_type has trivial copy + // constructor and destructor, and the allocator_type is the default + // libc_allocator_with_alloc. (Really, we want it to have "trivial + // move", because that's what realloc and memmove both do. But + // there's no way to capture that using type_traits, so we pretend + // that move(x, y) is equivalent to ""x.~T(); new(x) T(y);" + // which is pretty much correct, if a bit conservative.) + void erase_aux(size_type offset, true_type) { + // This isn't technically necessary, since we know we have a + // trivial destructor, but is a cheap way to get a bit more safety. + group[offset].~value_type(); + // This is equivalent to memmove(), but faster on my Intel P4, + // at lesat with gcc4.1 -O2 / glibc 2.3.6. + assert(settings.num_buckets > 0); + for (size_type i = offset; i < settings.num_buckets-1; ++i) + memcpy(group + i, group + i+1, sizeof(*group)); // hopefully inlined! + group = settings.realloc_or_die(group, settings.num_buckets-1); + } + + // Shrink the array, without any special assumptions about value_type and + // allocator_type. + void erase_aux(size_type offset, false_type) { + // This is valid because 0 <= offset < num_buckets. Note the inequality. + pointer p = allocate_group(settings.num_buckets - 1); + uninitialized_copy(group, group + offset, p); + uninitialized_copy(group + offset + 1, group + settings.num_buckets, + p + offset); + free_group(); + group = p; + } + + public: + // This takes the specified elements out of the group. This is + // "undefining", rather than "clearing". + // TODO(austern): Make this exception safe: handle exceptions from + // value_type's copy constructor. + void erase(size_type i) { + if ( bmtest(i) ) { // trivial to erase empty bucket + size_type offset = pos_to_offset(bitmap,i); // where we'll find (or insert) + if ( settings.num_buckets == 1 ) { + free_group(); + group = NULL; + } else { + typedef integral_constant::value && + has_trivial_destructor::value && + is_same< + allocator_type, + libc_allocator_with_realloc >::value)> + realloc_and_memmove_ok; // pretend mv(x,y) == "x.~T(); new(x) T(y)" + erase_aux(offset, realloc_and_memmove_ok()); + } + --settings.num_buckets; + bmclear(i); + } + } + + void erase(iterator pos) { + erase(pos.pos); + } + + void erase(iterator start_it, iterator end_it) { + // This could be more efficient, but to do so we'd need to make + // bmclear() clear a range of indices. Doesn't seem worth it. + for ( ; start_it != end_it; ++start_it ) + erase(start_it); + } + + + // I/O + // We support reading and writing groups to disk. We don't store + // the actual array contents (which we don't know how to store), + // just the bitmap and size. Meant to be used with table I/O. + // Returns true if all was ok + bool write_metadata(FILE *fp) const { + // we explicitly set to u_int16_t + assert(sizeof(settings.num_buckets) == 2); + PUT_(settings.num_buckets, 8); + PUT_(settings.num_buckets, 0); + if ( !fwrite(bitmap, sizeof(bitmap), 1, fp) ) + return false; + return true; + } + + // Reading destroys the old group contents! Returns true if all was ok + bool read_metadata(FILE *fp) { + clear(); + + int x; // the GET_ macro requires an 'int x' to be defined + GET_(settings.num_buckets, 8); + GET_(settings.num_buckets, 0); + + if ( !fread(bitmap, sizeof(bitmap), 1, fp) ) return false; + + // We'll allocate the space, but we won't fill it: it will be + // left as uninitialized raw memory. + group = allocate_group(settings.num_buckets); + return true; + } + + // If your keys and values are simple enough, we can write them + // to disk for you. "simple enough" means POD and no pointers. + // However, we don't try to normalize endianness + bool write_nopointer_data(FILE *fp) const { + for ( const_nonempty_iterator it = nonempty_begin(); + it != nonempty_end(); ++it ) { + if ( !fwrite(&*it, sizeof(*it), 1, fp) ) return false; + } + return true; + } + + // When reading, we have to override the potential const-ness of *it. + // Again, only meaningful if value_type is a POD. + bool read_nopointer_data(FILE *fp) { + for ( nonempty_iterator it = nonempty_begin(); + it != nonempty_end(); ++it ) { + if ( !fread(reinterpret_cast(&(*it)), sizeof(*it), 1, fp) ) + return false; + } + return true; + } + + // Comparisons. Note the comparisons are pretty arbitrary: we + // compare values of the first index that isn't equal (using default + // value for empty buckets). + bool operator==(const sparsegroup& x) const { + return ( settings.num_buckets == x.settings.num_buckets && + memcmp(bitmap, x.bitmap, sizeof(bitmap)) == 0 && + STL_NAMESPACE::equal(begin(), end(), x.begin()) ); // from algorithm + } + bool operator<(const sparsegroup& x) const { // also from algorithm + return STL_NAMESPACE::lexicographical_compare(begin(), end(), + x.begin(), x.end()); + } + bool operator!=(const sparsegroup& x) const { return !(*this == x); } + bool operator<=(const sparsegroup& x) const { return !(x < *this); } + bool operator>(const sparsegroup& x) const { return x < *this; } + bool operator>=(const sparsegroup& x) const { return !(*this < x); } + + private: + template + class alloc_impl : public A { + public: + typedef typename A::pointer pointer; + typedef typename A::size_type size_type; + + // Convert a normal allocator to one that has realloc_or_die() + alloc_impl(const A& a) : A(a) { } + + // realloc_or_die should only be used when using the default + // allocator (libc_allocator_with_realloc). + pointer realloc_or_die(pointer /*ptr*/, size_type /*n*/) { + fprintf(stderr, "realloc_or_die is only supported for " + "libc_allocator_with_realloc"); + exit(1); + return NULL; + } + }; + + // A template specialization of alloc_impl for + // libc_allocator_with_realloc that can handle realloc_or_die. + template + class alloc_impl > + : public libc_allocator_with_realloc { + public: + typedef typename libc_allocator_with_realloc::pointer pointer; + typedef typename libc_allocator_with_realloc::size_type size_type; + + alloc_impl(const libc_allocator_with_realloc& a) + : libc_allocator_with_realloc(a) { } + + pointer realloc_or_die(pointer ptr, size_type n) { + pointer retval = this->reallocate(ptr, n); + if (retval == NULL) { + // We really should use PRIuS here, but I don't want to have to add + // a whole new configure option, with concomitant macro namespace + // pollution, just to print this (unlikely) error message. So I cast. + fprintf(stderr, "sparsehash: FATAL ERROR: failed to reallocate " + "%lu elements for ptr %p", + static_cast(n), ptr); + exit(1); + } + return retval; + } + }; + + // Package allocator with num_buckets to eliminate memory needed for the + // zero-size allocator. + // If new fields are added to this class, we should add them to + // operator= and swap. + class Settings : public alloc_impl { + public: + Settings(const alloc_impl& a, u_int16_t n = 0) + : alloc_impl(a), num_buckets(n) { } + Settings(const Settings& s) + : alloc_impl(s), num_buckets(s.num_buckets) { } + + u_int16_t num_buckets; // limits GROUP_SIZE to 64K + }; + + // The actual data + pointer group; // (small) array of T's + Settings settings; // allocator and num_buckets + unsigned char bitmap[(GROUP_SIZE-1)/8 + 1]; // fancy math is so we round up +}; + +// We need a global swap as well +template +inline void swap(sparsegroup &x, + sparsegroup &y) { + x.swap(y); +} + +// --------------------------------------------------------------------------- + + +template > +class sparsetable { + private: + typedef typename Alloc::template rebind::other value_alloc_type; + typedef typename Alloc::template rebind< + sparsegroup >::other vector_alloc; + + public: + // Basic types + typedef T value_type; // stolen from stl_vector.h + typedef Alloc allocator_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::const_reference const_reference; + typedef typename value_alloc_type::pointer pointer; + typedef typename value_alloc_type::const_pointer const_pointer; + typedef table_iterator > iterator; + typedef const_table_iterator > + const_iterator; + typedef table_element_adaptor > + element_adaptor; + typedef STL_NAMESPACE::reverse_iterator const_reverse_iterator; + typedef STL_NAMESPACE::reverse_iterator reverse_iterator; + + // These are our special iterators, that go over non-empty buckets in a + // table. These aren't const only because you can change non-empty bcks. + typedef two_d_iterator< vector< sparsegroup, + vector_alloc> > + nonempty_iterator; + typedef const_two_d_iterator< vector< sparsegroup, + vector_alloc> > + const_nonempty_iterator; + typedef STL_NAMESPACE::reverse_iterator reverse_nonempty_iterator; + typedef STL_NAMESPACE::reverse_iterator const_reverse_nonempty_iterator; + // Another special iterator: it frees memory as it iterates (used to resize) + typedef destructive_two_d_iterator< vector< sparsegroup, + vector_alloc> > + destructive_iterator; + + // Iterator functions + iterator begin() { return iterator(this, 0); } + const_iterator begin() const { return const_iterator(this, 0); } + iterator end() { return iterator(this, size()); } + const_iterator end() const { return const_iterator(this, size()); } + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } + + // Versions for our special non-empty iterator + nonempty_iterator nonempty_begin() { + return nonempty_iterator(groups.begin(), groups.end(), groups.begin()); + } + const_nonempty_iterator nonempty_begin() const { + return const_nonempty_iterator(groups.begin(),groups.end(), groups.begin()); + } + nonempty_iterator nonempty_end() { + return nonempty_iterator(groups.begin(), groups.end(), groups.end()); + } + const_nonempty_iterator nonempty_end() const { + return const_nonempty_iterator(groups.begin(), groups.end(), groups.end()); + } + reverse_nonempty_iterator nonempty_rbegin() { + return reverse_nonempty_iterator(nonempty_end()); + } + const_reverse_nonempty_iterator nonempty_rbegin() const { + return const_reverse_nonempty_iterator(nonempty_end()); + } + reverse_nonempty_iterator nonempty_rend() { + return reverse_nonempty_iterator(nonempty_begin()); + } + const_reverse_nonempty_iterator nonempty_rend() const { + return const_reverse_nonempty_iterator(nonempty_begin()); + } + destructive_iterator destructive_begin() { + return destructive_iterator(groups.begin(), groups.end(), groups.begin()); + } + destructive_iterator destructive_end() { + return destructive_iterator(groups.begin(), groups.end(), groups.end()); + } + + typedef sparsegroup group_type; + typedef vector group_vector_type; + + typedef typename group_vector_type::reference GroupsReference; + typedef typename group_vector_type::const_reference GroupsConstReference; + typedef typename group_vector_type::iterator GroupsIterator; + typedef typename group_vector_type::const_iterator GroupsConstIterator; + + // How to deal with the proper group + static size_type num_groups(size_type num) { // how many to hold num buckets + return num == 0 ? 0 : ((num-1) / GROUP_SIZE) + 1; + } + + u_int16_t pos_in_group(size_type i) const { + return static_cast(i % GROUP_SIZE); + } + size_type group_num(size_type i) const { + return i / GROUP_SIZE; + } + GroupsReference which_group(size_type i) { + return groups[group_num(i)]; + } + GroupsConstReference which_group(size_type i) const { + return groups[group_num(i)]; + } + + public: + // Constructors -- default, normal (when you specify size), and copy + sparsetable(size_type sz = 0, Alloc alloc = Alloc()) + : groups(vector_alloc(alloc)), settings(alloc, sz) { + groups.resize(num_groups(sz), group_type(settings)); + } + // We can get away with using the default copy constructor, + // and default destructor, and hence the default operator=. Huzzah! + + // Many STL algorithms use swap instead of copy constructors + void swap(sparsetable& x) { + STL_NAMESPACE::swap(groups, x.groups); + STL_NAMESPACE::swap(settings.table_size, x.settings.table_size); + STL_NAMESPACE::swap(settings.num_buckets, x.settings.num_buckets); + } + + // It's always nice to be able to clear a table without deallocating it + void clear() { + GroupsIterator group; + for ( group = groups.begin(); group != groups.end(); ++group ) { + group->clear(); + } + settings.num_buckets = 0; + } + + // ACCESSOR FUNCTIONS for the things we templatize on, basically + allocator_type get_allocator() const { + return allocator_type(settings); + } + + + // Functions that tell you about size. + // NOTE: empty() is non-intuitive! It does not tell you the number + // of not-empty buckets (use num_nonempty() for that). Instead + // it says whether you've allocated any buckets or not. + size_type size() const { return settings.table_size; } + size_type max_size() const { return settings.max_size(); } + bool empty() const { return settings.table_size == 0; } + // We also may want to know how many *used* buckets there are + size_type num_nonempty() const { return settings.num_buckets; } + + // OK, we'll let you resize one of these puppies + void resize(size_type new_size) { + groups.resize(num_groups(new_size), group_type(settings)); + if ( new_size < settings.table_size) { + // lower num_buckets, clear last group + if ( pos_in_group(new_size) > 0 ) // need to clear inside last group + groups.back().erase(groups.back().begin() + pos_in_group(new_size), + groups.back().end()); + settings.num_buckets = 0; // refigure # of used buckets + GroupsConstIterator group; + for ( group = groups.begin(); group != groups.end(); ++group ) + settings.num_buckets += group->num_nonempty(); + } + settings.table_size = new_size; + } + + + // We let you see if a bucket is non-empty without retrieving it + bool test(size_type i) const { + return which_group(i).test(pos_in_group(i)); + } + bool test(iterator pos) const { + return which_group(pos.pos).test(pos_in_group(pos.pos)); + } + bool test(const_iterator pos) const { + return which_group(pos.pos).test(pos_in_group(pos.pos)); + } + + // We only return const_references because it's really hard to + // return something settable for empty buckets. Use set() instead. + const_reference get(size_type i) const { + assert(i < settings.table_size); + return which_group(i).get(pos_in_group(i)); + } + + // TODO(csilvers): make protected + friend + // This is used by sparse_hashtable to get an element from the table + // when we know it exists (because the caller has called test(i)). + const_reference unsafe_get(size_type i) const { + assert(i < settings.table_size); + assert(test(i)); + return which_group(i).unsafe_get(pos_in_group(i)); + } + + // TODO(csilvers): make protected + friend element_adaptor + reference mutating_get(size_type i) { // fills bucket i before getting + assert(i < settings.table_size); + size_type old_numbuckets = which_group(i).num_nonempty(); + reference retval = which_group(i).mutating_get(pos_in_group(i)); + settings.num_buckets += which_group(i).num_nonempty() - old_numbuckets; + return retval; + } + + // Syntactic sugar. As in sparsegroup, the non-const version is harder + const_reference operator[](size_type i) const { + return get(i); + } + + element_adaptor operator[](size_type i) { + return element_adaptor(this, i); + } + + // Needed for hashtables, gets as a nonempty_iterator. Crashes for empty bcks + const_nonempty_iterator get_iter(size_type i) const { + assert(test(i)); // how can a nonempty_iterator point to an empty bucket? + return const_nonempty_iterator( + groups.begin(), groups.end(), + groups.begin() + group_num(i), + (groups[group_num(i)].nonempty_begin() + + groups[group_num(i)].pos_to_offset(pos_in_group(i)))); + } + // For nonempty we can return a non-const version + nonempty_iterator get_iter(size_type i) { + assert(test(i)); // how can a nonempty_iterator point to an empty bucket? + return nonempty_iterator( + groups.begin(), groups.end(), + groups.begin() + group_num(i), + (groups[group_num(i)].nonempty_begin() + + groups[group_num(i)].pos_to_offset(pos_in_group(i)))); + } + + + // This returns a reference to the inserted item (which is a copy of val) + // The trick is to figure out whether we're replacing or inserting anew + reference set(size_type i, const_reference val) { + assert(i < settings.table_size); + size_type old_numbuckets = which_group(i).num_nonempty(); + reference retval = which_group(i).set(pos_in_group(i), val); + settings.num_buckets += which_group(i).num_nonempty() - old_numbuckets; + return retval; + } + + // This takes the specified elements out of the table. This is + // "undefining", rather than "clearing". + void erase(size_type i) { + assert(i < settings.table_size); + size_type old_numbuckets = which_group(i).num_nonempty(); + which_group(i).erase(pos_in_group(i)); + settings.num_buckets += which_group(i).num_nonempty() - old_numbuckets; + } + + void erase(iterator pos) { + erase(pos.pos); + } + + void erase(iterator start_it, iterator end_it) { + // This could be more efficient, but then we'd need to figure + // out if we spanned groups or not. Doesn't seem worth it. + for ( ; start_it != end_it; ++start_it ) + erase(start_it); + } + + + // We support reading and writing tables to disk. We don't store + // the actual array contents (which we don't know how to store), + // just the groups and sizes. Returns true if all went ok. + + private: + // Every time the disk format changes, this should probably change too + static const unsigned long MAGIC_NUMBER = 0x24687531; + + // Old versions of this code write all data in 32 bits. We need to + // support these files as well as having support for 64-bit systems. + // So we use the following encoding scheme: for values < 2^32-1, we + // store in 4 bytes in big-endian order. For values > 2^32, we + // store 0xFFFFFFF followed by 8 bytes in big-endian order. This + // causes us to mis-read old-version code that stores exactly + // 0xFFFFFFF, but I don't think that is likely to have happened for + // these particular values. + static bool write_32_or_64(FILE* fp, size_type value) { + if ( value < 0xFFFFFFFFULL ) { // fits in 4 bytes + PUT_(value, 24); + PUT_(value, 16); + PUT_(value, 8); + PUT_(value, 0); + } else if ( value == 0xFFFFFFFFUL ) { // special case in 32bit systems + PUT_(0xFF, 0); PUT_(0xFF, 0); PUT_(0xFF, 0); PUT_(0xFF, 0); // marker + PUT_(0, 0); PUT_(0, 0); PUT_(0, 0); PUT_(0, 0); + PUT_(0xFF, 0); PUT_(0xFF, 0); PUT_(0xFF, 0); PUT_(0xFF, 0); + } else { + PUT_(0xFF, 0); PUT_(0xFF, 0); PUT_(0xFF, 0); PUT_(0xFF, 0); // marker + PUT_(value, 56); + PUT_(value, 48); + PUT_(value, 40); + PUT_(value, 32); + PUT_(value, 24); + PUT_(value, 16); + PUT_(value, 8); + PUT_(value, 0); + } + return true; + } + + static bool read_32_or_64(FILE* fp, size_type *value) { // reads into value + size_type first4 = 0; + int x; + GET_(first4, 24); + GET_(first4, 16); + GET_(first4, 8); + GET_(first4, 0); + if ( first4 < 0xFFFFFFFFULL ) { + *value = first4; + } else { + GET_(*value, 56); + GET_(*value, 48); + GET_(*value, 40); + GET_(*value, 32); + GET_(*value, 24); + GET_(*value, 16); + GET_(*value, 8); + GET_(*value, 0); + } + return true; + } + + public: + bool write_metadata(FILE *fp) const { + if ( !write_32_or_64(fp, MAGIC_NUMBER) ) return false; + if ( !write_32_or_64(fp, settings.table_size) ) return false; + if ( !write_32_or_64(fp, settings.num_buckets) ) return false; + + GroupsConstIterator group; + for ( group = groups.begin(); group != groups.end(); ++group ) + if ( group->write_metadata(fp) == false ) return false; + return true; + } + + // Reading destroys the old table contents! Returns true if read ok. + bool read_metadata(FILE *fp) { + size_type magic_read = 0; + if ( !read_32_or_64(fp, &magic_read) ) return false; + if ( magic_read != MAGIC_NUMBER ) { + clear(); // just to be consistent + return false; + } + + if ( !read_32_or_64(fp, &settings.table_size) ) return false; + if ( !read_32_or_64(fp, &settings.num_buckets) ) return false; + + resize(settings.table_size); // so the vector's sized ok + GroupsIterator group; + for ( group = groups.begin(); group != groups.end(); ++group ) + if ( group->read_metadata(fp) == false ) return false; + return true; + } + + // This code is identical to that for SparseGroup + // If your keys and values are simple enough, we can write them + // to disk for you. "simple enough" means no pointers. + // However, we don't try to normalize endianness + bool write_nopointer_data(FILE *fp) const { + for ( const_nonempty_iterator it = nonempty_begin(); + it != nonempty_end(); ++it ) { + if ( !fwrite(&*it, sizeof(*it), 1, fp) ) return false; + } + return true; + } + + // When reading, we have to override the potential const-ness of *it + bool read_nopointer_data(FILE *fp) { + for ( nonempty_iterator it = nonempty_begin(); + it != nonempty_end(); ++it ) { + if ( !fread(reinterpret_cast(&(*it)), sizeof(*it), 1, fp) ) + return false; + } + return true; + } + + // Comparisons. Note the comparisons are pretty arbitrary: we + // compare values of the first index that isn't equal (using default + // value for empty buckets). + bool operator==(const sparsetable& x) const { + return ( settings.table_size == x.settings.table_size && + settings.num_buckets == x.settings.num_buckets && + groups == x.groups ); + } + bool operator<(const sparsetable& x) const { // also from algobase.h + return STL_NAMESPACE::lexicographical_compare(begin(), end(), + x.begin(), x.end()); + } + bool operator!=(const sparsetable& x) const { return !(*this == x); } + bool operator<=(const sparsetable& x) const { return !(x < *this); } + bool operator>(const sparsetable& x) const { return x < *this; } + bool operator>=(const sparsetable& x) const { return !(*this < x); } + + + private: + // Package allocator with table_size and num_buckets to eliminate memory + // needed for the zero-size allocator. + // If new fields are added to this class, we should add them to + // operator= and swap. + class Settings : public allocator_type { + public: + typedef typename allocator_type::size_type size_type; + + Settings(const allocator_type& a, size_type sz = 0, size_type n = 0) + : allocator_type(a), table_size(sz), num_buckets(n) { } + + Settings(const Settings& s) + : allocator_type(s), + table_size(s.table_size), num_buckets(s.num_buckets) { } + + size_type table_size; // how many buckets they want + size_type num_buckets; // number of non-empty buckets + }; + + // The actual data + group_vector_type groups; // our list of groups + Settings settings; // allocator, table size, buckets +}; + +// We need a global swap as well +template +inline void swap(sparsetable &x, + sparsetable &y) { + x.swap(y); +} + +#undef GET_ +#undef PUT_ + +_END_GOOGLE_NAMESPACE_ + +#endif diff --git a/external/google/type_traits.h b/external/google/type_traits.h new file mode 100644 index 00000000..87729d40 --- /dev/null +++ b/external/google/type_traits.h @@ -0,0 +1,336 @@ +// Copyright (c) 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ---- +// Author: Matt Austern +// +// Define a small subset of tr1 type traits. The traits we define are: +// is_integral +// is_floating_point +// is_pointer +// is_enum +// is_reference +// is_pod +// has_trivial_constructor +// has_trivial_copy +// has_trivial_assign +// has_trivial_destructor +// remove_const +// remove_volatile +// remove_cv +// remove_reference +// add_reference +// remove_pointer +// is_same +// is_convertible +// We can add more type traits as required. + +#ifndef BASE_TYPE_TRAITS_H_ +#define BASE_TYPE_TRAITS_H_ + +#include +#include // For pair + +_START_GOOGLE_NAMESPACE_ + +// integral_constant, defined in tr1, is a wrapper for an integer +// value. We don't really need this generality; we could get away +// with hardcoding the integer type to bool. We use the fully +// general integer_constant for compatibility with tr1. + +template +struct integral_constant { + static const T value = v; + typedef T value_type; + typedef integral_constant type; +}; + +template const T integral_constant::value; + +// Abbreviations: true_type and false_type are structs that represent +// boolean true and false values. +typedef integral_constant true_type; +typedef integral_constant false_type; + +// Types small_ and big_ are guaranteed such that sizeof(small_) < +// sizeof(big_) +typedef char small_; + +struct big_ { + char dummy[2]; +}; + +template struct is_integral; +template struct is_floating_point; +template struct is_pointer; +// MSVC can't compile this correctly, and neither can gcc 3.3.5 (at least) +#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) +// is_enum uses is_convertible, which is not available on MSVC. +template struct is_enum; +#endif +template struct is_reference; +template struct is_pod; +template struct has_trivial_constructor; +template struct has_trivial_copy; +template struct has_trivial_assign; +template struct has_trivial_destructor; +template struct remove_const; +template struct remove_volatile; +template struct remove_cv; +template struct remove_reference; +template struct add_reference; +template struct remove_pointer; +template struct is_same; +#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) +template struct is_convertible; +#endif + +// is_integral is false except for the built-in integer types. +template struct is_integral : false_type { }; +template<> struct is_integral : true_type { }; +template<> struct is_integral : true_type { }; +template<> struct is_integral : true_type { }; +template<> struct is_integral : true_type { }; +#if defined(_MSC_VER) +// wchar_t is not by default a distinct type from unsigned short in +// Microsoft C. +// See http://msdn2.microsoft.com/en-us/library/dh8che7s(VS.80).aspx +template<> struct is_integral<__wchar_t> : true_type { }; +#else +template<> struct is_integral : true_type { }; +#endif +template<> struct is_integral : true_type { }; +template<> struct is_integral : true_type { }; +template<> struct is_integral : true_type { }; +template<> struct is_integral : true_type { }; +template<> struct is_integral : true_type { }; +template<> struct is_integral : true_type { }; +#ifdef HAVE_LONG_LONG +template<> struct is_integral : true_type { }; +template<> struct is_integral : true_type { }; +#endif + + +// is_floating_point is false except for the built-in floating-point types. +template struct is_floating_point : false_type { }; +template<> struct is_floating_point : true_type { }; +template<> struct is_floating_point : true_type { }; +template<> struct is_floating_point : true_type { }; + + +// is_pointer is false except for pointer types. +template struct is_pointer : false_type { }; +template struct is_pointer : true_type { }; + +#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) + +namespace internal { + +template struct is_class_or_union { + template static small_ tester(void (U::*)()); + template static big_ tester(...); + static const bool value = sizeof(tester(0)) == sizeof(small_); +}; + +// is_convertible chokes if the first argument is an array. That's why +// we use add_reference here. +template struct is_enum_impl + : is_convertible::type, int> { }; + +template struct is_enum_impl : false_type { }; + +} // namespace internal + +// Specified by TR1 [4.5.1] primary type categories. + +// Implementation note: +// +// Each type is either void, integral, floating point, array, pointer, +// reference, member object pointer, member function pointer, enum, +// union or class. Out of these, only integral, floating point, reference, +// class and enum types are potentially convertible to int. Therefore, +// if a type is not a reference, integral, floating point or class and +// is convertible to int, it's a enum. +// +// Is-convertible-to-int check is done only if all other checks pass, +// because it can't be used with some types (e.g. void or classes with +// inaccessible conversion operators). +template struct is_enum + : internal::is_enum_impl< + is_same::value || + is_integral::value || + is_floating_point::value || + is_reference::value || + internal::is_class_or_union::value, + T> { }; + +template struct is_enum : is_enum { }; +template struct is_enum : is_enum { }; +template struct is_enum : is_enum { }; + +#endif + +// is_reference is false except for reference types. +template struct is_reference : false_type {}; +template struct is_reference : true_type {}; + + +// We can't get is_pod right without compiler help, so fail conservatively. +// We will assume it's false except for arithmetic types, enumerations, +// pointers and const versions thereof. Note that std::pair is not a POD. +template struct is_pod + : integral_constant::value || + is_floating_point::value || +#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) + // is_enum is not available on MSVC. + is_enum::value || +#endif + is_pointer::value)> { }; +template struct is_pod : is_pod { }; + + +// We can't get has_trivial_constructor right without compiler help, so +// fail conservatively. We will assume it's false except for: (1) types +// for which is_pod is true. (2) std::pair of types with trivial +// constructors. (3) array of a type with a trivial constructor. +// (4) const versions thereof. +template struct has_trivial_constructor : is_pod { }; +template struct has_trivial_constructor > + : integral_constant::value && + has_trivial_constructor::value)> { }; +template struct has_trivial_constructor + : has_trivial_constructor { }; +template struct has_trivial_constructor + : has_trivial_constructor { }; + +// We can't get has_trivial_copy right without compiler help, so fail +// conservatively. We will assume it's false except for: (1) types +// for which is_pod is true. (2) std::pair of types with trivial copy +// constructors. (3) array of a type with a trivial copy constructor. +// (4) const versions thereof. +template struct has_trivial_copy : is_pod { }; +template struct has_trivial_copy > + : integral_constant::value && + has_trivial_copy::value)> { }; +template struct has_trivial_copy + : has_trivial_copy { }; +template struct has_trivial_copy : has_trivial_copy { }; + +// We can't get has_trivial_assign right without compiler help, so fail +// conservatively. We will assume it's false except for: (1) types +// for which is_pod is true. (2) std::pair of types with trivial copy +// constructors. (3) array of a type with a trivial assign constructor. +template struct has_trivial_assign : is_pod { }; +template struct has_trivial_assign > + : integral_constant::value && + has_trivial_assign::value)> { }; +template struct has_trivial_assign + : has_trivial_assign { }; + +// We can't get has_trivial_destructor right without compiler help, so +// fail conservatively. We will assume it's false except for: (1) types +// for which is_pod is true. (2) std::pair of types with trivial +// destructors. (3) array of a type with a trivial destructor. +// (4) const versions thereof. +template struct has_trivial_destructor : is_pod { }; +template struct has_trivial_destructor > + : integral_constant::value && + has_trivial_destructor::value)> { }; +template struct has_trivial_destructor + : has_trivial_destructor { }; +template struct has_trivial_destructor + : has_trivial_destructor { }; + +// Specified by TR1 [4.7.1] +template struct remove_const { typedef T type; }; +template struct remove_const { typedef T type; }; +template struct remove_volatile { typedef T type; }; +template struct remove_volatile { typedef T type; }; +template struct remove_cv { + typedef typename remove_const::type>::type type; +}; + + +// Specified by TR1 [4.7.2] Reference modifications. +template struct remove_reference { typedef T type; }; +template struct remove_reference { typedef T type; }; + +template struct add_reference { typedef T& type; }; +template struct add_reference { typedef T& type; }; + +// Specified by TR1 [4.7.4] Pointer modifications. +template struct remove_pointer { typedef T type; }; +template struct remove_pointer { typedef T type; }; +template struct remove_pointer { typedef T type; }; +template struct remove_pointer { typedef T type; }; +template struct remove_pointer { + typedef T type; }; + +// Specified by TR1 [4.6] Relationships between types +template struct is_same : public false_type { }; +template struct is_same : public true_type { }; + +// Specified by TR1 [4.6] Relationships between types +#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) +namespace internal { + +// This class is an implementation detail for is_convertible, and you +// don't need to know how it works to use is_convertible. For those +// who care: we declare two different functions, one whose argument is +// of type To and one with a variadic argument list. We give them +// return types of different size, so we can use sizeof to trick the +// compiler into telling us which function it would have chosen if we +// had called it with an argument of type From. See Alexandrescu's +// _Modern C++ Design_ for more details on this sort of trick. + +template +struct ConvertHelper { + static small_ Test(To); + static big_ Test(...); + static From Create(); +}; +} // namespace internal + +// Inherits from true_type if From is convertible to To, false_type otherwise. +template +struct is_convertible + : integral_constant::Test( + internal::ConvertHelper::Create())) + == sizeof(small_)> { +}; +#endif + +_END_GOOGLE_NAMESPACE_ + +#endif // BASE_TYPE_TRAITS_H_ diff --git a/include/INode.h b/include/INode.h index 4ee6ce3b..10225432 100644 --- a/include/INode.h +++ b/include/INode.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2013 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -22,6 +22,7 @@ public: virtual void peerCountUpdated(size_t count) {} virtual void localBlockchainUpdated(uint64_t height) {} virtual void lastKnownBlockHeightUpdated(uint64_t height) {} + virtual void poolChanged() {} }; struct OutEntry { @@ -34,6 +35,12 @@ struct OutsForAmount { std::vector outs; }; +struct BlockCompleteEntry { + crypto::hash blockHash; + cryptonote::blobdata block; + std::list txs; +}; + class INode { public: typedef std::function Callback; @@ -48,11 +55,14 @@ public: virtual size_t getPeerCount() const = 0; virtual uint64_t getLastLocalBlockHeight() const = 0; virtual uint64_t getLastKnownBlockHeight() const = 0; + virtual uint64_t getLastLocalBlockTimestamp() const = 0; - virtual void relayTransaction(const cryptonote::transaction& transaction, const Callback& callback) = 0; + virtual void relayTransaction(const cryptonote::Transaction& transaction, const Callback& callback) = 0; virtual void getRandomOutsByAmounts(std::vector&& amounts, uint64_t outsCount, std::vector& result, const Callback& callback) = 0; virtual void getNewBlocks(std::list&& knownBlockIds, std::list& newBlocks, uint64_t& startHeight, const Callback& callback) = 0; virtual void getTransactionOutsGlobalIndices(const crypto::hash& transactionHash, std::vector& outsGlobalIndices, const Callback& callback) = 0; + virtual void queryBlocks(std::list&& knownBlockIds, uint64_t timestamp, std::list& newBlocks, uint64_t& startHeight, const Callback& callback) = 0; + virtual void getPoolSymmetricDifference(std::vector&& known_pool_tx_ids, crypto::hash known_block_id, bool& is_bc_actual, std::vector& new_txs, std::vector& deleted_tx_ids, const Callback& callback) = 0; }; } diff --git a/include/IObservable.h b/include/IObservable.h new file mode 100644 index 00000000..a16551ff --- /dev/null +++ b/include/IObservable.h @@ -0,0 +1,16 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +namespace CryptoNote { + +template +class IObservable { +public: + virtual void addObserver(T* observer) = 0; + virtual void removeObserver(T* observer) = 0; +}; + +} diff --git a/include/IStreamSerializable.h b/include/IStreamSerializable.h new file mode 100644 index 00000000..93836500 --- /dev/null +++ b/include/IStreamSerializable.h @@ -0,0 +1,17 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include + +namespace CryptoNote { + +class IStreamSerializable { +public: + virtual void save(std::ostream& os) = 0; + virtual void load(std::istream& in) = 0; +}; + +} diff --git a/include/ITransaction.h b/include/ITransaction.h new file mode 100644 index 00000000..58ffa159 --- /dev/null +++ b/include/ITransaction.h @@ -0,0 +1,167 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include + +namespace CryptoNote { + +typedef std::array PublicKey; +typedef std::array SecretKey; +typedef std::array KeyImage; +typedef std::array Hash; +typedef std::vector Blob; + +struct AccountAddress { + PublicKey spendPublicKey; + PublicKey viewPublicKey; +}; + +struct AccountKeys { + AccountAddress address; + SecretKey spendSecretKey; + SecretKey viewSecretKey; +}; + +struct KeyPair { + PublicKey publicKey; + SecretKey secretKey; +}; + +namespace TransactionTypes { + + enum class InputType : uint8_t { Invalid, Key, Multisignature, Generating }; + enum class OutputType : uint8_t { Invalid, Key, Multisignature }; + + struct InputKey { + uint64_t amount; + std::vector keyOffsets; + KeyImage keyImage; // double spending protection + }; + + struct InputMultisignature { + uint64_t amount; + uint32_t signatures; + uint64_t outputIndex; + }; + + struct OutputKey { + uint64_t amount; + PublicKey key; + }; + + struct OutputMultisignature { + uint64_t amount; + std::vector keys; + uint32_t requiredSignatures; + }; + + struct GlobalOutput { + PublicKey targetKey; + uint64_t outputIndex; + }; + + typedef std::vector GlobalOutputsContainer; + + struct OutputKeyInfo { + PublicKey transactionPublicKey; + size_t transactionIndex; + size_t outputInTransaction; + }; + + struct InputKeyInfo { + uint64_t amount; + GlobalOutputsContainer outputs; + OutputKeyInfo realOutput; + }; + +} + +// +// ITransactionReader +// +class ITransactionReader { +public: + virtual ~ITransactionReader() { } + + virtual Hash getTransactionHash() const = 0; + virtual Hash getTransactionPrefixHash() const = 0; + virtual PublicKey getTransactionPublicKey() const = 0; + virtual uint64_t getUnlockTime() const = 0; + + // extra + virtual bool getPaymentId(Hash& paymentId) const = 0; + virtual bool getExtraNonce(std::string& nonce) const = 0; + + // inputs + virtual size_t getInputCount() const = 0; + virtual uint64_t getInputTotalAmount() const = 0; + virtual TransactionTypes::InputType getInputType(size_t index) const = 0; + virtual void getInput(size_t index, TransactionTypes::InputKey& input) const = 0; + virtual void getInput(size_t index, TransactionTypes::InputMultisignature& input) const = 0; + + // outputs + virtual size_t getOutputCount() const = 0; + virtual uint64_t getOutputTotalAmount() const = 0; + virtual TransactionTypes::OutputType getOutputType(size_t index) const = 0; + virtual void getOutput(size_t index, TransactionTypes::OutputKey& output) const = 0; + virtual void getOutput(size_t index, TransactionTypes::OutputMultisignature& output) const = 0; + + // signatures + virtual size_t getRequiredSignaturesCount(size_t inputIndex) const = 0; + virtual bool findOutputsToAccount(const AccountAddress& addr, const SecretKey& viewSecretKey, std::vector& outs, uint64_t& outputAmount) const = 0; + + // various checks + virtual bool validateInputs() const = 0; + virtual bool validateOutputs() const = 0; + virtual bool validateSignatures() const = 0; + + // serialized transaction + virtual Blob getTransactionData() const = 0; +}; + +// +// ITransactionWriter +// +class ITransactionWriter { +public: + + virtual ~ITransactionWriter() { } + + // transaction parameters + virtual void setUnlockTime(uint64_t unlockTime) = 0; + + // extra + virtual void setPaymentId(const Hash& paymentId) = 0; + virtual void setExtraNonce(const std::string& nonce) = 0; + + // Inputs/Outputs + virtual size_t addInput(const TransactionTypes::InputKey& input) = 0; + virtual size_t addInput(const AccountKeys& senderKeys, const TransactionTypes::InputKeyInfo& info, KeyPair& ephKeys) = 0; + virtual size_t addInput(const TransactionTypes::InputMultisignature& input) = 0; + + virtual size_t addOutput(uint64_t amount, const AccountAddress& to) = 0; + virtual size_t addOutput(uint64_t amount, const std::vector& to, uint32_t requiredSignatures) = 0; + + // transaction info + virtual bool getTransactionSecretKey(SecretKey& key) const = 0; + virtual void setTransactionSecretKey(const SecretKey& key) = 0; + + // signing + virtual void signInputKey(size_t input, const TransactionTypes::InputKeyInfo& info, const KeyPair& ephKeys) = 0; + virtual void signInputMultisignature(size_t input, const PublicKey& sourceTransactionKey, size_t outputIndex, const AccountKeys& accountKeys) = 0; +}; + +class ITransaction : + public ITransactionReader, + public ITransactionWriter { +public: + virtual ~ITransaction() { } + +}; + +} diff --git a/include/ITransfersContainer.h b/include/ITransfersContainer.h new file mode 100644 index 00000000..549b363d --- /dev/null +++ b/include/ITransfersContainer.h @@ -0,0 +1,92 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include +#include "crypto/hash.h" +#include "ITransaction.h" +#include "IObservable.h" +#include "IStreamSerializable.h" + +namespace CryptoNote { + +const uint64_t UNCONFIRMED_TRANSACTION_GLOBAL_OUTPUT_INDEX = std::numeric_limits::max(); + +struct TransactionInformation { + // transaction info + Hash transactionHash; + PublicKey publicKey; + uint64_t blockHeight; + uint64_t timestamp; + uint64_t unlockTime; + uint64_t totalAmountIn; + uint64_t totalAmountOut; + std::vector extra; + Hash paymentId; +}; + + +struct TransactionOutputInformation { + // output info + TransactionTypes::OutputType type; + uint64_t amount; + uint64_t globalOutputIndex; + uint32_t outputInTransaction; + + // transaction info + Hash transactionHash; + PublicKey transactionPublicKey; + + union { + PublicKey outputKey; // Type: Key + uint32_t requiredSignatures; // Type: Multisignature + }; +}; + +struct TransactionSpentOutputInformation: public TransactionOutputInformation { + uint64_t spendingBlockHeight; + uint64_t timestamp; + Hash spendingTransactionHash; + KeyImage keyImage; //!< \attention Used only for TransactionTypes::OutputType::Key + uint32_t inputInTransaction; +}; + +class ITransfersContainer : public IStreamSerializable { +public: + enum Flags : uint32_t { + // state + IncludeStateUnlocked = 0x01, + IncludeStateLocked = 0x02, + IncludeStateSoftLocked = 0x04, + // output type + IncludeTypeKey = 0x100, + IncludeTypeMultisignature = 0x200, + // combinations + IncludeStateAll = 0xff, + IncludeTypeAll = 0xff00, + + IncludeKeyUnlocked = IncludeTypeKey | IncludeStateUnlocked, + IncludeKeyNotUnlocked = IncludeTypeKey | IncludeStateLocked | IncludeStateSoftLocked, + + IncludeAllLocked = IncludeTypeAll | IncludeStateLocked | IncludeStateSoftLocked, + IncludeAllUnlocked = IncludeTypeAll | IncludeStateUnlocked, + IncludeAll = IncludeTypeAll | IncludeStateAll, + + IncludeDefault = IncludeKeyUnlocked + }; + + virtual size_t transfersCount() = 0; + virtual size_t transactionsCount() = 0; + virtual uint64_t balance(uint32_t flags = IncludeDefault) = 0; + virtual void getOutputs(std::vector& transfers, uint32_t flags = IncludeDefault) = 0; + virtual bool getTransactionInformation(const Hash& transactionHash, TransactionInformation& info, int64_t& txBalance) = 0; + virtual std::vector getTransactionOutputs(const Hash& transactionHash, uint32_t flags = IncludeDefault) = 0; + virtual void getUnconfirmedTransactions(std::vector& transactions) = 0; + virtual std::vector getSpentOutputs() = 0; +}; + +} diff --git a/include/ITransfersSynchronizer.h b/include/ITransfersSynchronizer.h new file mode 100644 index 00000000..349e4a71 --- /dev/null +++ b/include/ITransfersSynchronizer.h @@ -0,0 +1,62 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include + +#include "ITransaction.h" +#include "ITransfersContainer.h" +#include "IStreamSerializable.h" + +namespace CryptoNote { + +struct SynchronizationStart { + uint64_t timestamp; + uint64_t height; +}; + +struct AccountSubscription { + AccountKeys keys; + SynchronizationStart syncStart; + size_t transactionSpendableAge; +}; + +class ITransfersSubscription; + +class ITransfersObserver { +public: + virtual void onError(ITransfersSubscription* object, + uint64_t height, std::error_code ec) {} + + virtual void onTransactionUpdated(ITransfersSubscription* object, const Hash& transactionHash) {} + + /** + * \note The sender must guarantee that onTransactionDeleted() is called only after onTransactionUpdated() is called + * for the same \a transactionHash. + */ + virtual void onTransactionDeleted(ITransfersSubscription* object, const Hash& transactionHash) { } +}; + +class ITransfersSubscription : public IObservable < ITransfersObserver > { +public: + virtual ~ITransfersSubscription() {} + + virtual AccountAddress getAddress() = 0; + virtual ITransfersContainer& getContainer() = 0; +}; + +class ITransfersSynchronizer : public IStreamSerializable { +public: + virtual ~ITransfersSynchronizer() {} + + virtual ITransfersSubscription& addSubscription(const AccountSubscription& acc) = 0; + virtual bool removeSubscription(const AccountAddress& acc) = 0; + virtual void getSubscriptions(std::vector& subscriptions) = 0; + // returns nullptr if address is not found + virtual ITransfersSubscription* getSubscription(const AccountAddress& acc) = 0; +}; + +} diff --git a/include/IWallet.h b/include/IWallet.h index a89538d8..b99f3351 100644 --- a/include/IWallet.h +++ b/include/IWallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2013 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -28,23 +28,48 @@ const TransactionId INVALID_TRANSACTION_ID = std::numeric_limits::max(); const uint64_t UNCONFIRMED_TRANSACTION_HEIGHT = std::numeric_limits::max(); -struct Transaction { - TransferId firstTransferId; - size_t transferCount; - int64_t totalAmount; - uint64_t fee; - TransactionHash hash; - bool isCoinbase; - uint64_t blockHeight; - uint64_t timestamp; - std::string extra; +enum class TransactionState : uint8_t { + Active, // --> {Deleted} + Deleted, // --> {Active} + + Sending, // --> {Active, Cancelled, Failed} + Cancelled, // --> {} + Failed // --> {} +}; + +struct TransactionInfo { + TransferId firstTransferId; + size_t transferCount; + int64_t totalAmount; + uint64_t fee; + uint64_t sentTime; + uint64_t unlockTime; + TransactionHash hash; + bool isCoinbase; + uint64_t blockHeight; + uint64_t timestamp; + std::string extra; + TransactionState state; +}; + +typedef std::array WalletPublicKey; +typedef std::array WalletSecretKey; + +struct WalletAccountKeys { + WalletPublicKey viewPublicKey; + WalletSecretKey viewSecretKey; + WalletPublicKey spendPublicKey; + WalletSecretKey spendSecretKey; }; class IWalletObserver { public: + virtual ~IWalletObserver() {} + virtual void initCompleted(std::error_code result) {} virtual void saveCompleted(std::error_code result) {} - virtual void synchronizationProgressUpdated(uint64_t current, uint64_t total, std::error_code result) {} + virtual void synchronizationProgressUpdated(uint64_t current, uint64_t total) {} + virtual void synchronizationCompleted(std::error_code result) {} virtual void actualBalanceUpdated(uint64_t actualBalance) {} virtual void pendingBalanceUpdated(uint64_t pendingBalance) {} virtual void externalTransactionCreated(TransactionId transactionId) {} @@ -60,7 +85,9 @@ public: virtual void initAndGenerate(const std::string& password) = 0; virtual void initAndLoad(std::istream& source, const std::string& password) = 0; + virtual void initWithKeys(const WalletAccountKeys& accountKeys, const std::string& password) = 0; virtual void shutdown() = 0; + virtual void reset() = 0; virtual void save(std::ostream& destination, bool saveDetailed = true, bool saveCache = true) = 0; @@ -76,12 +103,14 @@ public: virtual TransactionId findTransactionByTransferId(TransferId transferId) = 0; - virtual bool getTransaction(TransactionId transactionId, Transaction& transaction) = 0; + virtual bool getTransaction(TransactionId transactionId, TransactionInfo& transaction) = 0; virtual bool getTransfer(TransferId transferId, Transfer& transfer) = 0; virtual TransactionId sendTransaction(const Transfer& transfer, uint64_t fee, const std::string& extra = "", uint64_t mixIn = 0, uint64_t unlockTimestamp = 0) = 0; virtual TransactionId sendTransaction(const std::vector& transfers, uint64_t fee, const std::string& extra = "", uint64_t mixIn = 0, uint64_t unlockTimestamp = 0) = 0; virtual std::error_code cancelTransaction(size_t transferId) = 0; + + virtual void getAccountKeys(WalletAccountKeys& keys) = 0; }; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt old mode 100644 new mode 100755 index fb38e22c..e306a45d --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,7 +2,7 @@ add_definitions(-DSTATICLIB) file(GLOB_RECURSE COMMON common/*) file(GLOB_RECURSE CRYPTO crypto/*) -file(GLOB_RECURSE CRYPTONOTE_CORE cryptonote_core/*) +file(GLOB_RECURSE CRYPTONOTE_CORE cryptonote_core/* cryptonote_config.h) file(GLOB_RECURSE CRYPTONOTE_PROTOCOL cryptonote_protocol/*) file(GLOB_RECURSE DAEMON daemon/*) file(GLOB_RECURSE P2P p2p/*) @@ -12,6 +12,19 @@ file(GLOB_RECURSE CONN_TOOL connectivity_tool/*) file(GLOB_RECURSE WALLET wallet/*) file(GLOB_RECURSE MINER miner/*) file(GLOB_RECURSE NODE_RPC_PROXY node_rpc_proxy/*) +file(GLOB_RECURSE TRANSFERS transfers/*) +if(MSVC) +file(GLOB_RECURSE SYSTEM Platform/Windows/System/*) +elseif(APPLE) +file(GLOB_RECURSE SYSTEM Platform/OSX/System/*) +else() +file(GLOB_RECURSE SYSTEM Platform/Linux/System/*) +endif() +file(GLOB_RECURSE SERIALIZATION serialization/*) +file(GLOB_RECURSE LOGGER logger/*) +file(GLOB_RECURSE INPROCESS_NODE inprocess_node/*) +file(GLOB_RECURSE HTTP HTTP/*) + source_group(common FILES ${COMMON}) source_group(crypto FILES ${CRYPTO}) @@ -20,33 +33,43 @@ source_group(cryptonote_protocol FILES ${CRYPTONOTE_PROTOCOL}) source_group(daemon FILES ${DAEMON}) source_group(p2p FILES ${P2P}) source_group(rpc FILES ${RPC}) +source_group(System FILES ${SYSTEM} ${HTTP}) source_group(simplewallet FILES ${SIMPLEWALLET}) source_group(connectivity-tool FILES ${CONN_TOOL}) source_group(wallet FILES ${WALLET}) source_group(simpleminer FILES ${MINER}) source_group(node_rpc_proxy FILES ${NODE_RPC_PROXY}) +source_group(transfers FILES ${TRANSFERS}) +source_group(logger FILES ${LOGGER}) +source_group(inprocess_node FILES ${INPROCESS_NODE}) add_library(common ${COMMON}) add_library(crypto ${CRYPTO}) +add_library(serialization ${SERIALIZATION}) add_library(cryptonote_core ${CRYPTONOTE_CORE}) +add_library(node_rpc_proxy ${NODE_RPC_PROXY}) +add_library(inprocess_node ${INPROCESS_NODE}) add_executable(daemon ${DAEMON} ${P2P} ${CRYPTONOTE_PROTOCOL}) add_executable(connectivity_tool ${CONN_TOOL}) add_executable(simpleminer ${MINER}) -target_link_libraries(daemon rpc cryptonote_core crypto common upnpc-static ${Boost_LIBRARIES}) -target_link_libraries(connectivity_tool cryptonote_core crypto common ${Boost_LIBRARIES}) -target_link_libraries(simpleminer cryptonote_core crypto common ${Boost_LIBRARIES}) +target_link_libraries(daemon epee rpc cryptonote_core crypto common upnpc-static serialization ${Boost_LIBRARIES}) +target_link_libraries(connectivity_tool epee rpc cryptonote_core crypto common serialization ${Boost_LIBRARIES}) +target_link_libraries(simpleminer epee cryptonote_core crypto common serialization ${Boost_LIBRARIES}) add_library(rpc ${RPC}) +add_library(System ${SYSTEM} ${HTTP} System/TcpStream.cpp System/TcpStream.h) add_library(wallet ${WALLET}) add_executable(simplewallet ${SIMPLEWALLET} ) -target_link_libraries(simplewallet wallet rpc cryptonote_core crypto common upnpc-static ${Boost_LIBRARIES}) -add_library(node_rpc_proxy ${NODE_RPC_PROXY}) +target_link_libraries(simplewallet epee wallet transfers rpc cryptonote_core crypto common upnpc-static node_rpc_proxy serialization ${Boost_LIBRARIES}) +add_library(logger ${LOGGER}) +add_library(transfers ${TRANSFERS}) + +add_dependencies(connectivity_tool version) add_dependencies(daemon version) add_dependencies(rpc version) add_dependencies(simplewallet version) -set_property(TARGET common crypto cryptonote_core rpc wallet node_rpc_proxy PROPERTY FOLDER "libs") +set_property(TARGET common crypto cryptonote_core rpc System wallet node_rpc_proxy serialization logger transfers inprocess_node PROPERTY FOLDER "libs") set_property(TARGET daemon simplewallet connectivity_tool simpleminer PROPERTY FOLDER "prog") #TODO Specify the name of daemon for your currency -#set_property(TARGET daemon PROPERTY OUTPUT_NAME "cryptonoted") - +set_property(TARGET daemon PROPERTY OUTPUT_NAME "cryptonoted") diff --git a/src/HTTP/HttpParser.cpp b/src/HTTP/HttpParser.cpp new file mode 100644 index 00000000..bc239d91 --- /dev/null +++ b/src/HTTP/HttpParser.cpp @@ -0,0 +1,202 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "HttpParser.h" + +#include + +namespace cryptonote { + +HttpResponse::HTTP_STATUS HttpParser::parseResponseStatusFromString(const std::string& status) { + if (status == "200 OK" || status == "200 Ok") return cryptonote::HttpResponse::STATUS_200; + else if (status == "404 Not Found") return cryptonote::HttpResponse::STATUS_404; + else if (status == "500 Internal Server Error") return cryptonote::HttpResponse::STATUS_500; + else throw std::runtime_error("Unknown HTTP status code is given"); + + return cryptonote::HttpResponse::STATUS_200; //unaccessible +} + + +void HttpParser::receiveRequest(std::istream& stream, HttpRequest& request) { + readWord(stream, request.method); + readWord(stream, request.url); + + std::string httpVersion; + readWord(stream, httpVersion); + + readHeaders(stream, request.headers); + + std::string body; + size_t bodyLen = getBodyLen(request.headers); + if (bodyLen) { + readBody(stream, request.body, bodyLen); + } +} + + +void HttpParser::receiveResponse(std::istream& stream, HttpResponse& response) { + std::string httpVersion; + readWord(stream, httpVersion); + + std::string status; + char c; + + stream.get(c); + while (stream.good() && c != '\r') { //Till the end + status += c; + stream.get(c); + } + + if (!stream.good()) { + throw std::runtime_error("Parser error: stream is not good"); + } + + if (c == '\r') { + stream.get(c); + if (c != '\n') { + throw std::runtime_error("Parser error: '\\n' symbol is expected"); + } + } + + response.setStatus(parseResponseStatusFromString(status)); + + std::string name; + std::string value; + + while (readHeader(stream, name, value)) { + response.addHeader(name, value); + name.clear(); + value.clear(); + } + + response.addHeader(name, value); + auto headers = response.getHeaders(); + size_t length = 0; + auto it = headers.find("Content-Length"); + if (it != headers.end()) { + length = std::stoul(it->second); + } + + std::string body; + if (length) { + readBody(stream, body, length); + } + + response.setBody(body); +} + + +void HttpParser::readWord(std::istream& stream, std::string& word) { + char c; + + stream.get(c); + while (stream.good() && c != ' ' && c != '\r') { + word += c; + stream.get(c); + } + + if (!stream.good()) { + throw std::runtime_error("Parser error: stream is not good"); + } + + if (c == '\r') { + stream.get(c); + if (c != '\n') { + throw std::runtime_error("Parser error: '\\n' symbol is expected"); + } + } +} + +void HttpParser::readHeaders(std::istream& stream, HttpRequest::Headers& headers) { + std::string name; + std::string value; + + while (readHeader(stream, name, value)) { + headers[name] = value; //use insert + name.clear(); + value.clear(); + } + + headers[name] = value; //use insert +} + +bool HttpParser::readHeader(std::istream& stream, std::string& name, std::string& value) { + char c; + bool isName = true; + + stream.get(c); + while (stream.good() && c != '\r') { + if (c == ':') { + if (stream.peek() == ' ') { + stream.get(c); + } + + if (name.empty()) { + throw std::runtime_error("Header name must be not empty"); + } + + if (isName) { + isName = false; + stream.get(c); + continue; + } + } + + if (isName) { + name += c; + stream.get(c); + } else { + value += c; + stream.get(c); + } + } + + if (!stream.good()) { + throw std::runtime_error("Parser error: stream is not good"); + } + + stream.get(c); + if (c != '\n') { + throw std::runtime_error("Parser error: '\\n' symbol is expected"); + } + + c = stream.peek(); + if (c == '\r') { + stream.get(c).get(c); + if (c != '\n') { + throw std::runtime_error("Parser error: '\\n' symbol is expected"); + } + + return false; //no more headers + } + + return true; +} + +size_t HttpParser::getBodyLen(const HttpRequest::Headers& headers) { + auto it = headers.find("Content-Length"); + if (it != headers.end()) { + size_t bytes = std::stoul(it->second); + return bytes; + } + + return 0; +} + +void HttpParser::readBody(std::istream& stream, std::string& body, const size_t bodyLen) { + size_t read = 0; + + while (stream.good() && read < bodyLen) { + body += stream.get(); + ++read; + } + + if (!stream.good()) { + throw std::runtime_error("stream is not good"); + } +} + +} + + diff --git a/src/HTTP/HttpParser.h b/src/HTTP/HttpParser.h new file mode 100644 index 00000000..5f016c35 --- /dev/null +++ b/src/HTTP/HttpParser.h @@ -0,0 +1,34 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef HTTPPARSER_H_ +#define HTTPPARSER_H_ + +#include +#include +#include +#include "HttpRequest.h" +#include "HttpResponse.h" + +namespace cryptonote { + +//Blocking HttpParser +class HttpParser { +public: + HttpParser() {}; + + void receiveRequest(std::istream& stream, HttpRequest& request); + void receiveResponse(std::istream& stream, HttpResponse& response); + static HttpResponse::HTTP_STATUS parseResponseStatusFromString(const std::string& status); +private: + void readWord(std::istream& stream, std::string& word); + void readHeaders(std::istream& stream, HttpRequest::Headers &headers); + bool readHeader(std::istream& stream, std::string& name, std::string& value); + size_t getBodyLen(const HttpRequest::Headers& headers); + void readBody(std::istream& stream, std::string& body, const size_t bodyLen); +}; + +} //namespace cryptonote + +#endif /* HTTPPARSER_H_ */ diff --git a/src/HTTP/HttpRequest.cpp b/src/HTTP/HttpRequest.cpp new file mode 100644 index 00000000..7b17c7f5 --- /dev/null +++ b/src/HTTP/HttpRequest.cpp @@ -0,0 +1,60 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "HttpRequest.h" + +namespace cryptonote { + + const std::string& HttpRequest::getMethod() const { + return method; + } + + const std::string& HttpRequest::getUrl() const { + return url; + } + + const HttpRequest::Headers& HttpRequest::getHeaders() const { + return headers; + } + + const std::string& HttpRequest::getBody() const { + return body; + } + + void HttpRequest::addHeader(const std::string& name, const std::string& value) { + headers[name] = value; + } + void HttpRequest::setBody(const std::string& b) { + body = b; + if (!body.empty()) { + headers["Content-Length"] = std::to_string(body.size()); + } + else { + headers.erase("Content-Length"); + } + } + + void HttpRequest::setUrl(const std::string& u) { + url = u; + } + + std::ostream& HttpRequest::printHttpRequest(std::ostream& os) const { + os << "POST " << url << " HTTP/1.1\r\n"; + auto host = headers.find("Host"); + if (host == headers.end()) { + os << "Host: " << "127.0.0.1" << "\r\n"; + } + + for (auto pair : headers) { + os << pair.first << ": " << pair.second << "\r\n"; + } + + os << "\r\n"; + if (!body.empty()) { + os << body; + } + + return os; + } +} \ No newline at end of file diff --git a/src/HTTP/HttpRequest.h b/src/HTTP/HttpRequest.h new file mode 100644 index 00000000..4045b399 --- /dev/null +++ b/src/HTTP/HttpRequest.h @@ -0,0 +1,40 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include + +namespace cryptonote { + class HttpRequest { + public: + typedef std::map Headers; + + const std::string& getMethod() const; + const std::string& getUrl() const; + const Headers& getHeaders() const; + const std::string& getBody() const; + + void addHeader(const std::string& name, const std::string& value); + void setBody(const std::string& b); + void setUrl(const std::string& uri); + + private: + friend class HttpParser; + + std::string method; + std::string url; + Headers headers; + std::string body; + + friend std::ostream& operator<<(std::ostream& os, const HttpRequest& resp); + std::ostream& printHttpRequest(std::ostream& os) const; + }; + + inline std::ostream& operator<<(std::ostream& os, const HttpRequest& resp) { + return resp.printHttpRequest(os); + } +} \ No newline at end of file diff --git a/src/HTTP/HttpResponse.cpp b/src/HTTP/HttpResponse.cpp new file mode 100644 index 00000000..e948698b --- /dev/null +++ b/src/HTTP/HttpResponse.cpp @@ -0,0 +1,70 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "HttpResponse.h" + +#include + +namespace { + +const char* getStatusString(cryptonote::HttpResponse::HTTP_STATUS status) { + switch (status) { + case cryptonote::HttpResponse::STATUS_200: + return "200 OK"; + case cryptonote::HttpResponse::STATUS_404: + return "404 Not Found"; + case cryptonote::HttpResponse::STATUS_500: + return "500 Internal Server Error"; + default: + throw std::runtime_error("Unknown HTTP status code is given"); + } + + return ""; //unaccessible +} + + +} //namespace + +namespace cryptonote { + +HttpResponse::HttpResponse() { + status = STATUS_200; + headers["Server"] = "Cryptonote-based HTTP server"; +} + +void HttpResponse::setStatus(HTTP_STATUS s) { + status = s; +} + +void HttpResponse::addHeader(const std::string& name, const std::string& value) { + headers[name] = value; +} + +void HttpResponse::setBody(const std::string& b) { + body = b; + if (!body.empty()) { + headers["Content-Length"] = std::to_string(body.size()); + } else { + headers.erase("Content-Length"); + } +} + +std::ostream& HttpResponse::printHttpResponse(std::ostream& os) const { + os << "HTTP/1.1 " << getStatusString(status) << "\r\n"; + + for (auto pair: headers) { + os << pair.first << ": " << pair.second << "\r\n"; + } + os << "\r\n"; + + if (!body.empty()) { + os << body; + } + + return os; +} + +} //namespace cryptonote + + diff --git a/src/HTTP/HttpResponse.h b/src/HTTP/HttpResponse.h new file mode 100644 index 00000000..a4fc6561 --- /dev/null +++ b/src/HTTP/HttpResponse.h @@ -0,0 +1,44 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include + +namespace cryptonote { + + class HttpResponse { + public: + enum HTTP_STATUS { + STATUS_200, + STATUS_404, + STATUS_500 + }; + + HttpResponse(); + + void setStatus(HTTP_STATUS s); + void addHeader(const std::string& name, const std::string& value); + void setBody(const std::string& b); + + const std::map& getHeaders() const { return headers; } + HTTP_STATUS getStatus() const { return status; } + const std::string& getBody() const { return body; } + + private: + friend std::ostream& operator<<(std::ostream& os, const HttpResponse& resp); + std::ostream& printHttpResponse(std::ostream& os) const; + + HTTP_STATUS status; + std::map headers; + std::string body; + }; + + inline std::ostream& operator<<(std::ostream& os, const HttpResponse& resp) { + return resp.printHttpResponse(os); + } + +} //namespace cryptonote diff --git a/src/Platform/Linux/System/Dispatcher.cpp b/src/Platform/Linux/System/Dispatcher.cpp new file mode 100644 index 00000000..33f4b7b0 --- /dev/null +++ b/src/Platform/Linux/System/Dispatcher.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "Dispatcher.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace System; + +void Dispatcher::contextProcedureStatic(void *context) { + reinterpret_cast(context)->contextProcedure(); +} + +Dispatcher::Dispatcher() { + epoll = ::epoll_create1(0); + if (epoll == -1) { + std::cerr << "kqueue() fail errno=" << errno << std::endl; + } else { + currentContext = new ucontext_t; + if (getcontext(reinterpret_cast(currentContext)) == -1) { + std::cerr << "getcontext() fail errno=" << errno << std::endl; + } else { + contextCount = 0; + return; + } + } + throw std::runtime_error("Dispatcher::Dispatcher"); +} + +Dispatcher::~Dispatcher() { + assert(resumingContexts.empty()); + assert(reusableContexts.size() == contextCount); + assert(spawningProcedures.empty()); + assert(reusableContexts.size() == allocatedStacks.size()); + while (!reusableContexts.empty()) { + delete[] allocatedStacks.top(); + allocatedStacks.pop(); + delete static_cast(reusableContexts.top()); + reusableContexts.pop(); + } + + while (!timers.empty()) { + timers.pop(); + } + + if (-1 == close(epoll)) { + std::cerr << "close() fail errno=" << errno << std::endl; + } +} + +void* Dispatcher::getCurrentContext() const { + return currentContext; +} + +int Dispatcher::getEpoll() const { + return epoll; +} + +void Dispatcher::pushContext(void* context) { + resumingContexts.push(context); +} + +void Dispatcher::spawn(std::function&& procedure) { + ucontext_t *context; + if (reusableContexts.empty()) { + context = new ucontext_t; + if (getcontext(context) == -1) { //makecontext precondition + std::cerr << "getcontext() fail errno=" << errno << std::endl; + throw std::runtime_error("Dispatcher::spawn()"); + } + auto stackPointer = new uint8_t[64 * 1024]; + context->uc_stack.ss_sp = stackPointer; + allocatedStacks.push(stackPointer); + context->uc_stack.ss_size = 64 * 1024; + makecontext(context, (void(*)())contextProcedureStatic, 1, reinterpret_cast(this)); + ++contextCount; + } else { + context = static_cast(reusableContexts.top()); + reusableContexts.pop(); + } + + resumingContexts.push(context); + spawningProcedures.emplace(std::move(procedure)); +} + +void Dispatcher::clear() { +//TODO +} + +void Dispatcher::yield() { + void* context; + for (;;) { + if (!resumingContexts.empty()) { + context = resumingContexts.front(); + resumingContexts.pop(); + assert(context); + break; + } + + epoll_event event; + int count = epoll_wait(epoll, &event, 1, -1); + + if (count == 1) { + if ((event.events & EPOLLOUT) != 0) { + context = static_cast(event.data.ptr)->writeContext; + } else { + context = static_cast(event.data.ptr)->context; + } + assert(context); + break; + } + + if (errno != EINTR) { + std::cerr << "epoll_wait() failed, errno=" << errno << std::endl; + throw std::runtime_error("Dispatcher::yield()"); + } + } + + if (context != currentContext) { + ucontext_t* oldContext = static_cast(currentContext); + currentContext = context; + if (-1 == swapcontext(oldContext, static_cast(context))) { + std::cerr << "swapcontext() failed, errno=" << errno << std::endl; + throw std::runtime_error("Dispatcher::yield()"); + } + } +} + +void Dispatcher::contextProcedure() { + void* context = currentContext; + for (;;) { + assert(!spawningProcedures.empty()); + std::function procedure = std::move(spawningProcedures.front()); + spawningProcedures.pop(); + procedure(); + reusableContexts.push(context); + yield(); + } +} diff --git a/src/Platform/Linux/System/Dispatcher.h b/src/Platform/Linux/System/Dispatcher.h new file mode 100644 index 00000000..bedc4d83 --- /dev/null +++ b/src/Platform/Linux/System/Dispatcher.h @@ -0,0 +1,51 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include + +namespace System { + +class Dispatcher { +public: + Dispatcher(); + Dispatcher(const Dispatcher&) = delete; + ~Dispatcher(); + Dispatcher& operator=(const Dispatcher&) = delete; + void spawn(std::function&& procedure); + void yield(); + void clear(); + + struct ContextExt { + void *context; + void *writeContext; //required workaround + }; +private: + friend class Event; + friend class DispatcherAccessor; + friend class TcpConnection; + friend class TcpConnector; + friend class TcpListener; + friend class Timer; + int epoll; + void* currentContext; + std::size_t contextCount; + std::queue resumingContexts; + std::stack reusableContexts; + std::stack allocatedStacks; + std::queue> spawningProcedures; + std::stack timers; + + int getEpoll() const; + void pushContext(void* context); + void* getCurrentContext() const; + + void contextProcedure(); + static void contextProcedureStatic(void* context); +}; + +} diff --git a/src/Platform/Linux/System/Event.cpp b/src/Platform/Linux/System/Event.cpp new file mode 100644 index 00000000..432ed090 --- /dev/null +++ b/src/Platform/Linux/System/Event.cpp @@ -0,0 +1,93 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "Event.h" +#include +#include +#include "Dispatcher.h" + +using namespace System; + +namespace { + +struct Waiter { + Waiter* next; + void* context; +}; + +} + +Event::Event() : dispatcher(nullptr) { +} + +Event::Event(Dispatcher& dispatcher) : dispatcher(&dispatcher), first(nullptr), state(false) { +} + +Event::Event(Event&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + first = other.first; + if (other.first != nullptr) { + last = other.last; + } + + state = other.state; + other.dispatcher = nullptr; + } +} + +Event::~Event() { + assert(first == nullptr); +} + +Event& Event::operator=(Event&& other) { + assert(first == nullptr); + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + first = other.first; + if (other.first != nullptr) { + last = other.last; + } + + state = other.state; + other.dispatcher = nullptr; + } + + return *this; +} + +bool Event::get() const { + assert(dispatcher != nullptr); + return state; +} + +void Event::clear() { + assert(dispatcher != nullptr); + state = false; +} + +void Event::set() { + assert(dispatcher != nullptr); + state = true; + for (Waiter* waiter = static_cast(first); waiter != nullptr; waiter = waiter->next) { + dispatcher->pushContext(waiter->context); + } + + first = nullptr; +} + +void Event::wait() { + assert(dispatcher != nullptr); + if (!state) { + Waiter waiter = { nullptr, dispatcher->getCurrentContext() }; + if (first != nullptr) { + static_cast(last)->next = &waiter; + } else { + first = &waiter; + } + + last = &waiter; + dispatcher->yield(); + assert(dispatcher != nullptr); + } +} diff --git a/src/Platform/Linux/System/Event.h b/src/Platform/Linux/System/Event.h new file mode 100644 index 00000000..cb5b4aed --- /dev/null +++ b/src/Platform/Linux/System/Event.h @@ -0,0 +1,32 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +namespace System { + +class Dispatcher; + +class Event { +public: + Event(); + explicit Event(Dispatcher& dispatcher); + Event(const Event&) = delete; + Event(Event&& other); + ~Event(); + Event& operator=(const Event&) = delete; + Event& operator=(Event&& other); + bool get() const; + void clear(); + void set(); + void wait(); + +private: + Dispatcher* dispatcher; + void* first; + void* last; + bool state; +}; + +} diff --git a/tests/functional_tests/transactions_generation_from_blockchain.h b/src/Platform/Linux/System/InterruptedException.cpp similarity index 52% rename from tests/functional_tests/transactions_generation_from_blockchain.h rename to src/Platform/Linux/System/InterruptedException.cpp index 0cc2224a..77a0cc92 100644 --- a/tests/functional_tests/transactions_generation_from_blockchain.h +++ b/src/Platform/Linux/System/InterruptedException.cpp @@ -1,7 +1,5 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. - - -bool transactions_generation_from_blockchain(std::string& blockchain_path); +#include "InterruptedException.h" diff --git a/src/Platform/Linux/System/InterruptedException.h b/src/Platform/Linux/System/InterruptedException.h new file mode 100644 index 00000000..d938374d --- /dev/null +++ b/src/Platform/Linux/System/InterruptedException.h @@ -0,0 +1,10 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include + +class InterruptedException : public std::exception { +}; diff --git a/src/Platform/Linux/System/TcpConnection.cpp b/src/Platform/Linux/System/TcpConnection.cpp new file mode 100644 index 00000000..fb9d08a6 --- /dev/null +++ b/src/Platform/Linux/System/TcpConnection.cpp @@ -0,0 +1,284 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "TcpConnection.h" +#include +#include +#include +#include +#include +#include +#include +#include "Dispatcher.h" +#include "InterruptedException.h" + +using namespace System; + +namespace { + +struct ConnectionContext : public Dispatcher::ContextExt { + bool interrupted; +}; + +} + +TcpConnection::TcpConnection() : dispatcher(nullptr) { +} + +TcpConnection::TcpConnection(Dispatcher& dispatcher, int socket) : dispatcher(&dispatcher), connection(socket), stopped(false), context(nullptr) { + epoll_event connectionEvent; + connectionEvent.data.fd = connection; + connectionEvent.events = 0; + connectionEvent.data.ptr = nullptr; + + if (epoll_ctl(dispatcher.getEpoll(), EPOLL_CTL_ADD, socket, &connectionEvent) == -1) { + std::cerr << errno << std::endl; + throw std::runtime_error("epoll_ctl() fail"); + } +} + +TcpConnection::TcpConnection(TcpConnection&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + connection = other.connection; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } +} + +TcpConnection::~TcpConnection() { + if (dispatcher != nullptr) { + assert(context == nullptr); + if (close(connection) == -1) { + std::cerr << "close() failed, errno=" << errno << '.' << std::endl; + } + } +} + +TcpConnection& TcpConnection::operator=(TcpConnection&& other) { + if (dispatcher != nullptr) { + assert(context == nullptr); + if (close(connection) == -1) { + std::cerr << "close() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpConnection::operator="); + } + } + + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + connection = other.connection; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } + + return *this; +} + +void TcpConnection::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +void TcpConnection::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (context != nullptr) { + ConnectionContext *context2 = static_cast(context); + if (!context2->interrupted) { + + epoll_event connectionEvent; + connectionEvent.data.fd = connection; + connectionEvent.events = 0; + connectionEvent.data.ptr = nullptr; + + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_MOD, connection, &connectionEvent) == -1) { + std::cerr << errno << std::endl; + throw std::runtime_error("epoll_ctl() fail"); + } + + context2->interrupted = true; + + if (context2->context != nullptr) { + dispatcher->pushContext(context2->context); + } + + if (context2->writeContext != nullptr) { + dispatcher->pushContext(context2->writeContext); + } + } + } + + stopped = true; +} + +size_t TcpConnection::read(uint8_t* data, size_t size) { + assert(dispatcher != nullptr); + assert(context == nullptr || static_cast(context)->context == nullptr); + if (stopped) { + throw InterruptedException(); + } + + ssize_t transferred = ::recv(connection, (void *)data, size, 0); + if (transferred == -1) { + if (errno != EAGAIN && errno != EWOULDBLOCK) { + std::cerr << "recv failed, result=" << errno << '.' << std::endl; + } else { + epoll_event connectionEvent; + connectionEvent.data.fd = connection; + + ConnectionContext context2; + if (context == nullptr) { + context2.writeContext = nullptr; + context2.interrupted = false; + context2.context = dispatcher->getCurrentContext(); + context = &context2; + connectionEvent.events = EPOLLIN | EPOLLONESHOT; + } else { + assert(static_cast(context)->writeContext != nullptr); + connectionEvent.events = EPOLLIN | EPOLLOUT | EPOLLONESHOT; + } + + connectionEvent.data.ptr = context; + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_MOD, connection, &connectionEvent) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + } else { + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.context == dispatcher->getCurrentContext()); + if (static_cast(context)->interrupted) { + context = nullptr; + throw InterruptedException(); + } + + assert(static_cast(context)->context == context2.context); + if (static_cast(context)->writeContext != nullptr) { //write is presented, rearm + static_cast(context)->context = nullptr; + + epoll_event connectionEvent; + connectionEvent.data.fd = connection; + connectionEvent.events = EPOLLOUT | EPOLLONESHOT; + connectionEvent.data.ptr = context; + + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_MOD, connection, &connectionEvent) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpConnection::read"); + } + } else { + context = nullptr; + } + + ssize_t transferred = ::recv(connection, (void *)data, size, 0); + if (transferred == -1) { + std::cerr << "recv failed, errno=" << errno << '.' << std::endl; + } else { + if (transferred == 0) { + std::cerr << "recv return after yield with 0 bytes" << std::endl; + + int retval = -1; + socklen_t retValLen = sizeof(retval); + int s = getsockopt(connection, SOL_SOCKET, SO_ERROR, &retval, &retValLen); + if (s == -1) { + std::cerr << "getsockopt() failed, errno=" << errno << '.' << std::endl; + } else { + std::cerr << "recv getsockopt retval = " << retval << std::endl; + } + } + + assert(transferred <= size); + return transferred; + } + } + } + + throw std::runtime_error("TcpConnection::read"); + } + + assert(transferred <= size); + return transferred; +} + +void TcpConnection::write(const uint8_t* data, size_t size) { + assert(dispatcher != nullptr); + assert(context == nullptr || static_cast(context)->writeContext == nullptr); + if (stopped) { + throw InterruptedException(); + } + + if (size == 0) { + if (shutdown(connection, SHUT_WR) == -1) { + std::cerr << "shutdown failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpConnection::write"); + } + + return; + } + + ssize_t transferred = ::send(connection, (void *)data, size, 0); + if (transferred == -1) { + if (errno != EAGAIN && errno != EWOULDBLOCK) { + std::cerr << "send failed, result=" << errno << '.' << std::endl; + } else { + epoll_event connectionEvent; + connectionEvent.data.fd = connection; + + ConnectionContext context2; + if (context == nullptr) { + context2.context = nullptr; + context2.interrupted = false; + context2.writeContext = dispatcher->getCurrentContext(); + context = &context2; + connectionEvent.events = EPOLLOUT | EPOLLONESHOT; + } else { + assert(static_cast(context)->context != nullptr); + connectionEvent.events = EPOLLIN | EPOLLOUT | EPOLLONESHOT; + } + + connectionEvent.data.ptr = context; + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_MOD, connection, &connectionEvent) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + } else { + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.writeContext == dispatcher->getCurrentContext()); + if (static_cast(context)->interrupted) { + context = nullptr; + throw InterruptedException(); + } + + assert(static_cast(context)->writeContext == context2.writeContext); + if (static_cast(context)->context != nullptr) { //read is presented, rearm + static_cast(context)->writeContext = nullptr; + + epoll_event connectionEvent; + connectionEvent.data.fd = connection; + connectionEvent.events = EPOLLIN | EPOLLONESHOT; + connectionEvent.data.ptr = context; + + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_MOD, connection, &connectionEvent) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpConnection::write"); + } + } else { + context = nullptr; + } + + ssize_t transferred = ::send(connection, (void *)data, size, 0); + if (transferred == -1) { + std::cerr << "send failed, errno=" << errno << '.' << std::endl; + } else { + if (transferred == 0) { + throw std::runtime_error("send transferred 0 bytes."); + } + + assert(transferred == size); + return; + } + } + } + + throw std::runtime_error("TcpConnection::write"); + } +} diff --git a/src/Platform/Linux/System/TcpConnection.h b/src/Platform/Linux/System/TcpConnection.h new file mode 100644 index 00000000..1cd827bf --- /dev/null +++ b/src/Platform/Linux/System/TcpConnection.h @@ -0,0 +1,40 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include + +namespace System { + +class Dispatcher; + +class TcpConnection { +public: + TcpConnection(); + TcpConnection(const TcpConnection&) = delete; + TcpConnection(TcpConnection&& other); + ~TcpConnection(); + TcpConnection& operator=(const TcpConnection&) = delete; + TcpConnection& operator=(TcpConnection&& other); + void start(); + void stop(); + std::size_t read(uint8_t* data, std::size_t size); + void write(const uint8_t* data, std::size_t size); + +private: + friend class TcpConnector; + friend class TcpListener; + + explicit TcpConnection(Dispatcher& dispatcher, int socket); + + Dispatcher* dispatcher; + int connection; + bool stopped; + void* context; +}; + +} diff --git a/src/Platform/Linux/System/TcpConnector.cpp b/src/Platform/Linux/System/TcpConnector.cpp new file mode 100644 index 00000000..b2260de4 --- /dev/null +++ b/src/Platform/Linux/System/TcpConnector.cpp @@ -0,0 +1,200 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "TcpConnector.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "Dispatcher.h" +#include "TcpConnection.h" +#include "InterruptedException.h" + +using namespace System; + +namespace { + +struct ConnectorContext : public Dispatcher::ContextExt { + int connection; + bool interrupted; +}; + +} + +TcpConnector::TcpConnector() : dispatcher(nullptr) { +} + +TcpConnector::TcpConnector(Dispatcher& dispatcher, const std::string& address, uint16_t port) : dispatcher(&dispatcher), address(address), port(port), stopped(false), context(nullptr) { +} + +TcpConnector::TcpConnector(TcpConnector&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + address = other.address; + port = other.port; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } +} + +TcpConnector::~TcpConnector() { +} + +TcpConnector& TcpConnector::operator=(TcpConnector&& other) { + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + address = other.address; + port = other.port; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } + + return *this; +} + +void TcpConnector::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +TcpConnection TcpConnector::connect() { + assert(dispatcher != nullptr); + assert(context == nullptr); + if (stopped) { + throw InterruptedException(); + } + + std::ostringstream portStream; + portStream << port; + addrinfo hints = { 0, AF_INET, SOCK_STREAM, IPPROTO_TCP, 0, NULL, NULL, NULL }; + addrinfo *addressInfos; + int result = getaddrinfo(address.c_str(), portStream.str().c_str(), &hints, &addressInfos); + if (result == -1) { + std::cerr << "getaddrinfo failed, errno=" << errno << '.' << std::endl; + } else { + std::size_t count = 0; + for (addrinfo* addressInfo = addressInfos; addressInfo != nullptr; addressInfo = addressInfo->ai_next) { + ++count; + } + + std::random_device randomDevice; + std::mt19937 generator(randomDevice()); + std::uniform_int_distribution distribution(0, count - 1); + std::size_t index = distribution(generator); + addrinfo* addressInfo = addressInfos; + for (std::size_t i = 0; i < index; ++i) { + addressInfo = addressInfo->ai_next; + } + + sockaddr_in addressData = *reinterpret_cast(addressInfo->ai_addr); + freeaddrinfo(addressInfo); + int connection = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (connection == -1) { + std::cerr << "socket failed, errno=" << errno << '.' << std::endl; + } else { + sockaddr_in bindAddress; + bindAddress.sin_family = AF_INET; + bindAddress.sin_port = 0; + bindAddress.sin_addr.s_addr = INADDR_ANY; + if (bind(connection, reinterpret_cast(&bindAddress), sizeof bindAddress) != 0) { + std::cerr << "bind failed, errno=" << errno << '.' << std::endl; + } else { + int flags = fcntl(connection, F_GETFL, 0); + if (flags == -1 || fcntl(connection, F_SETFL, flags | O_NONBLOCK) == -1) { + std::cerr << "fcntl() failed errno=" << errno << std::endl; + } else { + int result = ::connect(connection, reinterpret_cast(&addressData), sizeof addressData); + if (result == -1) { + if (errno == EINPROGRESS) { + ConnectorContext context2; + context2.writeContext = dispatcher->getCurrentContext(); + context2.context = nullptr; + context2.interrupted = false; + context2.connection = connection; + context = &context2; + + epoll_event connectEvent; + connectEvent.data.fd = connection; + connectEvent.events = EPOLLOUT | EPOLLRDHUP | EPOLLERR | EPOLLONESHOT; + connectEvent.data.ptr = context; + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_ADD, connection, &connectEvent) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + } else { + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.writeContext == dispatcher->getCurrentContext()); + assert(context == &context2); + context = nullptr; + context2.writeContext = nullptr; + if (context2.interrupted) { + if (close(connection) == -1) { + std::cerr << "close failed, errno=" << errno << std::endl; + } + + throw InterruptedException(); + } + + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_DEL, connection, NULL) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + } else { + int retval = -1; + socklen_t retValLen = sizeof(retval); + int s = getsockopt(connection, SOL_SOCKET, SO_ERROR, &retval, &retValLen); + if (s == -1) { + std::cerr << "getsockopt() failed, errno=" << errno << '.' << std::endl; + } else { + if (retval != 0) { + std::cerr << "connect failed; getsockopt retval = " << retval << std::endl; + } else { + return TcpConnection(*dispatcher, connection); + } + } + } + } + } + } else { + return TcpConnection(*dispatcher, connection); + } + } + } + + if (close(connection) == -1) { + std::cerr << "close failed, errno=" << errno << std::endl; + } + } + } + + throw std::runtime_error("TcpConnector::connect"); +} + +void TcpConnector::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (context != nullptr) { + ConnectorContext* context2 = static_cast(context); + if (!context2->interrupted) { + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_DEL, context2->connection, NULL) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpConnector::stop"); + } + + dispatcher->pushContext(context2->writeContext); + context2->interrupted = true; + } + } + + stopped = true; +} diff --git a/src/Platform/Linux/System/TcpConnector.h b/src/Platform/Linux/System/TcpConnector.h new file mode 100644 index 00000000..9f15c6f3 --- /dev/null +++ b/src/Platform/Linux/System/TcpConnector.h @@ -0,0 +1,37 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include + +namespace System { + +class Dispatcher; +class TcpConnection; + +class TcpConnector { +public: + TcpConnector(); + TcpConnector(Dispatcher& dispatcher, const std::string& address, uint16_t port); + TcpConnector(const TcpConnector&) = delete; + TcpConnector(TcpConnector&& other); + ~TcpConnector(); + TcpConnector& operator=(const TcpConnector&) = delete; + TcpConnector& operator=(TcpConnector&& other); + void start(); + void stop(); + TcpConnection connect(); + +private: + Dispatcher* dispatcher; + std::string address; + uint16_t port; + bool stopped; + void* context; +}; + +} diff --git a/src/Platform/Linux/System/TcpListener.cpp b/src/Platform/Linux/System/TcpListener.cpp new file mode 100644 index 00000000..0c5cae0c --- /dev/null +++ b/src/Platform/Linux/System/TcpListener.cpp @@ -0,0 +1,194 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "TcpListener.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Dispatcher.h" +#include "TcpConnection.h" +#include "InterruptedException.h" + +using namespace System; + +namespace { + +struct ListenerContext : public Dispatcher::ContextExt { + bool interrupted; +}; + +} + +TcpListener::TcpListener() : dispatcher(nullptr) { +} + +TcpListener::TcpListener(TcpListener&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + listener = other.listener; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } +} + +TcpListener::~TcpListener() { + if (dispatcher != nullptr) { + assert(context == nullptr); + if (close(listener) == -1) { + std::cerr << "close() failed, errno=" << errno << '.' << std::endl; + } + } +} + +TcpListener& TcpListener::operator=(TcpListener&& other) { + if (dispatcher != nullptr) { + assert(context == nullptr); + if (close(listener) == -1) { + std::cerr << "close() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpListener::operator="); + } + } + + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + listener = other.listener; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } + + return *this; +} + +void TcpListener::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +TcpListener::TcpListener(Dispatcher& dispatcher, const std::string& address, uint16_t port) : dispatcher(&dispatcher) { + listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (listener == -1) { + std::cerr << "socket failed, errno=" << errno << std::endl; + } else { + int flags = fcntl(listener, F_GETFL, 0); + if (flags == -1 || fcntl(listener, F_SETFL, flags | O_NONBLOCK) == -1) { + std::cerr << "fcntl() failed errno=" << errno << std::endl; + } else { + sockaddr_in address; + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + if (bind(listener, reinterpret_cast(&address), sizeof address) != 0) { + std::cerr << "bind failed, errno=" << errno << std::endl; + } else if (listen(listener, SOMAXCONN) != 0) { + std::cerr << "listen failed, errno=" << errno << std::endl; + } else { + epoll_event listenEvent; + listenEvent.data.fd = listener; + listenEvent.events = 0; + listenEvent.data.ptr = nullptr; + + if (epoll_ctl(dispatcher.getEpoll(), EPOLL_CTL_ADD, listener, &listenEvent) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + } else { + stopped = false; + context = nullptr; + return; + } + } + } + + if (close(listener) == -1) { + std::cerr << "close failed, errno=" << errno << std::endl; + } + } + + throw std::runtime_error("TcpListener::TcpListener"); +} + +TcpConnection TcpListener::accept() { + assert(dispatcher != nullptr); + assert(context == nullptr); + if (stopped) { + throw InterruptedException(); + } + + ListenerContext context2; + context2.context = dispatcher->getCurrentContext(); + context2.writeContext = nullptr; + context2.interrupted = false; + + epoll_event listenEvent; + listenEvent.data.fd = listener; + listenEvent.events = EPOLLIN | EPOLLONESHOT; + listenEvent.data.ptr = &context2; + + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_MOD, listener, &listenEvent) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + } else { + context = &context2; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.context == dispatcher->getCurrentContext()); + assert(context2.writeContext == nullptr); + assert(context == &context2); + context = nullptr; + context2.context = nullptr; + if (context2.interrupted) { + if (close(listener) == -1) { + std::cerr << "close failed, errno=" << errno << std::endl; + } + throw InterruptedException(); + } + + sockaddr inAddr; + socklen_t inLen = sizeof(inAddr); + int connection = ::accept(listener, &inAddr, &inLen); + if (connection == -1) { + std::cerr << "accept() failed, errno=" << errno << '.' << std::endl; + } else { + int flags = fcntl(connection, F_GETFL, 0); + if (flags == -1 || fcntl(connection, F_SETFL, flags | O_NONBLOCK) == -1) { + std::cerr << "fcntl() failed errno=" << errno << std::endl; + } else { + return TcpConnection(*dispatcher, connection); + } + } + } + + throw std::runtime_error("TcpListener::accept"); +} + +void TcpListener::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (context != nullptr) { + ListenerContext* context2 = static_cast(context); + if (!context2->interrupted) { + context2->interrupted = true; + + epoll_event listenEvent; + listenEvent.data.fd = listener; + listenEvent.events = 0; + listenEvent.data.ptr = nullptr; + + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_MOD, listener, &listenEvent) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpListener::stop"); + } + + dispatcher->pushContext(context2->context); + } + } + + stopped = true; +} diff --git a/src/Platform/Linux/System/TcpListener.h b/src/Platform/Linux/System/TcpListener.h new file mode 100644 index 00000000..714078d3 --- /dev/null +++ b/src/Platform/Linux/System/TcpListener.h @@ -0,0 +1,36 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include + +namespace System { + +class Dispatcher; +class TcpConnection; + +class TcpListener { +public: + TcpListener(); + TcpListener(Dispatcher& dispatcher, const std::string& address, uint16_t port); + TcpListener(const TcpListener&) = delete; + TcpListener(TcpListener&& other); + ~TcpListener(); + TcpListener& operator=(const TcpListener&) = delete; + TcpListener& operator=(TcpListener&& other); + void start(); + void stop(); + TcpConnection accept(); + +private: + Dispatcher* dispatcher; + int listener; + bool stopped; + void* context; +}; + +} diff --git a/src/Platform/Linux/System/Timer.cpp b/src/Platform/Linux/System/Timer.cpp new file mode 100644 index 00000000..e69559d6 --- /dev/null +++ b/src/Platform/Linux/System/Timer.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "Timer.h" +#include +#include +#include +#include +#include +#include +#include +#include "Dispatcher.h" +#include "InterruptedException.h" + +using namespace System; + +namespace { + +struct TimerContext : public Dispatcher::ContextExt { + Dispatcher* dispatcher; + bool interrupted; +}; + +} + +Timer::Timer() : dispatcher(nullptr) { +} + +Timer::Timer(Dispatcher& dispatcher) : dispatcher(&dispatcher), stopped(false), context(nullptr) { + timer = timerfd_create(CLOCK_MONOTONIC, 0); + epoll_event timerEvent; + timerEvent.data.fd = timer; + timerEvent.events = 0; + timerEvent.data.ptr = nullptr; + + if (epoll_ctl(this->dispatcher->getEpoll(), EPOLL_CTL_ADD, timer, &timerEvent) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("Timer::Timer"); + } +} + +Timer::Timer(Timer&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + timer = other.timer; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } +} + +Timer::~Timer() { + if (dispatcher != nullptr) { + close(timer); + } +} + +Timer& Timer::operator=(Timer&& other) { + if (dispatcher != nullptr) { + assert(context == nullptr); + close(timer); + } + + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + timer = other.timer; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } + + return *this; +} + +void Timer::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +void Timer::sleep(std::chrono::milliseconds duration) { + assert(dispatcher != nullptr); + assert(context == nullptr); + if (stopped) { + throw InterruptedException(); + } + + auto seconds = std::chrono::duration_cast(duration); + + itimerspec expires; + expires.it_interval.tv_nsec = expires.it_interval.tv_sec = 0; + expires.it_value.tv_sec = seconds.count(); + expires.it_value.tv_nsec = std::chrono::duration_cast(duration - seconds).count(); + timerfd_settime(timer, 0, &expires, NULL); + + TimerContext context2; + context2.dispatcher = dispatcher; + context2.context = dispatcher->getCurrentContext(); + context2.writeContext = nullptr; + context2.interrupted = false; + + epoll_event timerEvent; + timerEvent.data.fd = timer; + timerEvent.events = EPOLLIN | EPOLLONESHOT; + timerEvent.data.ptr = &context2; + + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_MOD, timer, &timerEvent) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("Timer::sleep"); + } + + context = &context2; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.context == dispatcher->getCurrentContext()); + assert(context2.writeContext == nullptr); + assert(context == &context2); + context = nullptr; + context2.context = nullptr; + if (context2.interrupted) { + throw InterruptedException(); + } +} + +void Timer::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (context != nullptr) { + TimerContext* context2 = reinterpret_cast(context); + if (context2->context != nullptr) { + epoll_event timerEvent; + timerEvent.data.fd = timer; + timerEvent.events = 0; + timerEvent.data.ptr = nullptr; + + if (epoll_ctl(dispatcher->getEpoll(), EPOLL_CTL_MOD, timer, &timerEvent) == -1) { + std::cerr << "epoll_ctl() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("Timer::sleep"); + } + + dispatcher->pushContext(context2->context); + context2->interrupted = true; + } + } + + stopped = true; +} diff --git a/src/Platform/Linux/System/Timer.h b/src/Platform/Linux/System/Timer.h new file mode 100644 index 00000000..122ffe1d --- /dev/null +++ b/src/Platform/Linux/System/Timer.h @@ -0,0 +1,33 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include + +namespace System { + +class Dispatcher; + +class Timer { +public: + Timer(); + explicit Timer(Dispatcher& dispatcher); + Timer(const Timer&) = delete; + Timer(Timer&& other); + ~Timer(); + Timer& operator=(const Timer&) = delete; + Timer& operator=(Timer&& other); + void start(); + void stop(); + void sleep(std::chrono::milliseconds duration); + +private: + Dispatcher* dispatcher; + int timer; + bool stopped; + void* context; +}; + +} diff --git a/src/Platform/OSX/System/Dispatcher.cpp b/src/Platform/OSX/System/Dispatcher.cpp new file mode 100644 index 00000000..b8188391 --- /dev/null +++ b/src/Platform/OSX/System/Dispatcher.cpp @@ -0,0 +1,156 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "Dispatcher.h" +#include +#define _XOPEN_SOURCE +#include +#include +#include +#include +#include +#include + +using namespace System; + +void Dispatcher::contextProcedureStatic(void *context) { + reinterpret_cast(context)->contextProcedure(); +} + +Dispatcher::Dispatcher() : lastCreatedTimer(0) { + kqueue = ::kqueue(); + if (kqueue == -1) { + std::cerr << "kqueue() fail errno=" << errno << std::endl; + } else { + currentContext = new ucontext_t; + if (getcontext(reinterpret_cast(currentContext)) == -1) { + std::cerr << "getcontext() fail errno=" << errno << std::endl; + } else { + contextCount = 0; + return; + } + } + throw std::runtime_error("Dispatcher::Dispatcher"); +} + +Dispatcher::~Dispatcher() { + assert(resumingContexts.empty()); + assert(reusableContexts.size() == contextCount); + assert(spawningProcedures.empty()); + assert(reusableContexts.size() == allocatedStacks.size()); + while (!reusableContexts.empty()) { + delete[] allocatedStacks.top(); + allocatedStacks.pop(); + delete static_cast(reusableContexts.top()); + reusableContexts.pop(); + } + + while (!timers.empty()) { + timers.pop(); + } + + if (-1 == close(kqueue)) { + std::cerr << "close() fail errno=" << errno << std::endl; + } +} + +void* Dispatcher::getCurrentContext() const { + return currentContext; +} + +int Dispatcher::getKqueue() const { + return kqueue; +} + +void Dispatcher::pushContext(void* context) { + resumingContexts.push(context); +} + +void Dispatcher::spawn(std::function&& procedure) { + void* context; + if (reusableContexts.empty()) { + context = new ucontext_t; + if (-1 == getcontext(reinterpret_cast(context))) { //makecontext precondition + std::cerr << "getcontext() fail errno=" << errno << std::endl; + throw std::runtime_error("Dispatcher::spawn()"); + } + auto stackPointer = new uint8_t[64 * 1024]; + reinterpret_cast(context)->uc_stack.ss_sp = stackPointer; + allocatedStacks.push(stackPointer); + reinterpret_cast(context)->uc_stack.ss_size = 64 * 1024; + makecontext(reinterpret_cast(context), (void(*)())contextProcedureStatic, 1, reinterpret_cast(this)); + ++contextCount; + } else { + context = reusableContexts.top(); + reusableContexts.pop(); + } + + resumingContexts.push(context); + spawningProcedures.emplace(std::move(procedure)); +} + +int Dispatcher::getTimer() { + int timer; + if (timers.empty()) { + timer = ++lastCreatedTimer; + } else { + timer = timers.top(); + timers.pop(); + } + + return timer; +} + +void Dispatcher::pushTimer(int timer) { + timers.push(timer); +} + +void Dispatcher::clear() { +//TODO +} + +void Dispatcher::yield() { + void* context; + for (;;) { + if (!resumingContexts.empty()) { + context = resumingContexts.front(); + resumingContexts.pop(); + break; + } + + struct kevent event; + int count = kevent(kqueue, NULL, 0, &event, 1, NULL); + + if (count == 1) { + context = static_cast(event.udata)->context; + break; + } + + if (errno != EINTR) { + std::cerr << "kevent() failed, errno=" << errno << std::endl; + throw std::runtime_error("Dispatcher::yield()"); + } + } + + if (context != currentContext) { + ucontext_t* oldContext = static_cast(currentContext); + currentContext = context; + if (-1 == swapcontext(oldContext, static_cast(context))) { + std::cerr << "setcontext() failed, errno=" << errno << std::endl; + throw std::runtime_error("Dispatcher::yield()"); + } + } +} + +void Dispatcher::contextProcedure() { + void* context = currentContext; + for (;;) { + assert(!spawningProcedures.empty()); + std::function procedure = std::move(spawningProcedures.front()); + spawningProcedures.pop(); + procedure(); + reusableContexts.push(context); + yield(); + } +} diff --git a/src/Platform/OSX/System/Dispatcher.h b/src/Platform/OSX/System/Dispatcher.h new file mode 100644 index 00000000..146364de --- /dev/null +++ b/src/Platform/OSX/System/Dispatcher.h @@ -0,0 +1,53 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include + +namespace System { + +class Dispatcher { +public: + Dispatcher(); + Dispatcher(const Dispatcher&) = delete; + ~Dispatcher(); + Dispatcher& operator=(const Dispatcher&) = delete; + void spawn(std::function&& procedure); + void yield(); + void clear(); + + struct ContextExt { + void *context; + }; +private: + friend class Event; + friend class DispatcherAccessor; + friend class TcpConnection; + friend class TcpConnector; + friend class TcpListener; + friend class Timer; + int kqueue; + void* currentContext; + int lastCreatedTimer; + std::size_t contextCount; + std::queue resumingContexts; + std::stack reusableContexts; + std::stack allocatedStacks; + std::queue> spawningProcedures; + std::stack timers; + + int getKqueue() const; + int getTimer(); + void pushTimer(int timer); + void pushContext(void* context); + void* getCurrentContext() const; + + void contextProcedure(); + static void contextProcedureStatic(void* context); +}; + +} diff --git a/src/Platform/OSX/System/Event.cpp b/src/Platform/OSX/System/Event.cpp new file mode 100644 index 00000000..432ed090 --- /dev/null +++ b/src/Platform/OSX/System/Event.cpp @@ -0,0 +1,93 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "Event.h" +#include +#include +#include "Dispatcher.h" + +using namespace System; + +namespace { + +struct Waiter { + Waiter* next; + void* context; +}; + +} + +Event::Event() : dispatcher(nullptr) { +} + +Event::Event(Dispatcher& dispatcher) : dispatcher(&dispatcher), first(nullptr), state(false) { +} + +Event::Event(Event&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + first = other.first; + if (other.first != nullptr) { + last = other.last; + } + + state = other.state; + other.dispatcher = nullptr; + } +} + +Event::~Event() { + assert(first == nullptr); +} + +Event& Event::operator=(Event&& other) { + assert(first == nullptr); + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + first = other.first; + if (other.first != nullptr) { + last = other.last; + } + + state = other.state; + other.dispatcher = nullptr; + } + + return *this; +} + +bool Event::get() const { + assert(dispatcher != nullptr); + return state; +} + +void Event::clear() { + assert(dispatcher != nullptr); + state = false; +} + +void Event::set() { + assert(dispatcher != nullptr); + state = true; + for (Waiter* waiter = static_cast(first); waiter != nullptr; waiter = waiter->next) { + dispatcher->pushContext(waiter->context); + } + + first = nullptr; +} + +void Event::wait() { + assert(dispatcher != nullptr); + if (!state) { + Waiter waiter = { nullptr, dispatcher->getCurrentContext() }; + if (first != nullptr) { + static_cast(last)->next = &waiter; + } else { + first = &waiter; + } + + last = &waiter; + dispatcher->yield(); + assert(dispatcher != nullptr); + } +} diff --git a/src/Platform/OSX/System/Event.h b/src/Platform/OSX/System/Event.h new file mode 100644 index 00000000..cb5b4aed --- /dev/null +++ b/src/Platform/OSX/System/Event.h @@ -0,0 +1,32 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +namespace System { + +class Dispatcher; + +class Event { +public: + Event(); + explicit Event(Dispatcher& dispatcher); + Event(const Event&) = delete; + Event(Event&& other); + ~Event(); + Event& operator=(const Event&) = delete; + Event& operator=(Event&& other); + bool get() const; + void clear(); + void set(); + void wait(); + +private: + Dispatcher* dispatcher; + void* first; + void* last; + bool state; +}; + +} diff --git a/src/Platform/OSX/System/InterruptedException.cpp b/src/Platform/OSX/System/InterruptedException.cpp new file mode 100644 index 00000000..77a0cc92 --- /dev/null +++ b/src/Platform/OSX/System/InterruptedException.cpp @@ -0,0 +1,5 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "InterruptedException.h" diff --git a/src/Platform/OSX/System/InterruptedException.h b/src/Platform/OSX/System/InterruptedException.h new file mode 100644 index 00000000..a42a7cf2 --- /dev/null +++ b/src/Platform/OSX/System/InterruptedException.h @@ -0,0 +1,14 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include + +namespace System { + +class InterruptedException : public std::exception { +}; + +} diff --git a/src/Platform/OSX/System/TcpConnection.cpp b/src/Platform/OSX/System/TcpConnection.cpp new file mode 100644 index 00000000..b1558b37 --- /dev/null +++ b/src/Platform/OSX/System/TcpConnection.cpp @@ -0,0 +1,234 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "TcpConnection.h" +#include +#include +#include +#include +#include +#include +#include "Dispatcher.h" +#include "InterruptedException.h" + +using namespace System; + +namespace { + +struct ConnectionContext : public Dispatcher::ContextExt { + bool interrupted; +}; + +} + +TcpConnection::TcpConnection() : dispatcher(nullptr) { +} + +TcpConnection::TcpConnection(Dispatcher& dispatcher, int socket) : dispatcher(&dispatcher), connection(socket), stopped(false), readContext(nullptr), writeContext(nullptr) { +} + +TcpConnection::TcpConnection(TcpConnection&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + connection = other.connection; + stopped = other.stopped; + readContext = other.readContext; + writeContext = other.writeContext; + other.dispatcher = nullptr; + } +} + +TcpConnection::~TcpConnection() { + if (dispatcher != nullptr) { + assert(readContext == nullptr); + assert(writeContext == nullptr); + if (close(connection) == -1) { + std::cerr << "close() failed, errno=" << errno << '.' << std::endl; + } + } +} + +TcpConnection& TcpConnection::operator=(TcpConnection&& other) { + if (dispatcher != nullptr) { + assert(readContext == nullptr); + assert(writeContext == nullptr); + if (close(connection) == -1) { + std::cerr << "close() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpConnection::operator="); + } + } + + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + connection = other.connection; + stopped = other.stopped; + readContext = other.readContext; + writeContext = other.writeContext; + other.dispatcher = nullptr; + } + + return *this; +} + +void TcpConnection::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +size_t TcpConnection::read(uint8_t* data, size_t size) { + assert(dispatcher != nullptr); + assert(readContext == nullptr); + if (stopped) { + throw InterruptedException(); + } + + ssize_t transferred = ::recv(connection, (void *)data, size, 0); + if (transferred == -1) { + if (errno != EAGAIN && errno != EWOULDBLOCK) { + std::cerr << "recv failed, result=" << errno << '.' << std::endl; + } else { + ConnectionContext context2; + context2.context = dispatcher->getCurrentContext(); + context2.interrupted = false; + struct kevent event; + EV_SET(&event, connection, EVFILT_READ, EV_ADD | EV_ENABLE | EV_CLEAR | EV_ONESHOT, 0, 0, &context2); + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + } else { + readContext = &context2; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.context == dispatcher->getCurrentContext()); + assert(readContext == &context2); + readContext = nullptr; + context2.context = nullptr; + if (context2.interrupted) { + throw InterruptedException(); + } + + ssize_t transferred = ::recv(connection, (void *)data, size, 0); + if (transferred == -1) { + std::cerr << "recv failed, errno=" << errno << '.' << std::endl; + } else { + if (transferred == 0) { + std::cerr << "recv return after yield with 0 bytes" << std::endl; + + int retval = -1; + socklen_t retValLen = sizeof(retval); + int s = getsockopt(connection, SOL_SOCKET, SO_ERROR, &retval, &retValLen); + if (s == -1) { + std::cerr << "getsockopt() failed, errno=" << errno << '.' << std::endl; + } else { + std::cerr << "recv getsockopt retval = " << retval << std::endl; + } + } + + assert(transferred <= size); + return transferred; + } + } + } + + throw std::runtime_error("TcpConnection::read"); + } + + assert(transferred <= size); + return transferred; +} + +void TcpConnection::write(const uint8_t* data, size_t size) { + assert(dispatcher != nullptr); + assert(writeContext == nullptr); + if (stopped) { + throw InterruptedException(); + } + + if (size == 0) { + if (shutdown(connection, SHUT_WR) == -1) { + std::cerr << "shutdown failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpConnection::write"); + } + + return; + } + + ssize_t transferred = ::send(connection, (void *)data, size, 0); + if (transferred == -1) { + if (errno != EAGAIN && errno != EWOULDBLOCK) { + std::cerr << "send failed, result=" << errno << '.' << std::endl; + } else { + ConnectionContext context2; + context2.context = dispatcher->getCurrentContext(); + context2.interrupted = false; + struct kevent event; + EV_SET(&event, connection, EVFILT_WRITE, EV_ADD | EV_ENABLE | EV_ONESHOT, 0, 0, &context2); + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + } else { + writeContext = &context2; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.context == dispatcher->getCurrentContext()); + assert(writeContext == &context2); + writeContext = nullptr; + context2.context = nullptr; + if (context2.interrupted) { + throw InterruptedException(); + } + + ssize_t transferred = ::send(connection, (void *)data, size, 0); + if (transferred == -1) { + std::cerr << "recv failed, errno=" << errno << '.' << std::endl; + } else { + if (transferred == 0) { + throw std::runtime_error("send transferred 0 bytes."); + } + + assert(transferred == size); + return; + } + } + } + + throw std::runtime_error("TcpConnection::write"); + } +} + +void TcpConnection::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (writeContext != nullptr && static_cast(writeContext)->context != nullptr) { + ConnectionContext* context2 = static_cast(writeContext); + if (!context2->interrupted) { + struct kevent event; + EV_SET(&event, connection, EVFILT_WRITE, EV_DELETE | EV_DISABLE, 0, 0, NULL); + + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpListener::stop"); + } + + context2->interrupted = true; + dispatcher->pushContext(context2->context); + } + } + + if (readContext != nullptr && static_cast(readContext)->context != nullptr) { + ConnectionContext* context2 = static_cast(readContext); + if (!context2->interrupted) { + struct kevent event; + EV_SET(&event, connection, EVFILT_READ, EV_DELETE | EV_DISABLE, 0, 0, NULL); + + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpListener::stop"); + } + + context2->interrupted = true; + dispatcher->pushContext(context2->context); + } + } + + stopped = true; +} diff --git a/src/Platform/OSX/System/TcpConnection.h b/src/Platform/OSX/System/TcpConnection.h new file mode 100644 index 00000000..266bfe49 --- /dev/null +++ b/src/Platform/OSX/System/TcpConnection.h @@ -0,0 +1,40 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include + +namespace System { + +class Dispatcher; + +class TcpConnection { +public: + TcpConnection(); + TcpConnection(const TcpConnection&) = delete; + TcpConnection(TcpConnection&& other); + ~TcpConnection(); + TcpConnection& operator=(const TcpConnection&) = delete; + TcpConnection& operator=(TcpConnection&& other); + void start(); + void stop(); + std::size_t read(uint8_t* data, std::size_t size); + void write(const uint8_t* data, std::size_t size); + +private: + friend class TcpConnector; + friend class TcpListener; + + explicit TcpConnection(Dispatcher& dispatcher, int socket); + + Dispatcher* dispatcher; + int connection; + bool stopped; + void* readContext; + void* writeContext; +}; + +} diff --git a/src/Platform/OSX/System/TcpConnector.cpp b/src/Platform/OSX/System/TcpConnector.cpp new file mode 100644 index 00000000..8256162d --- /dev/null +++ b/src/Platform/OSX/System/TcpConnector.cpp @@ -0,0 +1,200 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "TcpConnector.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "InterruptedException.h" +#include "Dispatcher.h" +#include "TcpConnection.h" + +using namespace System; + +namespace { + +struct ConnectorContext : public Dispatcher::ContextExt { + int connection; + bool interrupted; +}; + +} + +TcpConnector::TcpConnector() : dispatcher(nullptr) { +} + +TcpConnector::TcpConnector(Dispatcher& dispatcher, const std::string& address, uint16_t port) : dispatcher(&dispatcher), address(address), port(port), stopped(false), context(nullptr) { +} + +TcpConnector::TcpConnector(TcpConnector&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + address = other.address; + port = other.port; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } +} + +TcpConnector::~TcpConnector() { +} + +TcpConnector& TcpConnector::operator=(TcpConnector&& other) { + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + address = other.address; + port = other.port; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } + + return *this; +} + +void TcpConnector::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +void TcpConnector::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (context != nullptr) { + ConnectorContext* context2 = static_cast(context); + if (!context2->interrupted) { + struct kevent event; + EV_SET(&event, context2->connection, EVFILT_WRITE, EV_DELETE | EV_DISABLE, 0, 0, NULL); + + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpConnector::stop"); + } + + dispatcher->pushContext(context2->context); + context2->interrupted = true; + } + } + + stopped = true; +} + +TcpConnection TcpConnector::connect() { + assert(dispatcher != nullptr); + assert(context == nullptr); + if (stopped) { + throw InterruptedException(); + } + + std::ostringstream portStream; + portStream << port; + addrinfo hints = { 0, AF_INET, SOCK_STREAM, IPPROTO_TCP, 0, NULL, NULL, NULL }; + addrinfo *addressInfos; + int result = getaddrinfo(address.c_str(), portStream.str().c_str(), &hints, &addressInfos); + if (result == -1) { + std::cerr << "getaddrinfo failed, errno=" << errno << '.' << std::endl; + } else { + std::size_t count = 0; + for (addrinfo* addressInfo = addressInfos; addressInfo != nullptr; addressInfo = addressInfo->ai_next) { + ++count; + } + + std::random_device randomDevice; + std::mt19937 generator(randomDevice()); + std::uniform_int_distribution distribution(0, count - 1); + std::size_t index = distribution(generator); + addrinfo* addressInfo = addressInfos; + for (std::size_t i = 0; i < index; ++i) { + addressInfo = addressInfo->ai_next; + } + + sockaddr_in addressData = *reinterpret_cast(addressInfo->ai_addr); + freeaddrinfo(addressInfo); + int connection = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (connection == -1) { + std::cerr << "socket failed, errno=" << errno << '.' << std::endl; + } else { + sockaddr_in bindAddress; + bindAddress.sin_family = AF_INET; + bindAddress.sin_port = 0; + bindAddress.sin_addr.s_addr = INADDR_ANY; + if (bind(connection, reinterpret_cast(&bindAddress), sizeof bindAddress) != 0) { + std::cerr << "bind failed, errno=" << errno << '.' << std::endl; + } else { + int flags = fcntl(connection, F_GETFL, 0); + if (flags == -1 || (fcntl(connection, F_SETFL, flags | O_NONBLOCK) == -1)) { + std::cerr << "fcntl() failed errno=" << errno << std::endl; + } else { + int result = ::connect(connection, reinterpret_cast(&addressData), sizeof addressData); + if (result == -1) { + if (errno == EINPROGRESS) { + + ConnectorContext context2; + context2.context = dispatcher->getCurrentContext(); + context2.interrupted = false; + context2.connection = connection; + + struct kevent event; + EV_SET(&event, connection, EVFILT_WRITE, EV_ADD | EV_ENABLE | EV_ONESHOT | EV_CLEAR, 0, 0, &context2); + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + } else { + context = &context2; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.context == dispatcher->getCurrentContext()); + assert(context == &context2); + context = nullptr; + context2.context = nullptr; + if (context2.interrupted) { + if (close(connection) == -1) { + std::cerr << "close failed, errno=" << errno << std::endl; + } + + throw InterruptedException(); + } + struct kevent event; + EV_SET(&event, connection, EVFILT_WRITE, EV_ADD | EV_DISABLE, 0, 0, NULL); + + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + } else { + int retval = -1; + socklen_t retValLen = sizeof(retval); + int s = getsockopt(connection, SOL_SOCKET, SO_ERROR, &retval, &retValLen); + if (s == -1) { + std::cerr << "getsockopt() failed, errno=" << errno << '.' << std::endl; + } else { + if (retval != 0) { + std::cerr << "connect failed; getsockopt retval = " << retval << std::endl; + } else { + return TcpConnection(*dispatcher, connection); + } + } + } + } + } + } else { + return TcpConnection(*dispatcher, connection); + } + } + } + + if (close(connection) == -1) { + std::cerr << "close failed, errno=" << errno << std::endl; + } + } + } + + throw std::runtime_error("TcpConnector::connect"); +} diff --git a/src/Platform/OSX/System/TcpConnector.h b/src/Platform/OSX/System/TcpConnector.h new file mode 100644 index 00000000..02a50074 --- /dev/null +++ b/src/Platform/OSX/System/TcpConnector.h @@ -0,0 +1,36 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include + +namespace System { + +class Dispatcher; +class TcpConnection; + +class TcpConnector { +public: + TcpConnector(); + TcpConnector(Dispatcher& dispatcher, const std::string& address, uint16_t port); + TcpConnector(const TcpConnector&) = delete; + TcpConnector(TcpConnector&& other); + ~TcpConnector(); + TcpConnector& operator=(const TcpConnector&) = delete; + TcpConnector& operator=(TcpConnector&& other); + void start(); + void stop(); + TcpConnection connect(); + +private: + Dispatcher* dispatcher; + std::string address; + uint16_t port; + bool stopped; + void* context; +}; + +} diff --git a/src/Platform/OSX/System/TcpListener.cpp b/src/Platform/OSX/System/TcpListener.cpp new file mode 100644 index 00000000..7f797171 --- /dev/null +++ b/src/Platform/OSX/System/TcpListener.cpp @@ -0,0 +1,189 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "TcpListener.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "InterruptedException.h" +#include "Dispatcher.h" +#include "TcpConnection.h" + +using namespace System; + +namespace { + +struct ListenerContext : public Dispatcher::ContextExt { + bool interrupted; +}; + +} + +TcpListener::TcpListener() : dispatcher(nullptr) { +} + +TcpListener::TcpListener(TcpListener&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + listener = other.listener; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } +} + +TcpListener::~TcpListener() { + if (dispatcher != nullptr) { + assert(context == nullptr); + if (close(listener) == -1) { + std::cerr << "close() failed, errno=" << errno << '.' << std::endl; + } + } +} + +TcpListener& TcpListener::operator=(TcpListener&& other) { + if (dispatcher != nullptr) { + assert(context == nullptr); + if (close(listener) == -1) { + std::cerr << "close() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpListener::operator="); + } + } + + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + listener = other.listener; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } + + return *this; +} + +void TcpListener::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +TcpListener::TcpListener(Dispatcher& dispatcher, const std::string& address, uint16_t port) : dispatcher(&dispatcher) { + listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (listener == -1) { + std::cerr << "socket failed, errno=" << errno << std::endl; + } else { + int flags = fcntl(listener, F_GETFL, 0); + if (flags == -1 || (fcntl(listener, F_SETFL, flags | O_NONBLOCK) == -1)) { + std::cerr << "fcntl() failed errno=" << errno << std::endl; + } else { + sockaddr_in address; + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + if (bind(listener, reinterpret_cast(&address), sizeof address) != 0) { + std::cerr << "bind failed, errno=" << errno << std::endl; + } else if (listen(listener, SOMAXCONN) != 0) { + std::cerr << "listen failed, errno=" << errno << std::endl; + } else { + struct kevent event; + EV_SET(&event, listener, EVFILT_READ, EV_ADD | EV_DISABLE, 0, SOMAXCONN, NULL); + + if (kevent(dispatcher.getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + } else { + stopped = false; + context = nullptr; + return; + } + } + } + + if (close(listener) == -1) { + std::cerr << "close failed, errno=" << errno << std::endl; + } + } + + throw std::runtime_error("TcpListener::TcpListener"); +} + +TcpConnection TcpListener::accept() { + assert(dispatcher != nullptr); + assert(context == nullptr); + if (stopped) { + throw InterruptedException(); + } + + ListenerContext context2; + context2.context = dispatcher->getCurrentContext(); + context2.interrupted = false; + + struct kevent event; + EV_SET(&event, listener, EVFILT_READ, EV_ADD | EV_ENABLE | EV_ONESHOT, 0, SOMAXCONN, &context2); + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + } else { + context = &context2; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.context == dispatcher->getCurrentContext()); + assert(context == &context2); + context = nullptr; + context2.context = nullptr; + if (context2.interrupted) { + if (close(listener) == -1) { + std::cerr << "close failed, errno=" << errno << std::endl; + } + throw InterruptedException(); + } + struct kevent event; + EV_SET(&event, listener, EVFILT_READ, EV_ADD | EV_DISABLE, 0, 0, NULL); + + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + } else { + sockaddr inAddr; + socklen_t inLen = sizeof(inAddr); + int connection = ::accept(listener, &inAddr, &inLen); + if (connection == -1) { + std::cerr << "accept() failed, errno=" << errno << '.' << std::endl; + } else { + int flags = fcntl(connection, F_GETFL, 0); + if (flags == -1 || (fcntl(connection, F_SETFL, flags | O_NONBLOCK) == -1)) { + std::cerr << "fcntl() failed errno=" << errno << std::endl; + } else { + return TcpConnection(*dispatcher, connection); + } + } + } + } + + throw std::runtime_error("TcpListener::accept"); +} + +void TcpListener::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (context != nullptr) { + ListenerContext* context2 = static_cast(context); + if (!context2->interrupted) { + context2->interrupted = true; + + struct kevent event; + EV_SET(&event, listener, EVFILT_READ, EV_DELETE | EV_DISABLE, 0, 0, NULL); + + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("TcpListener::stop"); + } + + dispatcher->pushContext(context2->context); + } + } + + stopped = true; +} diff --git a/src/Platform/OSX/System/TcpListener.h b/src/Platform/OSX/System/TcpListener.h new file mode 100644 index 00000000..f655b712 --- /dev/null +++ b/src/Platform/OSX/System/TcpListener.h @@ -0,0 +1,35 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include + +namespace System { + +class Dispatcher; +class TcpConnection; + +class TcpListener { +public: + TcpListener(); + TcpListener(Dispatcher& dispatcher, const std::string& address, uint16_t port); + TcpListener(const TcpListener&) = delete; + TcpListener(TcpListener&& other); + ~TcpListener(); + TcpListener& operator=(const TcpListener&) = delete; + TcpListener& operator=(TcpListener&& other); + void start(); + void stop(); + TcpConnection accept(); + +private: + Dispatcher* dispatcher; + int listener; + bool stopped; + void* context; +}; + +} diff --git a/src/Platform/OSX/System/Timer.cpp b/src/Platform/OSX/System/Timer.cpp new file mode 100644 index 00000000..964ac741 --- /dev/null +++ b/src/Platform/OSX/System/Timer.cpp @@ -0,0 +1,124 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "Timer.h" +#include +#include +#include +#include +#include +#include +#include "Dispatcher.h" +#include "InterruptedException.h" + +using namespace System; + +namespace { + +struct TimerContext : public Dispatcher::ContextExt { + Dispatcher* dispatcher; + bool interrupted; +}; + +} + +Timer::Timer() : dispatcher(nullptr) { +} + +Timer::Timer(Dispatcher& dispatcher) : dispatcher(&dispatcher), stopped(false), context(nullptr) { + timer = dispatcher.getTimer(); +} + +Timer::~Timer() { + if (dispatcher != nullptr) { + assert(context == nullptr); + dispatcher->pushTimer(timer); + } +} + +Timer::Timer(Timer&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + timer = other.timer; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } +} + +Timer& Timer::operator=(Timer&& other) { + if (dispatcher != nullptr) { + assert(context == nullptr); + dispatcher->pushTimer(timer); + } + + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + timer = other.timer; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } + + return *this; +} + +void Timer::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +void Timer::sleep(std::chrono::milliseconds duration) { + assert(dispatcher != nullptr); + assert(context == nullptr); + if (stopped) { + throw InterruptedException(); + } + + TimerContext context2; + context2.dispatcher = dispatcher; + context2.context = dispatcher->getCurrentContext(); + context2.interrupted = false; + + struct kevent event; + EV_SET(&event, timer, EVFILT_TIMER, EV_ADD | EV_ENABLE | EV_ONESHOT, 0, duration.count(), &context2); + + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("Timer::sleep"); + } + + context = &context2; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.context == dispatcher->getCurrentContext()); + assert(context == &context2); + context = nullptr; + context2.context = nullptr; + if (context2.interrupted) { + throw InterruptedException(); + } +} + +void Timer::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (context != nullptr) { + TimerContext* context2 = reinterpret_cast(context); + if (context2->context != nullptr) { + struct kevent event; + EV_SET(&event, timer, EVFILT_TIMER, EV_DISABLE, 0, 0, NULL); + + if (kevent(dispatcher->getKqueue(), &event, 1, NULL, 0, NULL) == -1) { + std::cerr << "kevent() failed, errno=" << errno << '.' << std::endl; + throw std::runtime_error("Timer::stop"); + } + + dispatcher->pushContext(context2->context); + context2->interrupted = true; + } + } + + stopped = true; +} diff --git a/src/Platform/OSX/System/Timer.h b/src/Platform/OSX/System/Timer.h new file mode 100644 index 00000000..122ffe1d --- /dev/null +++ b/src/Platform/OSX/System/Timer.h @@ -0,0 +1,33 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include + +namespace System { + +class Dispatcher; + +class Timer { +public: + Timer(); + explicit Timer(Dispatcher& dispatcher); + Timer(const Timer&) = delete; + Timer(Timer&& other); + ~Timer(); + Timer& operator=(const Timer&) = delete; + Timer& operator=(Timer&& other); + void start(); + void stop(); + void sleep(std::chrono::milliseconds duration); + +private: + Dispatcher* dispatcher; + int timer; + bool stopped; + void* context; +}; + +} diff --git a/src/Platform/Windows/System/Dispatcher.cpp b/src/Platform/Windows/System/Dispatcher.cpp new file mode 100644 index 00000000..37a87d09 --- /dev/null +++ b/src/Platform/Windows/System/Dispatcher.cpp @@ -0,0 +1,173 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "Dispatcher.h" +#include +#include +#define WIN32_LEAN_AND_MEAN +#include +#include + +using namespace System; + +namespace { + +struct OverlappedExt : public OVERLAPPED { + void* context; +}; + +} + +Dispatcher::Dispatcher() { + completionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); + if (completionPort == NULL) { + std::cerr << "CreateIoCompletionPort failed, result=" << GetLastError() << '.' << std::endl; + } else { + if (ConvertThreadToFiberEx(NULL, 0) == NULL) { + std::cerr << "ConvertThreadToFiberEx failed, result=" << GetLastError() << '.' << std::endl; + } else { + WSADATA wsaData; + int result = WSAStartup(0x0202, &wsaData); + if (result != 0) { + std::cerr << "WSAStartup failed, result=" << result << '.' << std::endl; + } else { + contextCount = 0; + return; + } + + if (ConvertFiberToThread() != TRUE) { + std::cerr << "ConvertFiberToThread failed, result=" << GetLastError() << '.' << std::endl; + } + } + + if (CloseHandle(completionPort) != TRUE) { + std::cerr << "CloseHandle failed, result=" << GetLastError() << '.' << std::endl; + } + } + + throw std::runtime_error("Dispatcher::Dispatcher"); +} + +Dispatcher::~Dispatcher() { + assert(resumingContexts.empty()); + assert(reusableContexts.size() == contextCount); + assert(spawningProcedures.empty()); + while (!reusableContexts.empty()) { + DeleteFiber(reusableContexts.top()); + reusableContexts.pop(); + } + + while (!timers.empty()) { + if (CloseHandle(timers.top()) != TRUE) { + std::cerr << "CloseHandle failed, result=" << GetLastError() << '.' << std::endl; + } + + timers.pop(); + } + + if (WSACleanup() != 0) { + std::cerr << "WSACleanup failed, result=" << WSAGetLastError() << '.' << std::endl; + } + + if (ConvertFiberToThread() != TRUE) { + std::cerr << "ConvertFiberToThread failed, result=" << GetLastError() << '.' << std::endl; + } + + if (CloseHandle(completionPort) != TRUE) { + std::cerr << "CloseHandle failed, result=" << GetLastError() << '.' << std::endl; + } +} + +void* Dispatcher::getCompletionPort() const { + return completionPort; +} + +void* Dispatcher::getTimer() { + void* timer; + if (timers.empty()) { + timer = CreateWaitableTimer(NULL, FALSE, NULL); + if (timer == NULL) { + std::cerr << "CreateWaitableTimer failed, result=" << GetLastError() << '.' << std::endl; + throw std::runtime_error("Dispatcher::getTimer"); + } + } else { + timer = timers.top(); + timers.pop(); + } + + return timer; +} + +void Dispatcher::pushTimer(void* timer) { + timers.push(timer); +} + +void Dispatcher::pushContext(void* context) { + resumingContexts.push(context); +} + +void Dispatcher::spawn(std::function&& procedure) { + void* context; + if (reusableContexts.empty()) { + context = CreateFiberEx(16384, 65536, 0, contextProcedureStatic, this); + if (context == NULL) { + std::cerr << "CreateFiberEx failed, result=" << GetLastError() << '.' << std::endl; + throw std::runtime_error("Dispatcher::spawn"); + } + ++contextCount; + } else { + context = reusableContexts.top(); + reusableContexts.pop(); + } + + resumingContexts.push(context); + spawningProcedures.emplace(std::move(procedure)); +} + +void Dispatcher::clear() { +//TODO +} + +void Dispatcher::yield() { + void* context; + for (;;) { + if (!resumingContexts.empty()) { + context = resumingContexts.front(); + resumingContexts.pop(); + break; + } + + OVERLAPPED_ENTRY entry; + ULONG actual = 0; + if (GetQueuedCompletionStatusEx(completionPort, &entry, 1, &actual, INFINITE, TRUE) == TRUE) { + context = reinterpret_cast(entry.lpOverlapped)->context; + break; + } + + DWORD lastError = GetLastError(); + if (lastError != WAIT_IO_COMPLETION) { + std::cerr << "GetQueuedCompletionStatusEx failed, result=" << lastError << '.' << std::endl; + throw std::runtime_error("Dispatcher::yield"); + } + } + + if (context != GetCurrentFiber()) { + SwitchToFiber(context); + } +} + +void Dispatcher::contextProcedure() { + for (;;) { + assert(!spawningProcedures.empty()); + std::function procedure = std::move(spawningProcedures.front()); + spawningProcedures.pop(); + procedure(); + reusableContexts.push(GetCurrentFiber()); + yield(); + } +} + +void __stdcall Dispatcher::contextProcedureStatic(void* context) { + static_cast(context)->contextProcedure(); +} diff --git a/src/Platform/Windows/System/Dispatcher.h b/src/Platform/Windows/System/Dispatcher.h new file mode 100644 index 00000000..c828764c --- /dev/null +++ b/src/Platform/Windows/System/Dispatcher.h @@ -0,0 +1,47 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include + +namespace System { + +class Dispatcher { +public: + Dispatcher(); + Dispatcher(const Dispatcher&) = delete; + ~Dispatcher(); + Dispatcher& operator=(const Dispatcher&) = delete; + void spawn(std::function&& procedure); + void yield(); + void clear(); + +private: + friend class Event; + friend class DispatcherAccessor; + friend class TcpConnection; + friend class TcpConnector; + friend class TcpListener; + friend class Timer; + + void* completionPort; + std::size_t contextCount; + std::queue resumingContexts; + std::stack reusableContexts; + std::queue> spawningProcedures; + std::stack timers; + + void* getCompletionPort() const; + void* getTimer(); + void pushTimer(void* timer); + void pushContext(void* context); + + void contextProcedure(); + static void __stdcall contextProcedureStatic(void* context); +}; + +} diff --git a/src/Platform/Windows/System/Event.cpp b/src/Platform/Windows/System/Event.cpp new file mode 100644 index 00000000..63cd46bb --- /dev/null +++ b/src/Platform/Windows/System/Event.cpp @@ -0,0 +1,94 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "Event.h" +#include +#define WIN32_LEAN_AND_MEAN +#include +#include "Dispatcher.h" + +using namespace System; + +namespace { + +struct Waiter { + Waiter* next; + void* context; +}; + +} + +Event::Event() : dispatcher(nullptr) { +} + +Event::Event(Dispatcher& dispatcher) : dispatcher(&dispatcher), first(nullptr), state(false) { +} + +Event::Event(Event&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + first = other.first; + if (other.first != nullptr) { + last = other.last; + } + + state = other.state; + other.dispatcher = nullptr; + } +} + +Event::~Event() { + assert(first == nullptr); +} + +Event& Event::operator=(Event&& other) { + assert(first == nullptr); + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + first = other.first; + if (other.first != nullptr) { + last = other.last; + } + + state = other.state; + other.dispatcher = nullptr; + } + + return *this; +} + +bool Event::get() const { + assert(dispatcher != nullptr); + return state; +} + +void Event::clear() { + assert(dispatcher != nullptr); + state = false; +} + +void Event::set() { + assert(dispatcher != nullptr); + state = true; + for (Waiter* waiter = static_cast(first); waiter != nullptr; waiter = waiter->next) { + dispatcher->pushContext(waiter->context); + } + + first = nullptr; +} + +void Event::wait() { + assert(dispatcher != nullptr); + if (!state) { + Waiter waiter = {nullptr, GetCurrentFiber()}; + if (first != nullptr) { + static_cast(last)->next = &waiter; + } else { + first = &waiter; + } + + last = &waiter; + dispatcher->yield(); + assert(dispatcher != nullptr); + } +} diff --git a/src/Platform/Windows/System/Event.h b/src/Platform/Windows/System/Event.h new file mode 100644 index 00000000..cb5b4aed --- /dev/null +++ b/src/Platform/Windows/System/Event.h @@ -0,0 +1,32 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +namespace System { + +class Dispatcher; + +class Event { +public: + Event(); + explicit Event(Dispatcher& dispatcher); + Event(const Event&) = delete; + Event(Event&& other); + ~Event(); + Event& operator=(const Event&) = delete; + Event& operator=(Event&& other); + bool get() const; + void clear(); + void set(); + void wait(); + +private: + Dispatcher* dispatcher; + void* first; + void* last; + bool state; +}; + +} diff --git a/src/Platform/Windows/System/InterruptedException.cpp b/src/Platform/Windows/System/InterruptedException.cpp new file mode 100644 index 00000000..77a0cc92 --- /dev/null +++ b/src/Platform/Windows/System/InterruptedException.cpp @@ -0,0 +1,5 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "InterruptedException.h" diff --git a/src/Platform/Windows/System/InterruptedException.h b/src/Platform/Windows/System/InterruptedException.h new file mode 100644 index 00000000..a42a7cf2 --- /dev/null +++ b/src/Platform/Windows/System/InterruptedException.h @@ -0,0 +1,14 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include + +namespace System { + +class InterruptedException : public std::exception { +}; + +} diff --git a/src/Platform/Windows/System/TcpConnection.cpp b/src/Platform/Windows/System/TcpConnection.cpp new file mode 100644 index 00000000..807c0df5 --- /dev/null +++ b/src/Platform/Windows/System/TcpConnection.cpp @@ -0,0 +1,227 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "TcpConnection.h" +#include +#include +#define WIN32_LEAN_AND_MEAN +#include +#include "InterruptedException.h" +#include "Dispatcher.h" + +using namespace System; + +namespace { + +struct OverlappedExt : public OVERLAPPED { + void* context; + bool interrupted; +}; + +struct Context { + Dispatcher* dispatcher; + OverlappedExt* read; + OverlappedExt* write; +}; + +} + +TcpConnection::TcpConnection() : dispatcher(nullptr) { +} + +TcpConnection::TcpConnection(Dispatcher& dispatcher, std::size_t connection) : dispatcher(&dispatcher), connection(connection), stopped(false), context(nullptr) { +} + + +TcpConnection::TcpConnection(TcpConnection&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + connection = other.connection; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } +} + +TcpConnection::~TcpConnection() { + if (dispatcher != nullptr) { + assert(context == nullptr); + if (closesocket(connection) != 0) { + std::cerr << "closesocket failed, result=" << WSAGetLastError() << '.' << std::endl; + } + } +} + +TcpConnection& TcpConnection::operator=(TcpConnection&& other) { + if (dispatcher != nullptr) { + assert(context == nullptr); + if (closesocket(connection) != 0) { + std::cerr << "closesocket failed, result=" << WSAGetLastError() << '.' << std::endl; + throw std::runtime_error("TcpConnection::operator="); + } + } + + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + connection = other.connection; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } + + return *this; +} + +void TcpConnection::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +void TcpConnection::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (context != nullptr) { + Context* context2 = static_cast(context); + if (context2->read != nullptr && !context2->read->interrupted) { + if (CancelIoEx(reinterpret_cast(connection), context2->read) != TRUE) { + std::cerr << "CancelIoEx failed, result=" << GetLastError() << '.' << std::endl; + throw std::runtime_error("TcpConnection::stop"); + } + + context2->read->interrupted = true; + } + + if (context2->write != nullptr && !context2->write->interrupted) { + if (CancelIoEx(reinterpret_cast(connection), context2->write) != TRUE) { + std::cerr << "CancelIoEx failed, result=" << GetLastError() << '.' << std::endl; + throw std::runtime_error("TcpConnection::stop"); + } + + context2->write->interrupted = true; + } + } + + stopped = true; +} + +size_t TcpConnection::read(uint8_t* data, size_t size) { + assert(dispatcher != nullptr); + assert(context == nullptr || static_cast(context)->read == nullptr); + if (stopped) { + throw InterruptedException(); + } + + WSABUF buf{static_cast(size), reinterpret_cast(data)}; + DWORD flags = 0; + OverlappedExt overlapped; + overlapped.hEvent = NULL; + if (WSARecv(connection, &buf, 1, NULL, &flags, &overlapped, NULL) != 0) { + int lastError = WSAGetLastError(); + if (lastError != WSA_IO_PENDING) { + std::cerr << "WSARecv failed, result=" << lastError << '.' << std::endl; + throw std::runtime_error("TcpConnection::read"); + } + } + + assert(flags == 0); + Context context2; + if (context == nullptr) { + context2.dispatcher = dispatcher; + context2.write = nullptr; + context = &context2; + } + + overlapped.context = GetCurrentFiber(); + overlapped.interrupted = false; + static_cast(context)->read = &overlapped; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(overlapped.context == GetCurrentFiber()); + assert(static_cast(context)->read == &overlapped); + if (static_cast(context)->write != nullptr) { + static_cast(context)->read = nullptr; + } else { + context = nullptr; + } + + DWORD transferred; + if (WSAGetOverlappedResult(connection, &overlapped, &transferred, FALSE, &flags) != TRUE) { + int lastError = WSAGetLastError(); + if (lastError == ERROR_OPERATION_ABORTED) { + assert(overlapped.interrupted); + throw InterruptedException(); + } + + std::cerr << "WSARecv failed, result=" << lastError << '.' << std::endl; + throw std::runtime_error("TcpConnection::read"); + } + + assert(transferred <= size); + assert(flags == 0); + return transferred; +} + +void TcpConnection::write(const uint8_t* data, size_t size) { + assert(dispatcher != nullptr); + assert(context == nullptr || static_cast(context)->write == nullptr); + if (stopped) { + throw InterruptedException(); + } + + if (size == 0) { + if (shutdown(connection, SD_SEND) != 0) { + std::cerr << "shutdown failed, result=" << WSAGetLastError() << '.' << std::endl; + throw std::runtime_error("TcpConnection::write"); + } + + return; + } + + WSABUF buf{static_cast(size), reinterpret_cast(const_cast(data))}; + OverlappedExt overlapped; + overlapped.hEvent = NULL; + if (WSASend(connection, &buf, 1, NULL, 0, &overlapped, NULL) != 0) { + int lastError = WSAGetLastError(); + if (lastError != WSA_IO_PENDING) { + std::cerr << "WSASend failed, result=" << lastError << '.' << std::endl; + throw std::runtime_error("TcpConnection::write"); + } + } + + Context context2; + if (context == nullptr) { + context2.dispatcher = dispatcher; + context2.read = nullptr; + context = &context2; + } + + overlapped.context = GetCurrentFiber(); + overlapped.interrupted = false; + static_cast(context)->write = &overlapped; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(overlapped.context == GetCurrentFiber()); + assert(static_cast(context)->write == &overlapped); + if (static_cast(context)->read != nullptr) { + static_cast(context)->write = nullptr; + } else { + context = nullptr; + } + + DWORD transferred; + DWORD flags; + if (WSAGetOverlappedResult(connection, &overlapped, &transferred, FALSE, &flags) != TRUE) { + int lastError = WSAGetLastError(); + if (lastError == ERROR_OPERATION_ABORTED) { + assert(overlapped.interrupted); + throw InterruptedException(); + } + + std::cerr << "WSSend failed, result=" << lastError << '.' << std::endl; + throw std::runtime_error("TcpConnection::write"); + } + + assert(transferred == size); + assert(flags == 0); +} diff --git a/src/Platform/Windows/System/TcpConnection.h b/src/Platform/Windows/System/TcpConnection.h new file mode 100644 index 00000000..d10dc391 --- /dev/null +++ b/src/Platform/Windows/System/TcpConnection.h @@ -0,0 +1,39 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include + +namespace System { + +class Dispatcher; + +class TcpConnection { +public: + TcpConnection(); + TcpConnection(const TcpConnection&) = delete; + TcpConnection(TcpConnection&& other); + ~TcpConnection(); + TcpConnection& operator=(const TcpConnection&) = delete; + TcpConnection& operator=(TcpConnection&& other); + void start(); + void stop(); + std::size_t read(uint8_t* data, std::size_t size); + void write(const uint8_t* data, std::size_t size); + +private: + friend class TcpConnector; + friend class TcpListener; + + explicit TcpConnection(Dispatcher& dispatcher, std::size_t connection); + + Dispatcher* dispatcher; + std::size_t connection; + bool stopped; + void* context; +}; + +} diff --git a/src/Platform/Windows/System/TcpConnector.cpp b/src/Platform/Windows/System/TcpConnector.cpp new file mode 100644 index 00000000..56eff4f0 --- /dev/null +++ b/src/Platform/Windows/System/TcpConnector.cpp @@ -0,0 +1,191 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "TcpConnector.h" +#include +#include +#include +#include +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#include "InterruptedException.h" +#include "Dispatcher.h" +#include "TcpConnection.h" + +using namespace System; + +namespace { + +struct Context : public OVERLAPPED { + void* context; + std::size_t connection; + bool interrupted; +}; + +LPFN_CONNECTEX connectEx = nullptr; + +} + +TcpConnector::TcpConnector() : dispatcher(nullptr) { +} + +TcpConnector::TcpConnector(Dispatcher& dispatcher, const std::string& address, uint16_t port) : dispatcher(&dispatcher), address(address), port(port), stopped(false), context(nullptr) { +} + +TcpConnector::TcpConnector(TcpConnector&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + address = other.address; + port = other.port; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } +} + +TcpConnector::~TcpConnector() { +} + +TcpConnector& TcpConnector::operator=(TcpConnector&& other) { + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + address = other.address; + port = other.port; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } + + return *this; +} + +void TcpConnector::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +void TcpConnector::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (context != nullptr) { + Context* context2 = static_cast(context); + if (!context2->interrupted) { + if (CancelIoEx(reinterpret_cast(context2->connection), context2) != TRUE) { + std::cerr << "CancelIoEx failed, result=" << GetLastError() << '.' << std::endl; + throw std::runtime_error("TcpConnector::stop"); + } + + context2->interrupted = true; + } + } + + stopped = true; +} + +TcpConnection TcpConnector::connect() { + assert(dispatcher != nullptr); + assert(context == nullptr); + if (stopped) { + throw InterruptedException(); + } + + std::ostringstream portStream; + portStream << port; + addrinfo hints = {0, AF_INET, SOCK_STREAM, IPPROTO_TCP, 0, NULL, NULL, NULL}; + addrinfo* addressInfos; + int result = getaddrinfo(address.c_str(), portStream.str().c_str(), &hints, &addressInfos); + if (result != 0) { + std::cerr << "getaddrinfo failed, result=" << result << '.' << std::endl; + } else { + std::size_t count = 0; + for (addrinfo* addressInfo = addressInfos; addressInfo != nullptr; addressInfo = addressInfo->ai_next) { + ++count; + } + + std::random_device randomDevice; + std::mt19937 generator(randomDevice()); + std::uniform_int_distribution distribution(0, count - 1); + std::size_t index = distribution(generator); + addrinfo* addressInfo = addressInfos; + for (std::size_t i = 0; i < index; ++i) { + addressInfo = addressInfo->ai_next; + } + + sockaddr_in addressData = *reinterpret_cast(addressInfo->ai_addr); + freeaddrinfo(addressInfo); + SOCKET connection = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (connection == INVALID_SOCKET) { + std::cerr << "socket failed, result=" << WSAGetLastError() << '.' << std::endl; + } else { + sockaddr_in bindAddress; + bindAddress.sin_family = AF_INET; + bindAddress.sin_port = 0; + bindAddress.sin_addr.s_addr = INADDR_ANY; + if (bind(connection, reinterpret_cast(&bindAddress), sizeof bindAddress) != 0) { + std::cerr << "bind failed, result=" << WSAGetLastError() << '.' << std::endl; + } else { + GUID guidConnectEx = WSAID_CONNECTEX; + DWORD read = sizeof connectEx; + if (connectEx == nullptr && WSAIoctl(connection, SIO_GET_EXTENSION_FUNCTION_POINTER, &guidConnectEx, sizeof guidConnectEx, &connectEx, sizeof connectEx, &read, NULL, NULL) != 0) { + std::cerr << "WSAIoctl failed, result=" << WSAGetLastError() << '.' << std::endl; + } else { + assert(read == sizeof connectEx); + if (CreateIoCompletionPort(reinterpret_cast(connection), dispatcher->getCompletionPort(), 0, 0) != dispatcher->getCompletionPort()) { + std::cerr << "CreateIoCompletionPort failed, result=" << GetLastError() << '.' << std::endl; + } else { + addressData.sin_port = htons(port); + Context context2; + context2.hEvent = NULL; + if (connectEx(connection, reinterpret_cast(&addressData), sizeof addressData, NULL, 0, NULL, &context2) == TRUE) { + std::cerr << "ConnectEx returned immediately, which is not supported." << std::endl; + } else { + int lastError = WSAGetLastError(); + if (lastError != WSA_IO_PENDING) { + std::cerr << "ConnectEx failed, result=" << lastError << '.' << std::endl; + } else { + context2.context = GetCurrentFiber(); + context2.connection = connection; + context2.interrupted = false; + context = &context2; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.context == GetCurrentFiber()); + assert(context2.connection == connection); + assert(context == &context2); + context = nullptr; + DWORD transferred; + DWORD flags; + if (WSAGetOverlappedResult(connection, &context2, &transferred, FALSE, &flags) != TRUE) { + lastError = WSAGetLastError(); + if (lastError == ERROR_OPERATION_ABORTED) { + assert(context2.interrupted); + if (closesocket(connection) != 0) { + std::cerr << "closesocket failed, result=" << WSAGetLastError() << '.' << std::endl; + } + + throw InterruptedException(); + } + + std::cerr << "ConnectEx failed, result=" << lastError << '.' << std::endl; + } else { + assert(transferred == 0); + assert(flags == 0); + return TcpConnection(*dispatcher, connection); + } + } + } + } + } + } + + if (closesocket(connection) != 0) { + std::cerr << "closesocket failed, result=" << WSAGetLastError() << '.' << std::endl; + } + } + } + + throw std::runtime_error("TcpConnector::connect"); +} diff --git a/src/Platform/Windows/System/TcpConnector.h b/src/Platform/Windows/System/TcpConnector.h new file mode 100644 index 00000000..02a50074 --- /dev/null +++ b/src/Platform/Windows/System/TcpConnector.h @@ -0,0 +1,36 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include + +namespace System { + +class Dispatcher; +class TcpConnection; + +class TcpConnector { +public: + TcpConnector(); + TcpConnector(Dispatcher& dispatcher, const std::string& address, uint16_t port); + TcpConnector(const TcpConnector&) = delete; + TcpConnector(TcpConnector&& other); + ~TcpConnector(); + TcpConnector& operator=(const TcpConnector&) = delete; + TcpConnector& operator=(TcpConnector&& other); + void start(); + void stop(); + TcpConnection connect(); + +private: + Dispatcher* dispatcher; + std::string address; + uint16_t port; + bool stopped; + void* context; +}; + +} diff --git a/src/Platform/Windows/System/TcpListener.cpp b/src/Platform/Windows/System/TcpListener.cpp new file mode 100644 index 00000000..36851bbd --- /dev/null +++ b/src/Platform/Windows/System/TcpListener.cpp @@ -0,0 +1,213 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "TcpListener.h" +#include +#include +#define WIN32_LEAN_AND_MEAN +#include +#include +#include "InterruptedException.h" +#include "Dispatcher.h" +#include "TcpConnection.h" + +using namespace System; + +namespace { + +struct Context : public OVERLAPPED { + void* context; + bool interrupted; +}; + +LPFN_ACCEPTEX acceptEx = nullptr; +LPFN_GETACCEPTEXSOCKADDRS getAcceptExSockaddrs = nullptr; + +} + +TcpListener::TcpListener() : dispatcher(nullptr) { +} + +TcpListener::TcpListener(Dispatcher& dispatcher, const std::string& address, uint16_t port) : dispatcher(&dispatcher) { + listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (listener == INVALID_SOCKET) { + std::cerr << "socket failed, result=" << WSAGetLastError() << '.' << std::endl; + } else { + sockaddr_in address; + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + if (bind(listener, reinterpret_cast(&address), sizeof address) != 0) { + std::cerr << "bind failed, result=" << WSAGetLastError() << '.' << std::endl; + } else if (listen(listener, SOMAXCONN) != 0) { + std::cerr << "listen failed, result=" << WSAGetLastError() << '.' << std::endl; + } else { + GUID guidAcceptEx = WSAID_ACCEPTEX; + DWORD read = sizeof acceptEx; + if (acceptEx == nullptr && WSAIoctl(listener, SIO_GET_EXTENSION_FUNCTION_POINTER, &guidAcceptEx, sizeof guidAcceptEx, &acceptEx, sizeof acceptEx, &read, NULL, NULL) != 0) { + std::cerr << "WSAIoctl failed, result=" << WSAGetLastError() << '.' << std::endl; + } else { + assert(read == sizeof acceptEx); + GUID guidGetAcceptExSockaddrs = WSAID_GETACCEPTEXSOCKADDRS; + read = sizeof getAcceptExSockaddrs; + if (getAcceptExSockaddrs == nullptr && WSAIoctl(listener, SIO_GET_EXTENSION_FUNCTION_POINTER, &guidGetAcceptExSockaddrs, sizeof guidGetAcceptExSockaddrs, &getAcceptExSockaddrs, sizeof getAcceptExSockaddrs, &read, NULL, NULL) != 0) { + std::cerr << "WSAIoctl failed, result=" << WSAGetLastError() << '.' << std::endl; + } else { + assert(read == sizeof getAcceptExSockaddrs); + if (CreateIoCompletionPort(reinterpret_cast(listener), dispatcher.getCompletionPort(), 0, 0) != dispatcher.getCompletionPort()) { + std::cerr << "CreateIoCompletionPort failed, result=" << GetLastError() << '.' << std::endl; + } else { + stopped = false; + context = nullptr; + return; + } + } + } + } + + if (closesocket(listener) != 0) { + std::cerr << "closesocket failed, result=" << WSAGetLastError() << '.' << std::endl; + } + } + + throw std::runtime_error("TcpListener::TcpListener"); +} + +TcpListener::TcpListener(TcpListener&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + listener = other.listener; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } +} + +TcpListener::~TcpListener() { + if (dispatcher != nullptr) { + assert(context == nullptr); + if (closesocket(listener) != 0) { + std::cerr << "closesocket failed, result=" << WSAGetLastError() << '.' << std::endl; + } + } +} + +TcpListener& TcpListener::operator=(TcpListener&& other) { + if (dispatcher != nullptr) { + assert(context == nullptr); + if (closesocket(listener) != 0) { + std::cerr << "closesocket failed, result=" << WSAGetLastError() << '.' << std::endl; + throw std::runtime_error("TcpListener::operator="); + } + } + + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + listener = other.listener; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } + + return *this; +} + +void TcpListener::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +void TcpListener::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (context != nullptr) { + Context* context2 = static_cast(context); + if (!context2->interrupted) { + if (CancelIoEx(reinterpret_cast(listener), context2) != TRUE) { + std::cerr << "CancelIoEx failed, result=" << GetLastError() << '.' << std::endl; + throw std::runtime_error("TcpListener::stop"); + } + + context2->interrupted = true; + } + } + + stopped = true; +} + +TcpConnection TcpListener::accept() { + assert(dispatcher != nullptr); + assert(context == nullptr); + if (stopped) { + throw InterruptedException(); + } + + SOCKET connection = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (connection == INVALID_SOCKET) { + std::cerr << "socket failed, result=" << WSAGetLastError() << '.' << std::endl; + } else { + uint8_t addresses[sizeof sockaddr_in * 2 + 32]; + DWORD received; + Context context2; + context2.hEvent = NULL; + if (acceptEx(listener, connection, addresses, 0, sizeof(sockaddr_in) + 16, sizeof(sockaddr_in) + 16, &received, &context2) == TRUE) { + std::cerr << "AcceptEx returned immediately, which is not supported." << std::endl; + } else { + int lastError = WSAGetLastError(); + if (lastError != WSA_IO_PENDING) { + std::cerr << "AcceptEx failed, result=" << lastError << '.' << std::endl; + } else { + context2.context = GetCurrentFiber(); + context2.interrupted = false; + context = &context2; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.context == GetCurrentFiber()); + assert(context == &context2); + context = nullptr; + DWORD transferred; + DWORD flags; + if (WSAGetOverlappedResult(listener, &context2, &transferred, FALSE, &flags) != TRUE) { + lastError = WSAGetLastError(); + if (lastError == ERROR_OPERATION_ABORTED) { + assert(context2.interrupted); + if (closesocket(connection) != 0) { + std::cerr << "closesocket failed, result=" << WSAGetLastError() << '.' << std::endl; + } + + throw InterruptedException(); + } + + std::cerr << "AcceptEx failed, result=" << lastError << '.' << std::endl; + } else { + assert(transferred == 0); + assert(flags == 0); + if (setsockopt(connection, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, reinterpret_cast(&listener), sizeof listener) != 0) { + std::cerr << "setsockopt failed, result=" << WSAGetLastError() << '.' << std::endl; + } else { + if (CreateIoCompletionPort(reinterpret_cast(connection), dispatcher->getCompletionPort(), 0, 0) != dispatcher->getCompletionPort()) { + std::cerr << "CreateIoCompletionPort failed, result=" << GetLastError() << '.' << std::endl; + } else { + //sockaddr_in* local; + //int localSize; + //sockaddr_in* remote; + //int remoteSize; + //static_cast(getAcceptExSockaddrs)(addresses, 0, sizeof(sockaddr_in) + 16, sizeof(sockaddr_in) + 16, reinterpret_cast(&local), &localSize, reinterpret_cast(&remote), &remoteSize); + //assert(localSize == sizeof sockaddr_in); + //assert(remoteSize == sizeof sockaddr_in); + //std::cout << "Client connected from " << static_cast(remote->sin_addr.S_un.S_un_b.s_b1) << '.' << static_cast(remote->sin_addr.S_un.S_un_b.s_b2) << '.' << static_cast(remote->sin_addr.S_un.S_un_b.s_b3) << '.' << static_cast(remote->sin_addr.S_un.S_un_b.s_b4) << ':' << remote->sin_port << std::endl; + return TcpConnection(*dispatcher, connection); + } + } + } + } + } + + if (closesocket(connection) != 0) { + std::cerr << "closesocket failed, result=" << WSAGetLastError() << '.' << std::endl; + } + } + + throw std::runtime_error("TcpListener::accept"); +} diff --git a/src/Platform/Windows/System/TcpListener.h b/src/Platform/Windows/System/TcpListener.h new file mode 100644 index 00000000..118b9b4f --- /dev/null +++ b/src/Platform/Windows/System/TcpListener.h @@ -0,0 +1,35 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include + +namespace System { + +class Dispatcher; +class TcpConnection; + +class TcpListener { +public: + TcpListener(); + TcpListener(Dispatcher& dispatcher, const std::string& address, uint16_t port); + TcpListener(const TcpListener&) = delete; + TcpListener(TcpListener&& other); + ~TcpListener(); + TcpListener& operator=(const TcpListener&) = delete; + TcpListener& operator=(TcpListener&& other); + void start(); + void stop(); + TcpConnection accept(); + +private: + Dispatcher* dispatcher; + std::size_t listener; + bool stopped; + void* context; +}; + +} diff --git a/src/Platform/Windows/System/Timer.cpp b/src/Platform/Windows/System/Timer.cpp new file mode 100644 index 00000000..7e4c324f --- /dev/null +++ b/src/Platform/Windows/System/Timer.cpp @@ -0,0 +1,133 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "Timer.h" +#include +#include +#define WIN32_LEAN_AND_MEAN +#include +#include "InterruptedException.h" +#include "Dispatcher.h" + +using namespace System; + +namespace System { + +class DispatcherAccessor { +public: + DispatcherAccessor(Dispatcher* dispatcher, void* context) { + dispatcher->pushContext(context); + } +}; + +} + +namespace { + +struct Context { + Dispatcher* dispatcher; + void* context; + bool interrupted; +}; + +void __stdcall callbackProcedure(void* lpArgToCompletionRoutine, DWORD dwTimerLowValue, DWORD dwTimerHighValue) { + Context* context = static_cast(lpArgToCompletionRoutine); + assert(context->context != nullptr); + DispatcherAccessor(context->dispatcher, context->context); + context->context = nullptr; +} + +} + +Timer::Timer() : dispatcher(nullptr) { +} + +Timer::Timer(Dispatcher& dispatcher) : dispatcher(&dispatcher), stopped(false), context(nullptr) { + timer = dispatcher.getTimer(); +} + +Timer::Timer(Timer&& other) : dispatcher(other.dispatcher) { + if (other.dispatcher != nullptr) { + timer = other.timer; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } +} + +Timer::~Timer() { + if (dispatcher != nullptr) { + assert(context == nullptr); + dispatcher->pushTimer(timer); + } +} + +Timer& Timer::operator=(Timer&& other) { + if (dispatcher != nullptr) { + assert(context == nullptr); + dispatcher->pushTimer(timer); + } + + dispatcher = other.dispatcher; + if (other.dispatcher != nullptr) { + timer = other.timer; + stopped = other.stopped; + context = other.context; + other.dispatcher = nullptr; + } + + return *this; +} + +void Timer::start() { + assert(dispatcher != nullptr); + assert(stopped); + stopped = false; +} + +void Timer::stop() { + assert(dispatcher != nullptr); + assert(!stopped); + if (context != nullptr) { + Context* context2 = static_cast(context); + if (context2->context != nullptr) { + if (CancelWaitableTimer(timer) != TRUE) { + std::cerr << "CancelWaitableTimer failed, result=" << GetLastError() << '.' << std::endl; + throw std::runtime_error("Timer::stop"); + } + + dispatcher->pushContext(context2->context); + context2->context = nullptr; + context2->interrupted = true; + } + } + + stopped = true; +} + +void Timer::sleep(std::chrono::nanoseconds duration) { + assert(dispatcher != nullptr); + assert(context == nullptr); + if (stopped) { + throw InterruptedException(); + } + + LARGE_INTEGER duration2; + duration2.QuadPart = static_cast(duration.count() / -100); + Context context2 = {dispatcher, GetCurrentFiber(), false}; + if (SetWaitableTimer(timer, &duration2, 0, callbackProcedure, &context2, FALSE) != TRUE) { + std::cerr << "SetWaitableTimer failed, result=" << GetLastError() << '.' << std::endl; + throw std::runtime_error("Timer::sleep"); + } + + context = &context2; + dispatcher->yield(); + assert(dispatcher != nullptr); + assert(context2.context == nullptr); + assert(context == &context2); + context = nullptr; + if (context2.interrupted) { + throw InterruptedException(); + } +} diff --git a/src/Platform/Windows/System/Timer.h b/src/Platform/Windows/System/Timer.h new file mode 100644 index 00000000..e19cdba4 --- /dev/null +++ b/src/Platform/Windows/System/Timer.h @@ -0,0 +1,33 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include + +namespace System { + +class Dispatcher; + +class Timer { +public: + Timer(); + explicit Timer(Dispatcher& dispatcher); + Timer(const Timer&) = delete; + Timer(Timer&& other); + ~Timer(); + Timer& operator=(const Timer&) = delete; + Timer& operator=(Timer&& other); + void start(); + void stop(); + void sleep(std::chrono::nanoseconds duration); + +private: + Dispatcher* dispatcher; + void* timer; + bool stopped; + void* context; +}; + +} diff --git a/src/platform/mingw/alloca.h b/src/Platform/mingw/alloca.h similarity index 77% rename from src/platform/mingw/alloca.h rename to src/Platform/mingw/alloca.h index 9e18ae9e..011dfb0c 100644 --- a/src/platform/mingw/alloca.h +++ b/src/Platform/mingw/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2013 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/platform/msc/alloca.h b/src/Platform/msc/alloca.h similarity index 78% rename from src/platform/msc/alloca.h rename to src/Platform/msc/alloca.h index 4e8c89ea..da2ecbf1 100644 --- a/src/platform/msc/alloca.h +++ b/src/Platform/msc/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2013 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/platform/msc/stdbool.h b/src/Platform/msc/stdbool.h similarity index 82% rename from src/platform/msc/stdbool.h rename to src/Platform/msc/stdbool.h index 2e1a0673..cf698000 100644 --- a/src/platform/msc/stdbool.h +++ b/src/Platform/msc/stdbool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2013 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/platform/msc/sys/param.h b/src/Platform/msc/sys/param.h similarity index 83% rename from src/platform/msc/sys/param.h rename to src/Platform/msc/sys/param.h index b44de84b..2a1e34dc 100644 --- a/src/platform/msc/sys/param.h +++ b/src/Platform/msc/sys/param.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2013 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/System/TcpStream.cpp b/src/System/TcpStream.cpp new file mode 100644 index 00000000..e314a8e7 --- /dev/null +++ b/src/System/TcpStream.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "TcpStream.h" + +#include + +using namespace System; + +TcpStreambuf::TcpStreambuf(TcpConnection& connection) : connection(connection) { + setg(&readBuf.front(), &readBuf.front(), &readBuf.front()); + setp(reinterpret_cast(&writeBuf.front()), reinterpret_cast(&writeBuf.front() + writeBuf.max_size())); +} + +TcpStreambuf::~TcpStreambuf() { + dumpBuffer(); +} + +std::streambuf::int_type TcpStreambuf::underflow() { + if (gptr() < egptr()) + return traits_type::to_int_type(*gptr()); + + size_t bytesRead; + try { + bytesRead = connection.read(reinterpret_cast(&readBuf.front()), readBuf.max_size()); + } catch (std::exception& ex) { + return traits_type::eof(); + } + + if (bytesRead == 0) { + return traits_type::eof(); + } + + setg(&readBuf.front(), &readBuf.front(), &readBuf.front() + bytesRead); + + return traits_type::to_int_type(*gptr()); +} + +int TcpStreambuf::sync() { + return dumpBuffer() ? 0 : -1; +} + +bool TcpStreambuf::dumpBuffer() { + try { + size_t count = pptr() - pbase(); + connection.write(&writeBuf.front(), count); + pbump(-count); + } catch (std::exception&) { + return false; + } + + return true; +} + +std::streambuf::int_type TcpStreambuf::overflow(std::streambuf::int_type ch) { + if (ch == traits_type::eof()) { + return traits_type::eof(); + } + + if (pptr() == epptr()) { + if (!dumpBuffer()) { + return traits_type::eof(); + } + } + + *pptr() = ch; + pbump(1); + + return ch; +} diff --git a/src/System/TcpStream.h b/src/System/TcpStream.h new file mode 100644 index 00000000..13c06af9 --- /dev/null +++ b/src/System/TcpStream.h @@ -0,0 +1,34 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include + +#include + +namespace System { + +class TcpStreambuf : public std::streambuf { +public: + TcpStreambuf(TcpConnection& connection); + TcpStreambuf(const TcpStreambuf&) = delete; + + virtual ~TcpStreambuf(); + +private: + std::streambuf::int_type underflow() override; + std::streambuf::int_type overflow(std::streambuf::int_type ch) override; + int sync() override; + + bool dumpBuffer(); + + TcpConnection& connection; + + std::array readBuf; + std::array writeBuf; +}; + +} diff --git a/src/common/BlockingQueue.cpp b/src/common/BlockingQueue.cpp new file mode 100644 index 00000000..b4db8754 --- /dev/null +++ b/src/common/BlockingQueue.cpp @@ -0,0 +1,5 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "BlockingQueue.h" diff --git a/src/common/BlockingQueue.h b/src/common/BlockingQueue.h new file mode 100644 index 00000000..3804ca6d --- /dev/null +++ b/src/common/BlockingQueue.h @@ -0,0 +1,114 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include +#include +#include + +template < typename T, typename Container = std::deque > +class BlockingQueue { +public: + + typedef BlockingQueue ThisType; + + BlockingQueue(size_t maxSize = 1) : + m_maxSize(maxSize), m_closed(false) {} + + template + bool push(TT&& v) { + std::unique_lock lk(m_mutex); + + while (!m_closed && m_queue.size() >= m_maxSize) { + m_haveSpace.wait(lk); + } + + if (m_closed) { + return false; + } + + m_queue.push_back(std::forward(v)); + m_haveData.notify_one(); + return true; + } + + bool pop(T& v) { + std::unique_lock lk(m_mutex); + + while (m_queue.empty()) { + if (m_closed) { + // all data has been processed, queue is closed + return false; + } + m_haveData.wait(lk); + } + + v = std::move(m_queue.front()); + m_queue.pop_front(); + + // we can have several waiting threads to unblock + if (m_closed && m_queue.empty()) + m_haveSpace.notify_all(); + else + m_haveSpace.notify_one(); + + return true; + } + + void close(bool wait = false) { + std::unique_lock lk(m_mutex); + m_closed = true; + m_haveData.notify_all(); // wake up threads in pop() + m_haveSpace.notify_all(); + + if (wait) { + while (!m_queue.empty()) { + m_haveSpace.wait(lk); + } + } + } + + size_t size() { + std::unique_lock lk(m_mutex); + return m_queue.size(); + } + + size_t capacity() const { + return m_maxSize; + } + +private: + + const size_t m_maxSize; + Container m_queue; + bool m_closed; + + std::mutex m_mutex; + std::condition_variable m_haveData; + std::condition_variable m_haveSpace; +}; + +template +class GroupClose { +public: + + GroupClose(QueueT& queue, size_t groupSize) + : m_queue(queue), m_count(groupSize) {} + + void close() { + if (m_count == 0) + return; + if (m_count.fetch_sub(1) == 1) + m_queue.close(); + } + +private: + + std::atomic m_count; + QueueT& m_queue; + +}; diff --git a/src/common/ObserverManager.h b/src/common/ObserverManager.h index 68ed0e91..3a64a411 100644 --- a/src/common/ObserverManager.h +++ b/src/common/ObserverManager.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -93,6 +93,19 @@ public: } } + template + void notify(F notification, const Arg0& arg0, const Arg1& arg1, const Arg2& arg2, const Arg3& arg3) { + std::vector observersCopy; + { + std::unique_lock lock(m_observersMutex); + observersCopy = m_observers; + } + + for (T* observer : observersCopy) { + (observer->*notification)(arg0, arg1, arg2, arg3); + } + } + #else template diff --git a/src/common/ShuffleGenerator.h b/src/common/ShuffleGenerator.h new file mode 100644 index 00000000..b7e7f26b --- /dev/null +++ b/src/common/ShuffleGenerator.h @@ -0,0 +1,57 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include + +template +class ShuffleGenerator { +public: + + ShuffleGenerator(T n, const Gen& gen = Gen()) : + N(n), generator(gen), count(n) {} + + T operator()() { + + if (count == 0) { + throw std::runtime_error("shuffle sequence ended"); + } + + typedef typename std::uniform_int_distribution distr_t; + typedef typename distr_t::param_type param_t; + + distr_t distr; + + T value = distr(generator, param_t(0, --count)); + + auto rvalIt = selected.find(count); + auto rval = rvalIt != selected.end() ? rvalIt->second : count; + + auto lvalIt = selected.find(value); + + if (lvalIt != selected.end()) { + value = lvalIt->second; + lvalIt->second = rval; + } else { + selected[value] = rval; + } + + return value; + } + + void reset() { + count = N; + selected.clear(); + } + +private: + + std::unordered_map selected; + T count; + const T N; + Gen generator; +}; + diff --git a/src/common/SignalHandler.cpp b/src/common/SignalHandler.cpp new file mode 100644 index 00000000..4c13098f --- /dev/null +++ b/src/common/SignalHandler.cpp @@ -0,0 +1,41 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "SignalHandler.h" + +#include +#include + +// epee +#include "include_base_utils.h" + + +namespace tools { + std::function SignalHandler::m_handler; + +#if defined(WIN32) + BOOL WINAPI SignalHandler::winHandler(DWORD type) { + if (CTRL_C_EVENT == type || CTRL_BREAK_EVENT == type) { + handleSignal(); + return TRUE; + } else { + LOG_PRINT_RED_L0("Got control signal " << type << ". Exiting without saving..."); + return FALSE; + } + return TRUE; + } + +#else + + void SignalHandler::posixHandler(int /*type*/) { + handleSignal(); + } +#endif + + void SignalHandler::handleSignal() { + static std::mutex m_mutex; + std::unique_lock lock(m_mutex); + m_handler(); + } +} diff --git a/src/common/SignalHandler.h b/src/common/SignalHandler.h new file mode 100644 index 00000000..4d1855be --- /dev/null +++ b/src/common/SignalHandler.h @@ -0,0 +1,46 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include + +#include "misc_os_dependent.h" + +namespace tools { + class SignalHandler + { + public: + template + static bool install(T t) + { +#if defined(WIN32) + bool r = TRUE == ::SetConsoleCtrlHandler(&winHandler, TRUE); + if (r) + { + m_handler = t; + } + return r; +#else + signal(SIGINT, posixHandler); + signal(SIGTERM, posixHandler); + m_handler = t; + return true; +#endif + } + + private: +#if defined(WIN32) + static BOOL WINAPI winHandler(DWORD type); +#else + static void posixHandler(int /*type*/); +#endif + + static void handleSignal(); + + private: + static std::function m_handler; + }; +} diff --git a/src/common/base58.cpp b/src/common/base58.cpp index 06ebee1f..36ab8803 100644 --- a/src/common/base58.cpp +++ b/src/common/base58.cpp @@ -1,8 +1,7 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. - #include "base58.h" #include @@ -11,7 +10,6 @@ #include "crypto/hash.h" #include "int-util.h" -#include "util.h" #include "varint.h" namespace tools diff --git a/src/common/base58.h b/src/common/base58.h index 4436550d..c09315d0 100644 --- a/src/common/base58.h +++ b/src/common/base58.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/common/boost_serialization_helper.h b/src/common/boost_serialization_helper.h index cdd5ceae..87e07afe 100644 --- a/src/common/boost_serialization_helper.h +++ b/src/common/boost_serialization_helper.h @@ -1,12 +1,21 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once +#if defined(WIN32) +#include +#endif + +#include + #include #include +// epee +#include "include_base_utils.h" +#include "misc_os_dependent.h" namespace tools { diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index 5be588d8..09b70b1e 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/common/command_line.h b/src/common/command_line.h index bd3726e6..9a1ea589 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/common/int-util.h b/src/common/int-util.h index 15cd1e99..1d47f821 100644 --- a/src/common/int-util.h +++ b/src/common/int-util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,6 +13,8 @@ #if defined(_MSC_VER) #include +#define inline __inline + static inline uint32_t rol32(uint32_t x, int r) { static_assert(sizeof(uint32_t) == sizeof(unsigned int), "this code assumes 32-bit integers"); return _rotl(x, r); diff --git a/src/common/pod-class.h b/src/common/pod-class.h index e1989356..1e4334df 100644 --- a/src/common/pod-class.h +++ b/src/common/pod-class.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/common/static_assert.h b/src/common/static_assert.h new file mode 100644 index 00000000..3e94a156 --- /dev/null +++ b/src/common/static_assert.h @@ -0,0 +1,13 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#ifndef __cplusplus +#ifdef __clang__ + +#define static_assert _Static_assert + +#endif +#endif diff --git a/src/common/unordered_containers_boost_serialization.h b/src/common/unordered_containers_boost_serialization.h index 45101a60..1c97509e 100644 --- a/src/common/unordered_containers_boost_serialization.h +++ b/src/common/unordered_containers_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,12 +8,15 @@ #include #include +#include "google/sparse_hash_set" +#include "google/sparse_hash_map" + namespace boost { namespace serialization { - template - inline void save(Archive &a, const std::unordered_map &x, const boost::serialization::version_type ver) + template + inline void save(Archive &a, const std::unordered_map &x, const boost::serialization::version_type ver) { size_t s = x.size(); a << s; @@ -24,8 +27,8 @@ namespace boost } } - template - inline void load(Archive &a, std::unordered_map &x, const boost::serialization::version_type ver) + template + inline void load(Archive &a, std::unordered_map &x, const boost::serialization::version_type ver) { x.clear(); size_t s = 0; @@ -95,9 +98,75 @@ namespace boost } } + template + inline void save(Archive &a, const ::google::sparse_hash_set &x, const boost::serialization::version_type ver) + { + size_t s = x.size(); + a << s; + BOOST_FOREACH(auto& v, x) + { + a << v; + } + } + + template + inline void load(Archive &a, ::google::sparse_hash_set &x, const boost::serialization::version_type ver) + { + x.clear(); + size_t s = 0; + a >> s; + x.resize(s); + for(size_t i = 0; i != s; i++) + { + hval v; + a >> v; + x.insert(v); + } + } + + template + inline void serialize(Archive &a, ::google::sparse_hash_set &x, const boost::serialization::version_type ver) + { + split_free(a, x, ver); + } template - inline void serialize(Archive &a, std::unordered_map &x, const boost::serialization::version_type ver) + inline void save(Archive &a, const ::google::sparse_hash_map &x, const boost::serialization::version_type ver) + { + size_t s = x.size(); + a << s; + BOOST_FOREACH(auto& v, x) + { + a << v.first; + a << v.second; + } + } + + template + inline void load(Archive &a, ::google::sparse_hash_map &x, const boost::serialization::version_type ver) + { + x.clear(); + size_t s = 0; + a >> s; + x.resize(s); + for(size_t i = 0; i != s; i++) + { + h_key k; + hval v; + a >> k; + a >> v; + x.insert(std::pair(k, v)); + } + } + + template + inline void serialize(Archive &a, ::google::sparse_hash_map &x, const boost::serialization::version_type ver) + { + split_free(a, x, ver); + } + + template + inline void serialize(Archive &a, std::unordered_map &x, const boost::serialization::version_type ver) { split_free(a, x, ver); } diff --git a/src/common/util.cpp b/src/common/util.cpp index 5a1f50cb..3b7ecc62 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -1,13 +1,17 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "util.h" + #include +#include + #include "include_base_utils.h" using namespace epee; -#include "util.h" +#include "p2p/p2p_protocol_defs.h" #include "cryptonote_config.h" #ifdef WIN32 @@ -21,8 +25,6 @@ using namespace epee; namespace tools { - std::function signal_handler::m_handler; - #ifdef WIN32 std::string get_windows_version_display_string() { @@ -298,7 +300,7 @@ std::string get_nix_version_display_string() std::string config_folder; #ifdef WIN32 // Windows - config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "/" + CRYPTONOTE_NAME; + config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "/" + cryptonote::CRYPTONOTE_NAME; #else std::string pathRet; char* pszHome = getenv("HOME"); @@ -309,10 +311,10 @@ std::string get_nix_version_display_string() #ifdef MAC_OSX // Mac pathRet /= "Library/Application Support"; - config_folder = (pathRet + "/" + CRYPTONOTE_NAME); + config_folder = (pathRet + "/" + cryptonote::CRYPTONOTE_NAME); #else // Unix - config_folder = (pathRet + "/." + CRYPTONOTE_NAME); + config_folder = (pathRet + "/." + cryptonote::CRYPTONOTE_NAME); #endif #endif @@ -361,4 +363,12 @@ std::string get_nix_version_display_string() #endif return std::error_code(code, std::system_category()); } + + crypto::hash get_proof_of_trust_hash(const nodetool::proof_of_trust& pot) + { + std::string s; + s.append(reinterpret_cast(&pot.peer_id), sizeof(pot.peer_id)); + s.append(reinterpret_cast(&pot.time), sizeof(pot.time)); + return crypto::cn_fast_hash(s.data(), s.size()); + } } diff --git a/src/common/util.h b/src/common/util.h index 0af40e28..aeb757ef 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,17 +1,18 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once -#include +#include #include -#include -#include "crypto/crypto.h" #include "crypto/hash.h" -#include "misc_language.h" -#include "p2p/p2p_protocol_defs.h" + + +namespace nodetool { + struct proof_of_trust; +} namespace tools { @@ -19,68 +20,5 @@ namespace tools std::string get_os_version_string(); bool create_directories_if_necessary(const std::string& path); std::error_code replace_file(const std::string& replacement_name, const std::string& replaced_name); - - inline crypto::hash get_proof_of_trust_hash(const nodetool::proof_of_trust& pot) - { - std::string s; - s.append(reinterpret_cast(&pot.peer_id), sizeof(pot.peer_id)); - s.append(reinterpret_cast(&pot.time), sizeof(pot.time)); - return crypto::cn_fast_hash(s.data(), s.size()); - } - - - class signal_handler - { - public: - template - static bool install(T t) - { -#if defined(WIN32) - bool r = TRUE == ::SetConsoleCtrlHandler(&win_handler, TRUE); - if (r) - { - m_handler = t; - } - return r; -#else - signal(SIGINT, posix_handler); - signal(SIGTERM, posix_handler); - m_handler = t; - return true; -#endif - } - - private: -#if defined(WIN32) - static BOOL WINAPI win_handler(DWORD type) - { - if (CTRL_C_EVENT == type || CTRL_BREAK_EVENT == type) - { - handle_signal(); - return TRUE; - } - else - { - LOG_PRINT_RED_L0("Got control signal " << type << ". Exiting without saving..."); - return FALSE; - } - return TRUE; - } -#else - static void posix_handler(int /*type*/) - { - handle_signal(); - } -#endif - - static void handle_signal() - { - static std::mutex m_mutex; - std::unique_lock lock(m_mutex); - m_handler(); - } - - private: - static std::function m_handler; - }; + crypto::hash get_proof_of_trust_hash(const nodetool::proof_of_trust& pot); } diff --git a/src/common/varint.h b/src/common/varint.h index a607d832..b354462e 100644 --- a/src/common/varint.h +++ b/src/common/varint.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/connectivity_tool/conn_tool.cpp b/src/connectivity_tool/conn_tool.cpp index 3d37f311..7ec94f4b 100644 --- a/src/connectivity_tool/conn_tool.cpp +++ b/src/connectivity_tool/conn_tool.cpp @@ -1,28 +1,28 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. - - -#include "include_base_utils.h" -#include "version.h" - -using namespace epee; #include -#include "p2p/p2p_protocol_defs.h" + +// epee +#include "include_base_utils.h" +#include "net/http_client.h" +#include "net/levin_client.h" +#include "storages/http_abstract_invoke.h" +#include "storages/levin_abstract_invoke2.h" +#include "storages/portable_storage_template_helper.h" + #include "common/command_line.h" +#include "crypto/crypto.h" #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" -#include "net/levin_client.h" -#include "storages/levin_abstract_invoke2.h" -#include "cryptonote_core/cryptonote_core.h" -#include "storages/portable_storage_template_helper.h" -#include "crypto/crypto.h" -#include "storages/http_abstract_invoke.h" -#include "net/http_client.h" +#include "p2p/p2p_protocol_defs.h" +#include "rpc/core_rpc_server_commands_defs.h" +#include "version.h" namespace po = boost::program_options; using namespace cryptonote; +using namespace epee; using namespace nodetool; namespace @@ -350,4 +350,3 @@ int main(int argc, char* argv[]) return 1; } - diff --git a/src/crypto/chacha8.h b/src/crypto/chacha8.h index d28d0856..cbfe08ad 100644 --- a/src/crypto/chacha8.h +++ b/src/crypto/chacha8.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/crypto-ops-data.c b/src/crypto/crypto-ops-data.c index a3ded0c8..7fd6cd26 100644 --- a/src/crypto/crypto-ops-data.c +++ b/src/crypto/crypto-ops-data.c @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 919eb983..3f26b021 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,7 +19,6 @@ static void ge_madd(ge_p1p1 *, const ge_p3 *, const ge_precomp *); static void ge_msub(ge_p1p1 *, const ge_p3 *, const ge_precomp *); static void ge_p2_0(ge_p2 *); static void ge_p3_dbl(ge_p1p1 *, const ge_p3 *); -static void ge_sub(ge_p1p1 *, const ge_p3 *, const ge_cached *); static void fe_divpowm1(fe, const fe, const fe); /* Common functions */ @@ -1565,7 +1564,7 @@ void ge_scalarmult_base(ge_p3 *h, const unsigned char *a) { r = p - q */ -static void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q) { +void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q) { fe t0; fe_add(r->X, p->Y, p->X); fe_sub(r->Y, p->Y, p->X); diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index 4639b517..c96186a2 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -90,6 +90,10 @@ void ge_p3_tobytes(unsigned char *, const ge_p3 *); extern const ge_precomp ge_base[32][8]; void ge_scalarmult_base(ge_p3 *, const unsigned char *); +/* From ge_sub.c */ + +void ge_sub(ge_p1p1 *, const ge_p3 *, const ge_cached *); + /* From ge_tobytes.c */ void ge_tobytes(unsigned char *, const ge_p2 *); diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index e25c966d..8228d310 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,12 +20,9 @@ namespace crypto { using std::abort; using std::int32_t; - using std::int64_t; using std::lock_guard; using std::mutex; using std::size_t; - using std::uint32_t; - using std::uint64_t; extern "C" { #include "crypto-ops.h" @@ -140,6 +137,26 @@ namespace crypto { sc_add(&derived_key, &base, &scalar); } + bool crypto_ops::underive_public_key(const key_derivation &derivation, size_t output_index, + const public_key &derived_key, public_key &base) { + ec_scalar scalar; + ge_p3 point1; + ge_p3 point2; + ge_cached point3; + ge_p1p1 point4; + ge_p2 point5; + if (ge_frombytes_vartime(&point1, &derived_key) != 0) { + return false; + } + derivation_to_scalar(derivation, output_index, scalar); + ge_scalarmult_base(&point2, &scalar); + ge_p3_to_cached(&point3, &point2); + ge_sub(&point4, &point1, &point3); + ge_p1p1_to_p2(&point5, &point4); + ge_tobytes(&base, &point5); + return true; + } + struct s_comm { hash h; ec_point key; diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 253ab98b..928e655e 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -1,10 +1,11 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once #include +#include #include #include @@ -14,6 +15,8 @@ namespace crypto { + using std::size_t; + extern "C" { #include "random.h" } @@ -74,6 +77,8 @@ namespace crypto { friend bool derive_public_key(const key_derivation &, std::size_t, const public_key &, public_key &); static void derive_secret_key(const key_derivation &, std::size_t, const secret_key &, secret_key &); friend void derive_secret_key(const key_derivation &, std::size_t, const secret_key &, secret_key &); + static bool underive_public_key(const key_derivation &, std::size_t, const public_key &, public_key &); + friend bool underive_public_key(const key_derivation &, std::size_t, const public_key &, public_key &); static void generate_signature(const hash &, const public_key &, const secret_key &, signature &); friend void generate_signature(const hash &, const public_key &, const secret_key &, signature &); static bool check_signature(const hash &, const public_key &, const signature &); @@ -100,6 +105,35 @@ namespace crypto { return res; } + /* Random number engine based on crypto::rand() + */ + template + class random_engine { + public: + typedef T result_type; + +#ifdef __clang__ + constexpr static T min() { + return (std::numeric_limits::min)(); + } + + constexpr static T max() { + return (std::numeric_limits::max)(); + } +#else + static T(min)() { + return (std::numeric_limits::min)(); + } + + static T(max)() { + return (std::numeric_limits::max)(); + } +#endif + typename std::enable_if::value, T>::type operator()() { + return rand(); + } + }; + /* Generate a new key pair */ inline void generate_keys(public_key &pub, secret_key &sec) { @@ -120,8 +154,8 @@ namespace crypto { /* To generate an ephemeral key used to send money to: * * The sender generates a new key pair, which becomes the transaction key. The public transaction key is included in "extra" field. - * * Both the sender and the receiver generate key derivation from the transaction key, the receivers' "view" key and the output index. - * * The sender uses key derivation and the receivers' "spend" key to derive an ephemeral public key. + * * Both the sender and the receiver generate key derivation from the transaction key and the receivers' "view" key. + * * The sender uses key derivation, the output index, and the receivers' "spend" key to derive an ephemeral public key. * * The receiver can either derive the public key (to check that the transaction is addressed to him) or the private key (to spend the money). */ inline bool generate_key_derivation(const public_key &key1, const secret_key &key2, key_derivation &derivation) { @@ -136,6 +170,13 @@ namespace crypto { crypto_ops::derive_secret_key(derivation, output_index, base, derived_key); } + /* Inverse function of derive_public_key. It can be used by the receiver to find which "spend" key was used to generate a transaction. This may be useful if the receiver used multiple addresses which only differ in "spend" key. + */ + inline bool underive_public_key(const key_derivation &derivation, std::size_t output_index, + const public_key &derived_key, public_key &base) { + return crypto_ops::underive_public_key(derivation, output_index, derived_key, base); + } + /* Generation and checking of a standard signature. */ inline void generate_signature(const hash &prefix_hash, const public_key &pub, const secret_key &sec, signature &sig) { @@ -181,6 +222,6 @@ namespace crypto { } } -CRYPTO_MAKE_COMPARABLE(public_key) +CRYPTO_MAKE_HASHABLE(public_key) CRYPTO_MAKE_HASHABLE(key_image) CRYPTO_MAKE_COMPARABLE(signature) diff --git a/src/crypto/generic-ops.h b/src/crypto/generic-ops.h index 378500c2..957fb4ce 100644 --- a/src/crypto/generic-ops.h +++ b/src/crypto/generic-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/hash-extra-blake.c b/src/crypto/hash-extra-blake.c index 50df08aa..775e84f4 100644 --- a/src/crypto/hash-extra-blake.c +++ b/src/crypto/hash-extra-blake.c @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/hash-extra-groestl.c b/src/crypto/hash-extra-groestl.c index 9d7b8b43..2d94b189 100644 --- a/src/crypto/hash-extra-groestl.c +++ b/src/crypto/hash-extra-groestl.c @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/hash-extra-jh.c b/src/crypto/hash-extra-jh.c index 9bbe77c5..95c32da2 100644 --- a/src/crypto/hash-extra-jh.c +++ b/src/crypto/hash-extra-jh.c @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/hash-extra-skein.c b/src/crypto/hash-extra-skein.c index 60fbcccd..05b81d22 100644 --- a/src/crypto/hash-extra-skein.c +++ b/src/crypto/hash-extra-skein.c @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index 2b464716..ed07e558 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,6 +11,7 @@ #include #include +#include "../common/static_assert.h" #include "common/int-util.h" #include "warnings.h" @@ -63,3 +64,6 @@ void hash_extra_jh(const void *data, size_t length, char *hash); void hash_extra_skein(const void *data, size_t length, char *hash); void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash); +size_t tree_depth(size_t count); +void tree_branch(const char (*hashes)[HASH_SIZE], size_t count, char (*branch)[HASH_SIZE]); +void tree_hash_from_branch(const char (*branch)[HASH_SIZE], size_t depth, const char *leaf, const void *path, char *root_hash); diff --git a/src/crypto/hash.c b/src/crypto/hash.c index c86418d7..ae1a56f3 100644 --- a/src/crypto/hash.c +++ b/src/crypto/hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/hash.h b/src/crypto/hash.h index 6730e148..f2dc4dc1 100644 --- a/src/crypto/hash.h +++ b/src/crypto/hash.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -61,6 +61,14 @@ namespace crypto { tree_hash(reinterpret_cast(hashes), count, reinterpret_cast(&root_hash)); } + inline void tree_branch(const hash *hashes, std::size_t count, hash *branch) { + tree_branch(reinterpret_cast(hashes), count, reinterpret_cast(branch)); + } + + inline void tree_hash_from_branch(const hash *branch, std::size_t depth, const hash &leaf, const void *path, hash &root_hash) { + tree_hash_from_branch(reinterpret_cast(branch), depth, reinterpret_cast(&leaf), path, reinterpret_cast(&root_hash)); + } + } CRYPTO_MAKE_HASHABLE(hash) diff --git a/src/crypto/initializer.h b/src/crypto/initializer.h index 611ae16c..5506e2e4 100644 --- a/src/crypto/initializer.h +++ b/src/crypto/initializer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/random.c b/src/crypto/random.c index a17cb47d..be727cf8 100644 --- a/src/crypto/random.c +++ b/src/crypto/random.c @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2013 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/random.h b/src/crypto/random.h index afc8f3b2..34f3075b 100644 --- a/src/crypto/random.h +++ b/src/crypto/random.h @@ -1,9 +1,11 @@ -// Copyright (c) 2012-2013 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once +#if !defined(__cplusplus) #include +#endif void generate_random_bytes(size_t n, void *result); diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 7baca1d4..db18519a 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/slow-hash.cpp b/src/crypto/slow-hash.cpp index 7141359a..d852d665 100644 --- a/src/crypto/slow-hash.cpp +++ b/src/crypto/slow-hash.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2013 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/slow-hash.inl b/src/crypto/slow-hash.inl index 0d9c3573..718eb78b 100644 --- a/src/crypto/slow-hash.inl +++ b/src/crypto/slow-hash.inl @@ -1,3 +1,7 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + static void #if defined(AESNI) cn_slow_hash_aesni diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index 87423fb8..b64e0f08 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2013 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,7 +19,7 @@ void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { size_t i, j; size_t cnt = count - 1; char (*ints)[HASH_SIZE]; - for (i = 1; i < 8 * sizeof(size_t); i <<= 1) { + for (i = 1; i < sizeof(size_t); i <<= 1) { cnt |= cnt >> i; } cnt &= ~(cnt >> 1); @@ -38,3 +38,75 @@ void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { cn_fast_hash(ints[0], 64, root_hash); } } + +size_t tree_depth(size_t count) { + size_t i; + size_t depth = 0; + assert(count > 0); + for (i = sizeof(size_t) << 2; i > 0; i >>= 1) { + if (count >> i > 0) { + count >>= i; + depth += i; + } + } + return depth; +} + +void tree_branch(const char (*hashes)[HASH_SIZE], size_t count, char (*branch)[HASH_SIZE]) { + size_t i, j; + size_t cnt = 1; + size_t depth = 0; + char (*ints)[HASH_SIZE]; + assert(count > 0); + for (i = sizeof(size_t) << 2; i > 0; i >>= 1) { + if (cnt << i <= count) { + cnt <<= i; + depth += i; + } + } + assert(cnt == 1ULL << depth); + assert(depth == tree_depth(count)); + ints = alloca((cnt - 1) * HASH_SIZE); + memcpy(ints, hashes + 1, (2 * cnt - count - 1) * HASH_SIZE); + for (i = 2 * cnt - count, j = 2 * cnt - count - 1; j < cnt - 1; i += 2, ++j) { + cn_fast_hash(hashes[i], 2 * HASH_SIZE, ints[j]); + } + assert(i == count); + while (depth > 0) { + assert(cnt == 1ULL << depth); + cnt >>= 1; + --depth; + memcpy(branch[depth], ints[0], HASH_SIZE); + for (i = 1, j = 0; j < cnt - 1; i += 2, ++j) { + cn_fast_hash(ints[i], 2 * HASH_SIZE, ints[j]); + } + } +} + +void tree_hash_from_branch(const char (*branch)[HASH_SIZE], size_t depth, const char *leaf, const void *path, char *root_hash) { + if (depth == 0) { + memcpy(root_hash, leaf, HASH_SIZE); + } else { + char buffer[2][HASH_SIZE]; + int from_leaf = 1; + char *leaf_path, *branch_path; + while (depth > 0) { + --depth; + if (path && (((const char *) path)[depth >> 3] & (1 << (depth & 7))) != 0) { + leaf_path = buffer[1]; + branch_path = buffer[0]; + } else { + leaf_path = buffer[0]; + branch_path = buffer[1]; + } + if (from_leaf) { + memcpy(leaf_path, leaf, HASH_SIZE); + from_leaf = 0; + } else { + cn_fast_hash(buffer, 2 * HASH_SIZE, leaf_path); + } + memcpy(branch_path, branch[depth], HASH_SIZE); + } + cn_fast_hash(buffer, 2 * HASH_SIZE, root_hash); + } +} diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 405ef672..bdf7b546 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -1,94 +1,125 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once -#define CRYPTONOTE_MAX_BLOCK_NUMBER 500000000 -#define CRYPTONOTE_MAX_BLOCK_SIZE 500000000 // block header blob limit, never used! -#define CRYPTONOTE_MAX_TX_SIZE 1000000000 -#define CRYPTONOTE_PUBLIC_ADDRESS_TEXTBLOB_VER 0 -//TODO Currency-specific address prefix -#define CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX -//TODO Choose maturity period for your currency -#define CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW -#define CURRENT_TRANSACTION_VERSION 1 -#define CURRENT_BLOCK_MAJOR_VERSION 1 -#define CURRENT_BLOCK_MINOR_VERSION 0 -#define CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT 60*60*2 +#include +#include -#define BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW 60 +namespace cryptonote { +namespace parameters { + +const uint64_t CRYPTONOTE_MAX_BLOCK_NUMBER = 500000000; +const size_t CRYPTONOTE_MAX_BLOCK_BLOB_SIZE = 500000000; +const size_t CRYPTONOTE_MAX_TX_SIZE = 1000000000; +//TODO Currency-specific address prefix +const uint64_t CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX = ; +//TODO Choose maturity period for your currency +const size_t CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW = 60; +const uint64_t CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT = 60 * 60 * 2; + +const size_t BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW = 60; //TODO Specify total number of available coins //TODO ((uint64_t)(-1)) equals to 18446744073709551616 coins //TODO or you can define number explicitly UINT64_C(858986905600000000) -#define MONEY_SUPPLY -#define EMISSION_SPEED_FACTOR (18) +const uint64_t MONEY_SUPPLY = ; +const unsigned EMISSION_SPEED_FACTOR = 18; +static_assert(EMISSION_SPEED_FACTOR <= 8 * sizeof(uint64_t), "Bad EMISSION_SPEED_FACTOR"); //TODO Define number of blocks for block size median calculation -#define CRYPTONOTE_REWARD_BLOCKS_WINDOW 100 -#define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE 10000 //size of block (bytes) after which reward for block calculated using block size -#define CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE 600 +const size_t CRYPTONOTE_REWARD_BLOCKS_WINDOW = 100; +const size_t CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE = 10000; //size of block (bytes) after which reward for block calculated using block size +const size_t CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE = 600; //TODO Define number of digits -#define CRYPTONOTE_DISPLAY_DECIMAL_POINT -//TODO Define number of smallest units in one coin -#define COIN -//TODO Define default fee for transactions -#define DEFAULT_FEE +const size_t CRYPTONOTE_DISPLAY_DECIMAL_POINT = 8; +//TODO Define minimum fee for transactions +const uint64_t MINIMUM_FEE = ; +const uint64_t DEFAULT_DUST_THRESHOLD = MINIMUM_FEE; //TODO Define preferred block's target time -#define DIFFICULTY_TARGET 120 // seconds -//TODO There are options to tune CryptoNote's difficulty retargeting function. +const uint64_t DIFFICULTY_TARGET = 120; // seconds +const uint64_t EXPECTED_NUMBER_OF_BLOCKS_PER_DAY = 24 * 60 * 60 / DIFFICULTY_TARGET; +//TODO There are options to tune CryptoNote's difficulty retargeting function. //TODO We recommend not to change it. -#define DIFFICULTY_WINDOW 720 // blocks -#define DIFFICULTY_LAG 15 -#define DIFFICULTY_CUT 60 // timestamps to cut after sorting -#define DIFFICULTY_BLOCKS_COUNT DIFFICULTY_WINDOW + DIFFICULTY_LAG +const size_t DIFFICULTY_WINDOW = EXPECTED_NUMBER_OF_BLOCKS_PER_DAY; // blocks +const size_t DIFFICULTY_CUT = 60; // timestamps to cut after sorting +const size_t DIFFICULTY_LAG = 15; +static_assert(2 * DIFFICULTY_CUT <= DIFFICULTY_WINDOW - 2, "Bad DIFFICULTY_WINDOW or DIFFICULTY_CUT"); +const size_t MAX_BLOCK_SIZE_INITIAL = 20 * 1024; +const uint64_t MAX_BLOCK_SIZE_GROWTH_SPEED_NUMERATOR = 100 * 1024; +const uint64_t MAX_BLOCK_SIZE_GROWTH_SPEED_DENOMINATOR = 365 * 24 * 60 * 60 / DIFFICULTY_TARGET; -#define CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS DIFFICULTY_TARGET * CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS -#define CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS 1 +const uint64_t CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS = 1; +const uint64_t CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS = DIFFICULTY_TARGET * CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS; +const uint64_t CRYPTONOTE_MEMPOOL_TX_LIVETIME = 60 * 60 * 24; //seconds, one day +const uint64_t CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME = 60 * 60 * 24 * 7; //seconds, one week -#define DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN DIFFICULTY_TARGET - - -#define BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT 10000 //by default, blocks ids count in synchronizing -#define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT 200 //by default, blocks count in blocks downloading -#define CRYPTONOTE_PROTOCOL_HOP_RELAX_COUNT 3 //value of hop, after which we use only announce of new block - -//TODO This port will be used by the daemon to establish connections with p2p network -#define P2P_DEFAULT_PORT -//TODO This port will be used by the daemon to interact with simlewallet -#define RPC_DEFAULT_PORT -#define COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT 1000 - -#define P2P_LOCAL_WHITE_PEERLIST_LIMIT 1000 -#define P2P_LOCAL_GRAY_PEERLIST_LIMIT 5000 - -#define P2P_DEFAULT_CONNECTIONS_COUNT 8 -#define P2P_DEFAULT_HANDSHAKE_INTERVAL 60 //secondes -#define P2P_DEFAULT_PACKET_MAX_SIZE 50000000 //50000000 bytes maximum packet size -#define P2P_DEFAULT_PEERS_IN_HANDSHAKE 250 -#define P2P_DEFAULT_CONNECTION_TIMEOUT 5000 //5 seconds -#define P2P_DEFAULT_PING_CONNECTION_TIMEOUT 2000 //2 seconds -#define P2P_DEFAULT_INVOKE_TIMEOUT 60*2*1000 //2 minutes -#define P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT 5000 //5 seconds -#define P2P_STAT_TRUSTED_PUB_KEY "8f80f9a5a434a9f1510d13336228debfee9c918ce505efe225d8c94d045fa115" -#define P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT 70 - -#define ALLOW_DEBUG_COMMANDS +const char CRYPTONOTE_BLOCKS_FILENAME[] = "blocks.dat"; +const char CRYPTONOTE_BLOCKINDEXES_FILENAME[] = "blockindexes.dat"; +const char CRYPTONOTE_BLOCKSCACHE_FILENAME[] = "blockscache.dat"; +const char CRYPTONOTE_POOLDATA_FILENAME[] = "poolstate.bin"; +const char P2P_NET_DATA_FILENAME[] = "p2pstate.bin"; +const char MINER_CONFIG_FILE_NAME[] = "miner_conf.json"; +} // parameters //TODO Put here the name of your currency -#define CRYPTONOTE_NAME -#define CRYPTONOTE_BLOCKS_FILENAME "blocks.dat" -#define CRYPTONOTE_BLOCKINDEXES_FILENAME "blockindexes.dat" -#define CRYPTONOTE_BLOCKSCACHE_FILENAME "blockscache.dat" -#define CRYPTONOTE_POOLDATA_FILENAME "poolstate.bin" -#define P2P_NET_DATA_FILENAME "p2pstate.bin" -#define MINER_CONFIG_FILE_NAME "miner_conf.json" +const char CRYPTONOTE_NAME[] = ""; +const char GENESIS_COINBASE_TX_HEX[] = ""; -#define THREAD_STACK_SIZE 5 * 1024 * 1024 +const uint8_t CURRENT_TRANSACTION_VERSION = 1; +const uint8_t BLOCK_MAJOR_VERSION_1 = 1; +const uint8_t BLOCK_MINOR_VERSION_0 = 0; -#define GENESIS_COINBASE_TX_HEX "" +const size_t BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT = 10000; //by default, blocks ids count in synchronizing +const size_t BLOCKS_SYNCHRONIZING_DEFAULT_COUNT = 200; //by default, blocks count in blocks downloading +const size_t COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT = 1000; +//TODO This port will be used by the daemon to establish connections with p2p network +const int P2P_DEFAULT_PORT = ; +//TODO This port will be used by the daemon to interact with simlewallet +const int RPC_DEFAULT_PORT = ; + +const size_t P2P_LOCAL_WHITE_PEERLIST_LIMIT = 1000; +const size_t P2P_LOCAL_GRAY_PEERLIST_LIMIT = 5000; + +const uint32_t P2P_DEFAULT_CONNECTIONS_COUNT = 8; +const uint32_t P2P_DEFAULT_HANDSHAKE_INTERVAL = 60; // seconds +const uint32_t P2P_DEFAULT_PACKET_MAX_SIZE = 50000000; // 50000000 bytes maximum packet size +const uint32_t P2P_DEFAULT_PEERS_IN_HANDSHAKE = 250; +const uint32_t P2P_DEFAULT_CONNECTION_TIMEOUT = 5000; // 5 seconds +const uint32_t P2P_DEFAULT_PING_CONNECTION_TIMEOUT = 2000; // 2 seconds +const uint64_t P2P_DEFAULT_INVOKE_TIMEOUT = 60 * 2 * 1000; // 2 minutes +const size_t P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT = 5000; // 5 seconds +const char P2P_STAT_TRUSTED_PUB_KEY[] = "8f80f9a5a434a9f1510d13336228debfee9c918ce505efe225d8c94d045fa115"; +const size_t P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT = 70; + +const unsigned THREAD_STACK_SIZE = 5 * 1024 * 1024; + +//TODO Add here your network seed nodes +const std::initializer_list SEED_NODES = { + //"your_seed_ip1.com:8080", + //"your_seed_ip2.com:8080", +}; + +struct CheckpointData { + uint64_t height; + const char* blockId; +}; + +#ifdef __GNUC__ +__attribute__((unused)) +#endif + +// You may add here other checkpoints using the following format: +// {, ""}, +const std::initializer_list CHECKPOINTS = { + //{ 10000, "84b6345731e2702cdaadc6ce5e5238c4ca5ecf48e3447136b2ed829b8a95f3ad" }, +}; + +} // cryptonote + +#define ALLOW_DEBUG_COMMANDS diff --git a/src/cryptonote_core/AccountKVSerialization.h b/src/cryptonote_core/AccountKVSerialization.h new file mode 100644 index 00000000..8fa45631 --- /dev/null +++ b/src/cryptonote_core/AccountKVSerialization.h @@ -0,0 +1,100 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include "cryptonote_core/account.h" + +// epee +#include "serialization/keyvalue_serialization.h" + +namespace cryptonote { + template struct AccountPublicAddressSerializer; + template struct AccountKeysSerializer; + template struct AccountBaseSerializer; + + template<> + struct AccountPublicAddressSerializer { + const AccountPublicAddress& m_account_address; + + AccountPublicAddressSerializer(const AccountPublicAddress& account_address) : m_account_address(account_address) { + } + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(m_account_address.m_spendPublicKey, "m_spend_public_key") + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(m_account_address.m_viewPublicKey, "m_view_public_key") + END_KV_SERIALIZE_MAP() + }; + + template<> + struct AccountPublicAddressSerializer { + AccountPublicAddress& m_account_address; + + AccountPublicAddressSerializer(AccountPublicAddress& account_address) : m_account_address(account_address) { + } + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(m_account_address.m_spendPublicKey, "m_spend_public_key") + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(m_account_address.m_viewPublicKey, "m_view_public_key") + END_KV_SERIALIZE_MAP() + }; + + template<> + struct AccountKeysSerializer { + const account_keys& m_keys; + + AccountKeysSerializer(const account_keys& keys) : m_keys(keys) { + } + + BEGIN_KV_SERIALIZE_MAP() + AccountPublicAddressSerializer addressSerializer(this_ref.m_keys.m_account_address); + epee::serialization::selector::serialize(addressSerializer, stg, hparent_section, "m_account_address"); + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(m_keys.m_spend_secret_key, "m_spend_secret_key") + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(m_keys.m_view_secret_key, "m_view_secret_key") + END_KV_SERIALIZE_MAP() + }; + + template<> + struct AccountKeysSerializer { + account_keys& m_keys; + + AccountKeysSerializer(account_keys& keys) : m_keys(keys) { + } + + BEGIN_KV_SERIALIZE_MAP() + AccountPublicAddressSerializer addressSerializer(this_ref.m_keys.m_account_address); + epee::serialization::selector::serialize(addressSerializer, stg, hparent_section, "m_account_address"); + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(m_keys.m_spend_secret_key, "m_spend_secret_key") + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(m_keys.m_view_secret_key, "m_view_secret_key") + END_KV_SERIALIZE_MAP() + }; + + template<> + struct AccountBaseSerializer { + const account_base& m_account; + + AccountBaseSerializer(const account_base& account) : m_account(account) { + } + + BEGIN_KV_SERIALIZE_MAP() + AccountKeysSerializer keysSerializer(this_ref.m_account.m_keys); + epee::serialization::selector::serialize(keysSerializer, stg, hparent_section, "m_keys"); + KV_SERIALIZE_N(m_account.m_creation_timestamp, "m_creation_timestamp") + END_KV_SERIALIZE_MAP() + }; + + template<> + struct AccountBaseSerializer { + account_base& m_account; + + AccountBaseSerializer(account_base& account) : m_account(account) { + } + + BEGIN_KV_SERIALIZE_MAP() + AccountKeysSerializer keysSerializer(this_ref.m_account.m_keys); + epee::serialization::selector::serialize(keysSerializer, stg, hparent_section, "m_keys"); + KV_SERIALIZE_N(m_account.m_creation_timestamp, "m_creation_timestamp") + END_KV_SERIALIZE_MAP() + }; +} diff --git a/src/cryptonote_core/BlockIndex.cpp b/src/cryptonote_core/BlockIndex.cpp new file mode 100644 index 00000000..9873f39b --- /dev/null +++ b/src/cryptonote_core/BlockIndex.cpp @@ -0,0 +1,74 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "BlockIndex.h" +#include + +namespace CryptoNote +{ + crypto::hash BlockIndex::getBlockId(uint64_t height) const { + if (height >= m_container.size()) + return boost::value_initialized(); + return m_container[static_cast(height)]; + } + + bool BlockIndex::getBlockIds(uint64_t startHeight, size_t maxCount, std::list& items) const { + if (startHeight >= m_container.size()) + return false; + + for (size_t i = startHeight; i < (startHeight + maxCount) && i < m_container.size(); ++i) { + items.push_back(m_container[i]); + } + + return true; + } + + + bool BlockIndex::findSupplement(const std::list& ids, uint64_t& offset) const { + + for (const auto& id : ids) { + if (getBlockHeight(id, offset)) + return true; + } + + return false; + } + + bool BlockIndex::getShortChainHistory(std::list& ids) const { + size_t i = 0; + size_t current_multiplier = 1; + size_t sz = size(); + + if (!sz) + return true; + + size_t current_back_offset = 1; + bool genesis_included = false; + + while (current_back_offset < sz) { + ids.push_back(m_container[sz - current_back_offset]); + if (sz - current_back_offset == 0) + genesis_included = true; + if (i < 10) { + ++current_back_offset; + } else { + current_back_offset += current_multiplier *= 2; + } + ++i; + } + + if (!genesis_included) + ids.push_back(m_container[0]); + + return true; + } + + crypto::hash BlockIndex::getTailId() const { + if (m_container.empty()) + return boost::value_initialized(); + return m_container.back(); + } + + +} diff --git a/src/cryptonote_core/BlockIndex.h b/src/cryptonote_core/BlockIndex.h new file mode 100644 index 00000000..5fce8117 --- /dev/null +++ b/src/cryptonote_core/BlockIndex.h @@ -0,0 +1,77 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// multi index +#include +#include +#include + +#include "crypto/hash.h" +#include + +namespace CryptoNote +{ + class BlockIndex { + + public: + + BlockIndex() : + m_index(m_container.get<1>()) {} + + void pop() { + m_container.pop_back(); + } + + // returns true if new element was inserted, false if already exists + bool push(const crypto::hash& h) { + auto result = m_container.push_back(h); + return result.second; + } + + bool hasBlock(const crypto::hash& h) const { + return m_index.find(h) != m_index.end(); + } + + bool getBlockHeight(const crypto::hash& h, uint64_t& height) const { + auto hi = m_index.find(h); + if (hi == m_index.end()) + return false; + + height = std::distance(m_container.begin(), m_container.project<0>(hi)); + return true; + } + + size_t size() const { + return m_container.size(); + } + + void clear() { + m_container.clear(); + } + + crypto::hash getBlockId(uint64_t height) const; + bool getBlockIds(uint64_t startHeight, size_t maxCount, std::list& items) const; + bool findSupplement(const std::list& ids, uint64_t& offset) const; + bool getShortChainHistory(std::list& ids) const; + crypto::hash getTailId() const; + + template void serialize(Archive& ar, const unsigned int version) { + ar & m_container; + } + + private: + + typedef boost::multi_index_container < + crypto::hash, + boost::multi_index::indexed_by< + boost::multi_index::random_access<>, + boost::multi_index::hashed_unique> + > + > ContainerT; + + ContainerT m_container; + ContainerT::nth_index<1>::type& m_index; + + }; +} diff --git a/src/cryptonote_core/CoreConfig.cpp b/src/cryptonote_core/CoreConfig.cpp new file mode 100644 index 00000000..7953797a --- /dev/null +++ b/src/cryptonote_core/CoreConfig.cpp @@ -0,0 +1,23 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "CoreConfig.h" + +#include "common/util.h" +#include "common/command_line.h" + +namespace cryptonote { + +CoreConfig::CoreConfig() { + configFolder = tools::get_default_data_dir(); +} + +void CoreConfig::init(const boost::program_options::variables_map& options) { + configFolder = command_line::get_arg(options, command_line::arg_data_dir); +} + +void CoreConfig::initOptions(boost::program_options::options_description& desc) { +} +} //namespace cryptonote + diff --git a/src/cryptonote_core/CoreConfig.h b/src/cryptonote_core/CoreConfig.h new file mode 100644 index 00000000..e3d44b7e --- /dev/null +++ b/src/cryptonote_core/CoreConfig.h @@ -0,0 +1,23 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include + +#include + +namespace cryptonote { + +class CoreConfig { +public: + CoreConfig(); + + static void initOptions(boost::program_options::options_description& desc); + void init(const boost::program_options::variables_map& options); + + std::string configFolder; +}; + +} //namespace cryptonote diff --git a/src/cryptonote_core/Currency.cpp b/src/cryptonote_core/Currency.cpp new file mode 100644 index 00000000..00862bdb --- /dev/null +++ b/src/cryptonote_core/Currency.cpp @@ -0,0 +1,350 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "Currency.h" + +#include +#include + +// epee +#include "include_base_utils.h" +#include "string_tools.h" + +#include "common/base58.h" +#include "common/int-util.h" +#include "cryptonote_core/account.h" +#include "cryptonote_core/cryptonote_format_utils.h" + +namespace cryptonote { + bool Currency::init() { + bool r; + r = generateGenesisBlock(); + CHECK_AND_ASSERT_MES(r, false, "Failed to generate genesis block"); + + r = get_block_hash(m_genesisBlock, m_genesisBlockHash); + CHECK_AND_ASSERT_MES(r, false, "Failed to get genesis block hash"); + + if (isTestnet()) { + m_blocksFileName = "testnet_" + m_blocksFileName; + m_blocksCacheFileName = "testnet_" + m_blocksCacheFileName; + m_blockIndexesFileName = "testnet_" + m_blockIndexesFileName; + m_txPoolFileName = "testnet_" + m_txPoolFileName; + } + + return true; + } + + bool Currency::generateGenesisBlock() { + m_genesisBlock = boost::value_initialized(); + + // Hard code coinbase tx in genesis block, because "tru" generating tx use random, but genesis should be always the same + std::string genesisCoinbaseTxHex = GENESIS_COINBASE_TX_HEX; + + blobdata minerTxBlob; + epee::string_tools::parse_hexstr_to_binbuff(genesisCoinbaseTxHex, minerTxBlob); + bool r = parse_and_validate_tx_from_blob(minerTxBlob, m_genesisBlock.minerTx); + CHECK_AND_ASSERT_MES(r, false, "failed to parse coinbase tx from hard coded blob"); + m_genesisBlock.majorVersion = BLOCK_MAJOR_VERSION_1; + m_genesisBlock.minorVersion = BLOCK_MINOR_VERSION_0; + m_genesisBlock.timestamp = 0; + m_genesisBlock.nonce = 70; + if (m_testnet) { + ++m_genesisBlock.nonce; + } + //miner::find_nonce_for_given_block(bl, 1, 0); + + return true; + } + + bool Currency::getBlockReward(size_t medianSize, size_t currentBlockSize, uint64_t alreadyGeneratedCoins, + uint64_t fee, uint64_t& reward, int64_t& emissionChange) const { + assert(alreadyGeneratedCoins <= m_moneySupply); + assert(m_emissionSpeedFactor > 0 && m_emissionSpeedFactor <= 8 * sizeof(uint64_t)); + + uint64_t baseReward = (m_moneySupply - alreadyGeneratedCoins) >> m_emissionSpeedFactor; + + medianSize = std::max(medianSize, m_blockGrantedFullRewardZone); + if (currentBlockSize > UINT64_C(2) * medianSize) { + LOG_PRINT_L4("Block cumulative size is too big: " << currentBlockSize << ", expected less than " << 2 * medianSize); + return false; + } + + uint64_t penalizedBaseReward = getPenalizedAmount(baseReward, medianSize, currentBlockSize); + uint64_t penalizedFee = getPenalizedAmount(fee, medianSize, currentBlockSize); + + emissionChange = penalizedBaseReward - (fee - penalizedFee); + reward = penalizedBaseReward + penalizedFee; + + return true; + } + + size_t Currency::maxBlockCumulativeSize(uint64_t height) const { + assert(height <= std::numeric_limits::max() / m_maxBlockSizeGrowthSpeedNumerator); + size_t maxSize = static_cast(m_maxBlockSizeInitial + + (height * m_maxBlockSizeGrowthSpeedNumerator) / m_maxBlockSizeGrowthSpeedDenominator); + assert(maxSize >= m_maxBlockSizeInitial); + return maxSize; + } + + bool Currency::constructMinerTx(size_t height, size_t medianSize, uint64_t alreadyGeneratedCoins, size_t currentBlockSize, + uint64_t fee, const AccountPublicAddress& minerAddress, Transaction& tx, + const blobdata& extraNonce/* = blobdata()*/, size_t maxOuts/* = 1*/) const { + tx.vin.clear(); + tx.vout.clear(); + tx.extra.clear(); + + KeyPair txkey = KeyPair::generate(); + add_tx_pub_key_to_extra(tx, txkey.pub); + if (!extraNonce.empty()) { + if (!add_extra_nonce_to_tx_extra(tx.extra, extraNonce)) { + return false; + } + } + + TransactionInputGenerate in; + in.height = height; + + uint64_t blockReward; + int64_t emissionChange; + if (!getBlockReward(medianSize, currentBlockSize, alreadyGeneratedCoins, fee, blockReward, emissionChange)) { + LOG_PRINT_L0("Block is too big"); + return false; + } +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: reward " << blockReward << ", fee " << fee); +#endif + + std::vector outAmounts; + decompose_amount_into_digits(blockReward, m_defaultDustThreshold, + [&outAmounts](uint64_t a_chunk) { outAmounts.push_back(a_chunk); }, + [&outAmounts](uint64_t a_dust) { outAmounts.push_back(a_dust); }); + + CHECK_AND_ASSERT_MES(1 <= maxOuts, false, "max_out must be non-zero"); + while (maxOuts < outAmounts.size()) { + outAmounts[outAmounts.size() - 2] += outAmounts.back(); + outAmounts.resize(outAmounts.size() - 1); + } + + uint64_t summaryAmounts = 0; + for (size_t no = 0; no < outAmounts.size(); no++) { + crypto::key_derivation derivation = boost::value_initialized(); + crypto::public_key outEphemeralPubKey = boost::value_initialized(); + bool r = crypto::generate_key_derivation(minerAddress.m_viewPublicKey, txkey.sec, derivation); + CHECK_AND_ASSERT_MES(r, false, "while creating outs: failed to generate_key_derivation(" << + minerAddress.m_viewPublicKey << ", " << txkey.sec << ")"); + + r = crypto::derive_public_key(derivation, no, minerAddress.m_spendPublicKey, outEphemeralPubKey); + CHECK_AND_ASSERT_MES(r, false, "while creating outs: failed to derive_public_key(" << derivation << ", " << + no << ", "<< minerAddress.m_spendPublicKey << ")"); + + TransactionOutputToKey tk; + tk.key = outEphemeralPubKey; + + TransactionOutput out; + summaryAmounts += out.amount = outAmounts[no]; + out.target = tk; + tx.vout.push_back(out); + } + + CHECK_AND_ASSERT_MES(summaryAmounts == blockReward, false, + "Failed to construct miner tx, summaryAmounts = " << summaryAmounts << " not equal blockReward = " << blockReward); + + tx.version = CURRENT_TRANSACTION_VERSION; + //lock + tx.unlockTime = height + m_minedMoneyUnlockWindow; + tx.vin.push_back(in); + return true; + } + + std::string Currency::accountAddressAsString(const account_base& account) const { + return getAccountAddressAsStr(m_publicAddressBase58Prefix, account.get_keys().m_account_address); + } + + bool Currency::parseAccountAddressString(const std::string& str, AccountPublicAddress& addr) const { + uint64_t prefix; + if (!cryptonote::parseAccountAddressString(prefix, addr, str)) { + return false; + } + + if (prefix != m_publicAddressBase58Prefix) { + LOG_PRINT_L1("Wrong address prefix: " << prefix << ", expected " << m_publicAddressBase58Prefix); + return false; + } + + return true; + } + + std::string Currency::formatAmount(uint64_t amount) const { + std::string s = std::to_string(amount); + if (s.size() < m_numberOfDecimalPlaces + 1) { + s.insert(0, m_numberOfDecimalPlaces + 1 - s.size(), '0'); + } + s.insert(s.size() - m_numberOfDecimalPlaces, "."); + return s; + } + + bool Currency::parseAmount(const std::string& str, uint64_t& amount) const { + std::string strAmount = str; + boost::algorithm::trim(strAmount); + + size_t pointIndex = strAmount.find_first_of('.'); + size_t fractionSize; + if (std::string::npos != pointIndex) { + fractionSize = strAmount.size() - pointIndex - 1; + while (m_numberOfDecimalPlaces < fractionSize && '0' == strAmount.back()) { + strAmount.erase(strAmount.size() - 1, 1); + --fractionSize; + } + if (m_numberOfDecimalPlaces < fractionSize) { + return false; + } + strAmount.erase(pointIndex, 1); + } else { + fractionSize = 0; + } + + if (strAmount.empty()) { + return false; + } + + if (fractionSize < m_numberOfDecimalPlaces) { + strAmount.append(m_numberOfDecimalPlaces - fractionSize, '0'); + } + + return epee::string_tools::get_xtype_from_string(amount, strAmount); + } + + difficulty_type Currency::nextDifficulty(std::vector timestamps, + std::vector cumulativeDifficulties) const { + assert(m_difficultyWindow >= 2); + + if (timestamps.size() > m_difficultyWindow) { + timestamps.resize(m_difficultyWindow); + cumulativeDifficulties.resize(m_difficultyWindow); + } + + size_t length = timestamps.size(); + assert(length == cumulativeDifficulties.size()); + assert(length <= m_difficultyWindow); + if (length <= 1) { + return 1; + } + + sort(timestamps.begin(), timestamps.end()); + + size_t cutBegin, cutEnd; + assert(2 * m_difficultyCut <= m_difficultyWindow - 2); + if (length <= m_difficultyWindow - 2 * m_difficultyCut) { + cutBegin = 0; + cutEnd = length; + } else { + cutBegin = (length - (m_difficultyWindow - 2 * m_difficultyCut) + 1) / 2; + cutEnd = cutBegin + (m_difficultyWindow - 2 * m_difficultyCut); + } + assert(/*cut_begin >= 0 &&*/ cutBegin + 2 <= cutEnd && cutEnd <= length); + uint64_t timeSpan = timestamps[cutEnd - 1] - timestamps[cutBegin]; + if (timeSpan == 0) { + timeSpan = 1; + } + + difficulty_type totalWork = cumulativeDifficulties[cutEnd - 1] - cumulativeDifficulties[cutBegin]; + assert(totalWork > 0); + + uint64_t low, high; + low = mul128(totalWork, m_difficultyTarget, &high); + if (high != 0 || low + timeSpan - 1 < low) { + return 0; + } + + return (low + timeSpan - 1) / timeSpan; + } + + bool Currency::checkProofOfWork(crypto::cn_context& context, const Block& block, difficulty_type currentDiffic, crypto::hash& proofOfWork) const { + if (!get_block_longhash(context, block, proofOfWork)) { + return false; + } + + return check_hash(proofOfWork, currentDiffic); + } + + CurrencyBuilder::CurrencyBuilder() { + maxBlockNumber(parameters::CRYPTONOTE_MAX_BLOCK_NUMBER); + maxBlockBlobSize(parameters::CRYPTONOTE_MAX_BLOCK_BLOB_SIZE); + maxTxSize(parameters::CRYPTONOTE_MAX_TX_SIZE); + publicAddressBase58Prefix(parameters::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX); + minedMoneyUnlockWindow(parameters::CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW); + + timestampCheckWindow(parameters::BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW); + blockFutureTimeLimit(parameters::CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT); + + moneySupply(parameters::MONEY_SUPPLY); + emissionSpeedFactor(parameters::EMISSION_SPEED_FACTOR); + + rewardBlocksWindow(parameters::CRYPTONOTE_REWARD_BLOCKS_WINDOW); + blockGrantedFullRewardZone(parameters::CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE); + minerTxBlobReservedSize(parameters::CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); + + numberOfDecimalPlaces(parameters::CRYPTONOTE_DISPLAY_DECIMAL_POINT); + + mininumFee(parameters::MINIMUM_FEE); + defaultDustThreshold(parameters::DEFAULT_DUST_THRESHOLD); + + difficultyTarget(parameters::DIFFICULTY_TARGET); + difficultyWindow(parameters::DIFFICULTY_WINDOW); + difficultyLag(parameters::DIFFICULTY_LAG); + difficultyCut(parameters::DIFFICULTY_CUT); + + maxBlockSizeInitial(parameters::MAX_BLOCK_SIZE_INITIAL); + maxBlockSizeGrowthSpeedNumerator(parameters::MAX_BLOCK_SIZE_GROWTH_SPEED_NUMERATOR); + maxBlockSizeGrowthSpeedDenominator(parameters::MAX_BLOCK_SIZE_GROWTH_SPEED_DENOMINATOR); + + lockedTxAllowedDeltaSeconds(parameters::CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS); + lockedTxAllowedDeltaBlocks(parameters::CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS); + + mempoolTxLiveTime(parameters::CRYPTONOTE_MEMPOOL_TX_LIVETIME); + mempoolTxFromAltBlockLiveTime(parameters::CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME); + + blocksFileName(parameters::CRYPTONOTE_BLOCKS_FILENAME); + blocksCacheFileName(parameters::CRYPTONOTE_BLOCKSCACHE_FILENAME); + blockIndexesFileName(parameters::CRYPTONOTE_BLOCKINDEXES_FILENAME); + txPoolFileName(parameters::CRYPTONOTE_POOLDATA_FILENAME); + + testnet(false); + } + + Transaction CurrencyBuilder::generateGenesisTransaction() { + cryptonote::Transaction tx; + cryptonote::AccountPublicAddress ac = boost::value_initialized(); + m_currency.constructMinerTx(0, 0, 0, 0, 0, ac, tx); // zero fee in genesis + + return tx; + } + + CurrencyBuilder& CurrencyBuilder::emissionSpeedFactor(unsigned int val) { + if (val <= 0 || val > 8 * sizeof(uint64_t)) { + throw std::invalid_argument("val at emissionSpeedFactor()"); + } + + m_currency.m_emissionSpeedFactor = val; + return *this; + } + + CurrencyBuilder& CurrencyBuilder::numberOfDecimalPlaces(size_t val) { + m_currency.m_numberOfDecimalPlaces = val; + m_currency.m_coin = 1; + for (size_t i = 0; i < m_currency.m_numberOfDecimalPlaces; ++i) { + m_currency.m_coin *= 10; + } + + return *this; + } + + CurrencyBuilder& CurrencyBuilder::difficultyWindow(size_t val) { + if (val < 2) { + throw std::invalid_argument("val at difficultyWindow()"); + } + m_currency.m_difficultyWindow = val; + return *this; + } +} diff --git a/src/cryptonote_core/Currency.h b/src/cryptonote_core/Currency.h new file mode 100644 index 00000000..c12a661d --- /dev/null +++ b/src/cryptonote_core/Currency.h @@ -0,0 +1,206 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include + +#include + +#include "crypto/hash.h" +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/difficulty.h" +#include "cryptonote_config.h" +#include "cryptonote_protocol/blobdatatype.h" + +namespace cryptonote +{ + class Currency { + public: + uint64_t maxBlockHeight() const { return m_maxBlockHeight; } + size_t maxBlockBlobSize() const { return m_maxBlockBlobSize; } + size_t maxTxSize() const { return m_maxTxSize; } + uint64_t publicAddressBase58Prefix() const { return m_publicAddressBase58Prefix; } + size_t minedMoneyUnlockWindow() const { return m_minedMoneyUnlockWindow; } + + size_t timestampCheckWindow() const { return m_timestampCheckWindow; } + uint64_t blockFutureTimeLimit() const { return m_blockFutureTimeLimit; } + + uint64_t moneySupply() const { return m_moneySupply; } + unsigned int emissionSpeedFactor() const { return m_emissionSpeedFactor; } + + size_t rewardBlocksWindow() const { return m_rewardBlocksWindow; } + size_t blockGrantedFullRewardZone() const { return m_blockGrantedFullRewardZone; } + size_t minerTxBlobReservedSize() const { return m_minerTxBlobReservedSize; } + + size_t numberOfDecimalPlaces() const { return m_numberOfDecimalPlaces; } + uint64_t coin() const { return m_coin; } + + uint64_t minimumFee() const { return m_mininumFee; } + uint64_t defaultDustThreshold() const { return m_defaultDustThreshold; } + + uint64_t difficultyTarget() const { return m_difficultyTarget; } + size_t difficultyWindow() const { return m_difficultyWindow; } + size_t difficultyLag() const { return m_difficultyLag; } + size_t difficultyCut() const { return m_difficultyCut; } + size_t difficultyBlocksCount() const { return m_difficultyWindow + m_difficultyLag; } + + size_t maxBlockSizeInitial() const { return m_maxBlockSizeInitial; } + uint64_t maxBlockSizeGrowthSpeedNumerator() const { return m_maxBlockSizeGrowthSpeedNumerator; } + uint64_t maxBlockSizeGrowthSpeedDenominator() const { return m_maxBlockSizeGrowthSpeedDenominator; } + + uint64_t lockedTxAllowedDeltaSeconds() const { return m_lockedTxAllowedDeltaSeconds; } + size_t lockedTxAllowedDeltaBlocks() const { return m_lockedTxAllowedDeltaBlocks; } + + uint64_t mempoolTxLiveTime() const { return m_mempoolTxLiveTime; } + uint64_t mempoolTxFromAltBlockLiveTime() const { return m_mempoolTxFromAltBlockLiveTime; } + + const std::string& blocksFileName() const { return m_blocksFileName; } + const std::string& blocksCacheFileName() const { return m_blocksCacheFileName; } + const std::string& blockIndexesFileName() const { return m_blockIndexesFileName; } + const std::string& txPoolFileName() const { return m_txPoolFileName; } + + bool isTestnet() const { return m_testnet; } + + const Block& genesisBlock() const { return m_genesisBlock; } + const crypto::hash& genesisBlockHash() const { return m_genesisBlockHash; } + + bool getBlockReward(size_t medianSize, size_t currentBlockSize, uint64_t alreadyGeneratedCoins, uint64_t fee, + uint64_t& reward, int64_t& emissionChange) const; + size_t maxBlockCumulativeSize(uint64_t height) const; + + bool constructMinerTx(size_t height, size_t medianSize, uint64_t alreadyGeneratedCoins, size_t currentBlockSize, + uint64_t fee, const AccountPublicAddress& minerAddress, Transaction& tx, + const blobdata& extraNonce = blobdata(), size_t maxOuts = 1) const; + + std::string accountAddressAsString(const account_base& account) const; + bool parseAccountAddressString(const std::string& str, AccountPublicAddress& addr) const; + + std::string formatAmount(uint64_t amount) const; + bool parseAmount(const std::string& str, uint64_t& amount) const; + + difficulty_type nextDifficulty(std::vector timestamps, std::vector cumulativeDifficulties) const; + bool checkProofOfWork(crypto::cn_context& context, const Block& block, difficulty_type currentDiffic, crypto::hash& proofOfWork) const; + + private: + Currency() { + } + + bool init(); + + bool generateGenesisBlock(); + + private: + uint64_t m_maxBlockHeight; + size_t m_maxBlockBlobSize; + size_t m_maxTxSize; + uint64_t m_publicAddressBase58Prefix; + size_t m_minedMoneyUnlockWindow; + + size_t m_timestampCheckWindow; + uint64_t m_blockFutureTimeLimit; + + uint64_t m_moneySupply; + unsigned int m_emissionSpeedFactor; + + size_t m_rewardBlocksWindow; + size_t m_blockGrantedFullRewardZone; + size_t m_minerTxBlobReservedSize; + + size_t m_numberOfDecimalPlaces; + uint64_t m_coin; + + uint64_t m_mininumFee; + uint64_t m_defaultDustThreshold; + + uint64_t m_difficultyTarget; + size_t m_difficultyWindow; + size_t m_difficultyLag; + size_t m_difficultyCut; + + size_t m_maxBlockSizeInitial; + uint64_t m_maxBlockSizeGrowthSpeedNumerator; + uint64_t m_maxBlockSizeGrowthSpeedDenominator; + + uint64_t m_lockedTxAllowedDeltaSeconds; + size_t m_lockedTxAllowedDeltaBlocks; + + uint64_t m_mempoolTxLiveTime; + uint64_t m_mempoolTxFromAltBlockLiveTime; + + std::string m_blocksFileName; + std::string m_blocksCacheFileName; + std::string m_blockIndexesFileName; + std::string m_txPoolFileName; + + bool m_testnet; + + Block m_genesisBlock; + crypto::hash m_genesisBlockHash; + + friend class CurrencyBuilder; + }; + + class CurrencyBuilder : boost::noncopyable { + public: + CurrencyBuilder(); + + Currency currency() { + if (!m_currency.init()) { + throw std::runtime_error("Failed to initialize currency object"); + } + return m_currency; + } + + Transaction generateGenesisTransaction(); + + CurrencyBuilder& maxBlockNumber(uint64_t val) { m_currency.m_maxBlockHeight = val; return *this; } + CurrencyBuilder& maxBlockBlobSize(size_t val) { m_currency.m_maxBlockBlobSize = val; return *this; } + CurrencyBuilder& maxTxSize(size_t val) { m_currency.m_maxTxSize = val; return *this; } + CurrencyBuilder& publicAddressBase58Prefix(uint64_t val) { m_currency.m_publicAddressBase58Prefix = val; return *this; } + CurrencyBuilder& minedMoneyUnlockWindow(size_t val) { m_currency.m_minedMoneyUnlockWindow = val; return *this; } + + CurrencyBuilder& timestampCheckWindow(size_t val) { m_currency.m_timestampCheckWindow = val; return *this; } + CurrencyBuilder& blockFutureTimeLimit(uint64_t val) { m_currency.m_blockFutureTimeLimit = val; return *this; } + + CurrencyBuilder& moneySupply(uint64_t val) { m_currency.m_moneySupply = val; return *this; } + CurrencyBuilder& emissionSpeedFactor(unsigned int val); + + CurrencyBuilder& rewardBlocksWindow(size_t val) { m_currency.m_rewardBlocksWindow = val; return *this; } + CurrencyBuilder& blockGrantedFullRewardZone(size_t val) { m_currency.m_blockGrantedFullRewardZone = val; return *this; } + CurrencyBuilder& minerTxBlobReservedSize(size_t val) { m_currency.m_minerTxBlobReservedSize = val; return *this; } + + CurrencyBuilder& numberOfDecimalPlaces(size_t val); + + CurrencyBuilder& mininumFee(uint64_t val) { m_currency.m_mininumFee = val; return *this; } + CurrencyBuilder& defaultDustThreshold(uint64_t val) { m_currency.m_defaultDustThreshold = val; return *this; } + + CurrencyBuilder& difficultyTarget(uint64_t val) { m_currency.m_difficultyTarget = val; return *this; } + CurrencyBuilder& difficultyWindow(size_t val); + CurrencyBuilder& difficultyLag(size_t val) { m_currency.m_difficultyLag = val; return *this; } + CurrencyBuilder& difficultyCut(size_t val) { m_currency.m_difficultyCut = val; return *this; } + + CurrencyBuilder& maxBlockSizeInitial(size_t val) { m_currency.m_maxBlockSizeInitial = val; return *this; } + CurrencyBuilder& maxBlockSizeGrowthSpeedNumerator(uint64_t val) { m_currency.m_maxBlockSizeGrowthSpeedNumerator = val; return *this; } + CurrencyBuilder& maxBlockSizeGrowthSpeedDenominator(uint64_t val) { m_currency.m_maxBlockSizeGrowthSpeedDenominator = val; return *this; } + + CurrencyBuilder& lockedTxAllowedDeltaSeconds(uint64_t val) { m_currency.m_lockedTxAllowedDeltaSeconds = val; return *this; } + CurrencyBuilder& lockedTxAllowedDeltaBlocks(size_t val) { m_currency.m_lockedTxAllowedDeltaBlocks = val; return *this; } + + CurrencyBuilder& mempoolTxLiveTime(uint64_t val) { m_currency.m_mempoolTxLiveTime = val; return *this; } + CurrencyBuilder& mempoolTxFromAltBlockLiveTime(uint64_t val) { m_currency.m_mempoolTxFromAltBlockLiveTime = val; return *this; } + + CurrencyBuilder& blocksFileName(const std::string& val) { m_currency.m_blocksFileName = val; return *this; } + CurrencyBuilder& blocksCacheFileName(const std::string& val) { m_currency.m_blocksCacheFileName = val; return *this; } + CurrencyBuilder& blockIndexesFileName(const std::string& val) { m_currency.m_blockIndexesFileName = val; return *this; } + CurrencyBuilder& txPoolFileName(const std::string& val) { m_currency.m_txPoolFileName = val; return *this; } + + CurrencyBuilder& testnet(bool val) { m_currency.m_testnet = val; return *this; } + + private: + Currency m_currency; + }; +} diff --git a/src/cryptonote_core/IBlockchainStorageObserver.h b/src/cryptonote_core/IBlockchainStorageObserver.h new file mode 100644 index 00000000..199b6940 --- /dev/null +++ b/src/cryptonote_core/IBlockchainStorageObserver.h @@ -0,0 +1,13 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +namespace cryptonote { + class IBlockchainStorageObserver { + public: + virtual ~IBlockchainStorageObserver() { + } + + virtual void blockchainUpdated() = 0; + }; +} diff --git a/src/cryptonote_core/ICore.h b/src/cryptonote_core/ICore.h new file mode 100644 index 00000000..ea4bf0d5 --- /dev/null +++ b/src/cryptonote_core/ICore.h @@ -0,0 +1,48 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include +#include + +#include "crypto/hash.h" +#include "cryptonote_protocol/blobdatatype.h" + +namespace cryptonote { +struct COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_request; +struct COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_response; +struct NOTIFY_RESPONSE_CHAIN_ENTRY_request; +struct Block; +struct Transaction; +struct i_cryptonote_protocol; +struct tx_verification_context; +struct BlockFullInfo; +class ICoreObserver; + +class ICore { +public: + virtual ~ICore() {} + + virtual bool addObserver(ICoreObserver* observer) = 0; + virtual bool removeObserver(ICoreObserver* observer) = 0; + + virtual bool get_blockchain_top(uint64_t& height, crypto::hash& top_id) = 0; + virtual bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, + uint64_t& total_height, uint64_t& start_height, size_t max_count) = 0; + virtual bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY_request& resp) = 0; + virtual bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_response& res) = 0; + virtual bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) = 0; + virtual i_cryptonote_protocol* get_protocol() = 0; + virtual bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block) = 0; + virtual bool getPoolSymmetricDifference(const std::vector& known_pool_tx_ids, const crypto::hash& known_block_id, bool& isBcActual, std::vector& new_txs, std::vector& deleted_tx_ids) = 0; + virtual bool queryBlocks(const std::list& block_ids, uint64_t timestamp, + uint64_t& start_height, uint64_t& current_height, uint64_t& full_offset, std::list& entries) = 0; + + virtual bool getBlockByHash(const crypto::hash &h, Block &blk) = 0; +}; + +} //namespace cryptonote diff --git a/src/cryptonote_core/ICoreObserver.h b/src/cryptonote_core/ICoreObserver.h new file mode 100644 index 00000000..e9e9c0c2 --- /dev/null +++ b/src/cryptonote_core/ICoreObserver.h @@ -0,0 +1,16 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +namespace cryptonote { + +class ICoreObserver { +public: + virtual ~ICoreObserver() {}; + virtual void blockchainUpdated() {}; + virtual void poolUpdated() {}; +}; + +} //namespace cryptonote diff --git a/src/cryptonote_core/ITimeProvider.cpp b/src/cryptonote_core/ITimeProvider.cpp new file mode 100644 index 00000000..60417f0d --- /dev/null +++ b/src/cryptonote_core/ITimeProvider.cpp @@ -0,0 +1,5 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "ITimeProvider.h" diff --git a/src/cryptonote_core/ITimeProvider.h b/src/cryptonote_core/ITimeProvider.h new file mode 100644 index 00000000..9dd70137 --- /dev/null +++ b/src/cryptonote_core/ITimeProvider.h @@ -0,0 +1,22 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include + +namespace CryptoNote { + + struct ITimeProvider { + virtual time_t now() = 0; + virtual ~ITimeProvider() {} + }; + + struct RealTimeProvider : public ITimeProvider { + virtual time_t now() { + return time(nullptr); + } + }; + +} diff --git a/src/cryptonote_core/ITransactionValidator.h b/src/cryptonote_core/ITransactionValidator.h new file mode 100644 index 00000000..b8a9e186 --- /dev/null +++ b/src/cryptonote_core/ITransactionValidator.h @@ -0,0 +1,38 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include "cryptonote_core/cryptonote_basic.h" + +namespace CryptoNote { + + struct BlockInfo { + uint64_t height; + crypto::hash id; + + BlockInfo() { + clear(); + } + + void clear() { + height = 0; + id = cryptonote::null_hash; + } + + bool empty() const { + return id == cryptonote::null_hash; + } + }; + + class ITransactionValidator { + public: + virtual ~ITransactionValidator() {} + + virtual bool checkTransactionInputs(const cryptonote::Transaction& tx, BlockInfo& maxUsedBlock) = 0; + virtual bool checkTransactionInputs(const cryptonote::Transaction& tx, BlockInfo& maxUsedBlock, BlockInfo& lastFailed) = 0; + virtual bool haveSpentKeyImages(const cryptonote::Transaction& tx) = 0; + }; + +} diff --git a/src/cryptonote_core/ITxPoolObserver.h b/src/cryptonote_core/ITxPoolObserver.h new file mode 100644 index 00000000..14e70787 --- /dev/null +++ b/src/cryptonote_core/ITxPoolObserver.h @@ -0,0 +1,13 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +namespace cryptonote { +class ITxPoolObserver { +public: + virtual ~ITxPoolObserver() { + } + + virtual void txDeletedFromPool() = 0; +}; +} diff --git a/src/cryptonote_core/MinerConfig.cpp b/src/cryptonote_core/MinerConfig.cpp new file mode 100644 index 00000000..d5d3508c --- /dev/null +++ b/src/cryptonote_core/MinerConfig.cpp @@ -0,0 +1,42 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "MinerConfig.h" + +#include "common/command_line.h" + +namespace cryptonote { + +namespace { +const command_line::arg_descriptor arg_extra_messages = {"extra-messages-file", "Specify file for extra messages to include into coinbase transactions", "", true}; +const command_line::arg_descriptor arg_start_mining = {"start-mining", "Specify wallet address to mining for", "", true}; +const command_line::arg_descriptor arg_mining_threads = {"mining-threads", "Specify mining threads count", 0, true}; +} + +MinerConfig::MinerConfig() { + miningThreads = 0; +} + +void MinerConfig::initOptions(boost::program_options::options_description& desc) { + command_line::add_arg(desc, arg_extra_messages); + command_line::add_arg(desc, arg_start_mining); + command_line::add_arg(desc, arg_mining_threads); +} + +void MinerConfig::init(const boost::program_options::variables_map& options) { + if(command_line::has_arg(options, arg_extra_messages)) { + extraMessages = command_line::get_arg(options, arg_extra_messages); + } + + if (command_line::has_arg(options, arg_start_mining)) { + startMining = command_line::get_arg(options, arg_start_mining); + } + + if (command_line::has_arg(options, arg_mining_threads)) { + miningThreads = command_line::get_arg(options, arg_mining_threads); + } +} + +} //namespace cryptonote + diff --git a/src/cryptonote_core/MinerConfig.h b/src/cryptonote_core/MinerConfig.h new file mode 100644 index 00000000..9df3d17d --- /dev/null +++ b/src/cryptonote_core/MinerConfig.h @@ -0,0 +1,26 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include + +#include + +namespace cryptonote { + +class MinerConfig { +public: + MinerConfig(); + + static void initOptions(boost::program_options::options_description& desc); + void init(const boost::program_options::variables_map& options); + + std::string extraMessages; + std::string startMining; + uint32_t miningThreads; +}; + +} //namespace cryptonote diff --git a/src/cryptonote_core/SwappedMap.cpp b/src/cryptonote_core/SwappedMap.cpp old mode 100755 new mode 100644 index d0b7aca1..24d708c6 --- a/src/cryptonote_core/SwappedMap.cpp +++ b/src/cryptonote_core/SwappedMap.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/cryptonote_core/SwappedMap.h b/src/cryptonote_core/SwappedMap.h old mode 100755 new mode 100644 index d1faf607..f4ade005 --- a/src/cryptonote_core/SwappedMap.h +++ b/src/cryptonote_core/SwappedMap.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/cryptonote_core/SwappedVector.cpp b/src/cryptonote_core/SwappedVector.cpp old mode 100755 new mode 100644 index 5c59a202..ce9a03b3 --- a/src/cryptonote_core/SwappedVector.cpp +++ b/src/cryptonote_core/SwappedVector.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/cryptonote_core/SwappedVector.h b/src/cryptonote_core/SwappedVector.h old mode 100755 new mode 100644 index 027fad58..0592d031 --- a/src/cryptonote_core/SwappedVector.h +++ b/src/cryptonote_core/SwappedVector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,12 +12,119 @@ #include #include #include -//#include -//#include #include "serialization/binary_archive.h" template class SwappedVector { public: + typedef T value_type; + + class const_iterator { + public: + typedef ptrdiff_t difference_type; + typedef std::random_access_iterator_tag iterator_category; + typedef const T* pointer; + typedef const T& reference; + typedef T value_type; + + const_iterator() { + } + + const_iterator(SwappedVector* swappedVector, std::size_t index) : m_swappedVector(swappedVector), m_index(index) { + } + + bool operator!=(const const_iterator& other) const { + return m_index != other.m_index; + } + + bool operator<(const const_iterator& other) const { + return m_index < other.m_index; + } + + bool operator<=(const const_iterator& other) const { + return m_index <= other.m_index; + } + + bool operator==(const const_iterator& other) const { + return m_index == other.m_index; + } + + bool operator>(const const_iterator& other) const { + return m_index > other.m_index; + } + + bool operator>=(const const_iterator& other) const { + return m_index >= other.m_index; + } + + const_iterator& operator++() { + ++m_index; + return *this; + } + + const_iterator operator++(int) { + const_iterator i = *this; + ++m_index; + return i; + } + + const_iterator& operator--() { + --m_index; + return *this; + } + + const_iterator operator--(int) { + const_iterator i = *this; + --m_index; + return i; + } + + const_iterator& operator+=(difference_type n) { + m_index += n; + return *this; + } + + const_iterator& operator-=(difference_type n) { + m_index -= n; + return *this; + } + + const_iterator operator+(difference_type n) const { + return const_iterator(m_swappedVector, m_index + n); + } + + friend const_iterator operator+(difference_type n, const const_iterator& i) { + return const_iterator(i.m_swappedVector, n + i.m_index); + } + + difference_type operator-(const const_iterator& other) const { + return m_index - other.m_index; + } + + const_iterator& operator-(difference_type n) const { + return const_iterator(m_swappedVector, m_index - n); + } + + const T& operator*() const { + return (*m_swappedVector)[m_index]; + } + + const T* operator->() const { + return &(*m_swappedVector)[m_index]; + } + + const T& operator[](difference_type offset) const { + return (*m_swappedVector)[m_index + offset]; + } + + std::size_t index() const { + return m_index; + } + + private: + SwappedVector* m_swappedVector; + std::size_t m_index; + }; + SwappedVector(); //SwappedVector(const SwappedVector&) = delete; ~SwappedVector(); @@ -28,6 +135,8 @@ public: bool empty() const; uint64_t size() const; + const_iterator begin(); + const_iterator end(); const T& operator[](uint64_t index); const T& front(); const T& back(); @@ -136,6 +245,14 @@ template uint64_t SwappedVector::size() const { return m_offsets.size(); } +template typename SwappedVector::const_iterator SwappedVector::begin() { + return const_iterator(this, 0); +} + +template typename SwappedVector::const_iterator SwappedVector::end() { + return const_iterator(this, m_offsets.size()); +} + template const T& SwappedVector::operator[](uint64_t index) { auto itemIter = m_items.find(index); if (itemIter != m_items.end()) { @@ -157,13 +274,6 @@ template const T& SwappedVector::operator[](uint64_t index) { m_itemsFile.seekg(m_offsets[index]); T tempItem; - //try { - //boost::archive::binary_iarchive archive(m_itemsFile); - //archive & tempItem; - //} catch (std::exception&) { - // throw std::runtime_error("SwappedVector::operator[]"); - //} - binary_archive archive(m_itemsFile); if (!do_serialize(archive, tempItem)) { throw std::runtime_error("SwappedVector::operator[]"); @@ -231,13 +341,6 @@ template void SwappedVector::push_back(const T& item) { } m_itemsFile.seekp(m_itemsFileSize); - //try { - // boost::archive::binary_oarchive archive(m_itemsFile); - // archive & item; - //} catch (std::exception&) { - // throw std::runtime_error("SwappedVector::push_back"); - //} - binary_archive archive(m_itemsFile); if (!do_serialize(archive, *const_cast(&item))) { throw std::runtime_error("SwappedVector::push_back"); diff --git a/src/cryptonote_core/Transaction.cpp b/src/cryptonote_core/Transaction.cpp new file mode 100644 index 00000000..4b1431de --- /dev/null +++ b/src/cryptonote_core/Transaction.cpp @@ -0,0 +1,595 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "ITransaction.h" +#include "TransactionExtra.h" + +#include "cryptonote_format_utils.h" +#include "account.h" + +#include +#include +#include + +namespace { + + using namespace cryptonote; + using namespace CryptoNote; + + void derivePublicKey(const AccountAddress& to, const crypto::secret_key& txKey, size_t outputIndex, crypto::public_key& ephemeralKey) { + crypto::key_derivation derivation; + crypto::generate_key_derivation(*reinterpret_cast(&to.viewPublicKey), txKey, derivation); + crypto::derive_public_key(derivation, outputIndex, *reinterpret_cast(&to.spendPublicKey), ephemeralKey); + } + + bool checkInputsKeyimagesDiff(const cryptonote::Transaction& tx) { + std::unordered_set ki; + for (const auto& in : tx.vin) { + if (in.type() == typeid(TransactionInputToKey)) { + if (!ki.insert(boost::get(in).keyImage).second) + return false; + } + } + return true; + } + + + // TransactionInput helper functions + + size_t getRequiredSignaturesCount(const TransactionInput& in) { + if (in.type() == typeid(TransactionInputToKey)) { + return boost::get(in).keyOffsets.size(); + } + if (in.type() == typeid(TransactionInputMultisignature)) { + return boost::get(in).signatures; + } + return 0; + } + + uint64_t getTransactionInputAmount(const TransactionInput& in) { + if (in.type() == typeid(TransactionInputToKey)) { + return boost::get(in).amount; + } + if (in.type() == typeid(TransactionInputMultisignature)) { + return boost::get(in).amount; + } + return 0; + } + + TransactionTypes::InputType getTransactionInputType(const TransactionInput& in) { + if (in.type() == typeid(TransactionInputToKey)) { + return TransactionTypes::InputType::Key; + } + if (in.type() == typeid(TransactionInputMultisignature)) { + return TransactionTypes::InputType::Multisignature; + } + if (in.type() == typeid(TransactionInputGenerate)) { + return TransactionTypes::InputType::Generating; + } + return TransactionTypes::InputType::Invalid; + } + + const TransactionInput& getInputChecked(const cryptonote::Transaction& transaction, size_t index) { + if (transaction.vin.size() <= index) { + throw std::runtime_error("Transaction input index out of range"); + } + return transaction.vin[index]; + } + + const TransactionInput& getInputChecked(const cryptonote::Transaction& transaction, size_t index, TransactionTypes::InputType type) { + const auto& input = getInputChecked(transaction, index); + if (getTransactionInputType(input) != type) { + throw std::runtime_error("Unexpected transaction input type"); + } + return input; + } + + // TransactionOutput helper functions + + TransactionTypes::OutputType getTransactionOutputType(const TransactionOutputTarget& out) { + if (out.type() == typeid(TransactionOutputToKey)) { + return TransactionTypes::OutputType::Key; + } + if (out.type() == typeid(TransactionOutputMultisignature)) { + return TransactionTypes::OutputType::Multisignature; + } + return TransactionTypes::OutputType::Invalid; + } + + const TransactionOutput& getOutputChecked(const cryptonote::Transaction& transaction, size_t index) { + if (transaction.vout.size() <= index) { + throw std::runtime_error("Transaction output index out of range"); + } + return transaction.vout[index]; + } + + const TransactionOutput& getOutputChecked(const cryptonote::Transaction& transaction, size_t index, TransactionTypes::OutputType type) { + const auto& output = getOutputChecked(transaction, index); + if (getTransactionOutputType(output.target) != type) { + throw std::runtime_error("Unexpected transaction output target type"); + } + return output; + } +} + + +namespace CryptoNote { + + using namespace TransactionTypes; + + //////////////////////////////////////////////////////////////////////// + // class Transaction declaration + //////////////////////////////////////////////////////////////////////// + + class Transaction : public ITransaction { + public: + Transaction(); + Transaction(const Blob& txblob); + Transaction(const cryptonote::Transaction& tx); + + // ITransactionReader + virtual Hash getTransactionHash() const override; + virtual Hash getTransactionPrefixHash() const override; + virtual PublicKey getTransactionPublicKey() const override; + virtual uint64_t getUnlockTime() const override; + virtual bool getPaymentId(Hash& hash) const override; + virtual bool getExtraNonce(std::string& nonce) const override; + + // inputs + virtual size_t getInputCount() const override; + virtual uint64_t getInputTotalAmount() const override; + virtual TransactionTypes::InputType getInputType(size_t index) const override; + virtual void getInput(size_t index, TransactionTypes::InputKey& input) const override; + virtual void getInput(size_t index, TransactionTypes::InputMultisignature& input) const override; + + // outputs + virtual size_t getOutputCount() const override; + virtual uint64_t getOutputTotalAmount() const override; + virtual TransactionTypes::OutputType getOutputType(size_t index) const override; + virtual void getOutput(size_t index, TransactionTypes::OutputKey& output) const override; + virtual void getOutput(size_t index, TransactionTypes::OutputMultisignature& output) const override; + + virtual size_t getRequiredSignaturesCount(size_t index) const override; + virtual bool findOutputsToAccount(const AccountAddress& addr, const SecretKey& viewSecretKey, std::vector& outs, uint64_t& outputAmount) const override; + + // various checks + virtual bool validateInputs() const override; + virtual bool validateOutputs() const override; + virtual bool validateSignatures() const override; + + // get serialized transaction + virtual Blob getTransactionData() const override; + + // ITransactionWriter + + virtual void setUnlockTime(uint64_t unlockTime) override; + virtual void setPaymentId(const Hash& hash) override; + virtual void setExtraNonce(const std::string& nonce) override; + + // Inputs/Outputs + virtual size_t addInput(const TransactionTypes::InputKey& input) override; + virtual size_t addInput(const AccountKeys& senderKeys, const TransactionTypes::InputKeyInfo& info, KeyPair& ephKeys) override; + virtual size_t addInput(const TransactionTypes::InputMultisignature& input) override; + + virtual size_t addOutput(uint64_t amount, const AccountAddress& to) override; + virtual size_t addOutput(uint64_t amount, const std::vector& to, uint32_t requiredSignatures) override; + + virtual void signInputKey(size_t input, const TransactionTypes::InputKeyInfo& info, const KeyPair& ephKeys) override; + virtual void signInputMultisignature(size_t input, const PublicKey& sourceTransactionKey, size_t outputIndex, const AccountKeys& accountKeys) override; + + // secret key + virtual bool getTransactionSecretKey(SecretKey& key) const override; + virtual void setTransactionSecretKey(const SecretKey& key) override; + + private: + + std::vector& getSignatures(size_t input); + + const crypto::secret_key& txSecretKey() const { + if (!secretKey) { + throw std::runtime_error("Operation requires transaction secret key"); + } + return *secretKey; + } + + cryptonote::Transaction constructFinalTransaction() const { + cryptonote::Transaction tx(transaction); + tx.extra = extra.serialize(); + return tx; + } + + void checkIfSigning() const { + if (!transaction.signatures.empty()) { + throw std::runtime_error("Cannot perform requested operation, since it will invalidate transaction signatures"); + } + } + + cryptonote::Transaction transaction; + boost::optional secretKey; + TransactionExtra extra; + }; + + + //////////////////////////////////////////////////////////////////////// + // class Transaction implementation + //////////////////////////////////////////////////////////////////////// + + std::unique_ptr createTransaction() { + return std::unique_ptr(new Transaction()); + } + + std::unique_ptr createTransaction(const Blob& transactionBlob) { + return std::unique_ptr(new Transaction(transactionBlob)); + } + + std::unique_ptr createTransaction(const cryptonote::Transaction& tx) { + return std::unique_ptr(new Transaction(tx)); + } + + Transaction::Transaction() { + cryptonote::KeyPair txKeys(cryptonote::KeyPair::generate()); + + transaction.version = CURRENT_TRANSACTION_VERSION; + transaction.unlockTime = 0; + + tx_extra_pub_key pk = { txKeys.pub }; + extra.set(pk); + + secretKey = txKeys.sec; + } + + Transaction::Transaction(const Blob& data) { + cryptonote::blobdata blob(reinterpret_cast(data.data()), data.size()); + if (!cryptonote::parse_and_validate_tx_from_blob(blob, transaction)) { + throw std::runtime_error("Invalid transaction data"); + } + + extra.parse(transaction.extra); + } + + Transaction::Transaction(const cryptonote::Transaction& tx) : transaction(tx) { + extra.parse(transaction.extra); + } + + Hash Transaction::getTransactionHash() const { + auto hash = get_transaction_hash(constructFinalTransaction()); + return reinterpret_cast(hash); + } + + Hash Transaction::getTransactionPrefixHash() const { + auto hash = get_transaction_prefix_hash(constructFinalTransaction()); + return reinterpret_cast(hash); + } + + PublicKey Transaction::getTransactionPublicKey() const { + crypto::public_key pk(null_pkey); + extra.getPublicKey(pk); + return reinterpret_cast(pk); + } + + uint64_t Transaction::getUnlockTime() const { + return transaction.unlockTime; + } + + void Transaction::setUnlockTime(uint64_t unlockTime) { + checkIfSigning(); + transaction.unlockTime = unlockTime; + } + + bool Transaction::getTransactionSecretKey(SecretKey& key) const { + if (!secretKey) { + return false; + } + key = reinterpret_cast(secretKey.get()); + return true; + } + + void Transaction::setTransactionSecretKey(const SecretKey& key) { + const auto& sk = reinterpret_cast(key); + crypto::public_key pk; + crypto::public_key txPubKey; + + crypto::secret_key_to_public_key(sk, pk); + extra.getPublicKey(txPubKey); + + if (txPubKey != pk) { + throw std::runtime_error("Secret transaction key does not match public key"); + } + + secretKey = reinterpret_cast(key); + } + + size_t Transaction::addInput(const InputKey& input) { + checkIfSigning(); + TransactionInputToKey inKey = { input.amount, input.keyOffsets, *reinterpret_cast(&input.keyImage) }; + transaction.vin.emplace_back(inKey); + return transaction.vin.size() - 1; + } + + size_t Transaction::addInput(const AccountKeys& senderKeys, const TransactionTypes::InputKeyInfo& info, KeyPair& ephKeys) { + checkIfSigning(); + InputKey input; + input.amount = info.amount; + + generate_key_image_helper( + reinterpret_cast(senderKeys), + reinterpret_cast(info.realOutput.transactionPublicKey), + info.realOutput.outputInTransaction, + reinterpret_cast(ephKeys), + reinterpret_cast(input.keyImage)); + + // fill outputs array and use relative offsets + for (const auto& out : info.outputs) { + input.keyOffsets.push_back(out.outputIndex); + } + input.keyOffsets = absolute_output_offsets_to_relative(input.keyOffsets); + + return addInput(input); + } + + size_t Transaction::addInput(const InputMultisignature& input) { + checkIfSigning(); + TransactionInputMultisignature inMsig; + inMsig.amount = input.amount; + inMsig.outputIndex = input.outputIndex; + inMsig.signatures = input.signatures; + transaction.vin.push_back(inMsig); + return transaction.vin.size() - 1; + } + + size_t Transaction::addOutput(uint64_t amount, const AccountAddress& to) { + checkIfSigning(); + TransactionOutputToKey outKey; + derivePublicKey(to, txSecretKey(), transaction.vout.size(), outKey.key); + TransactionOutput out = { amount, outKey }; + transaction.vout.emplace_back(out); + return transaction.vout.size() - 1; + } + + size_t Transaction::addOutput(uint64_t amount, const std::vector& to, uint32_t requiredSignatures) { + checkIfSigning(); + const auto& txKey = txSecretKey(); + size_t outputIndex = transaction.vout.size(); + TransactionOutputMultisignature outMsig; + outMsig.requiredSignatures = requiredSignatures; + outMsig.keys.resize(to.size()); + for (int i = 0; i < to.size(); ++i) { + derivePublicKey(to[i], txKey, outputIndex, outMsig.keys[i]); + } + TransactionOutput out = { amount, outMsig }; + transaction.vout.emplace_back(out); + return outputIndex; + } + + void Transaction::signInputKey(size_t index, const TransactionTypes::InputKeyInfo& info, const KeyPair& ephKeys) { + const auto& input = boost::get(getInputChecked(transaction, index, InputType::Key)); + Hash prefixHash = getTransactionPrefixHash(); + + std::vector signatures; + std::vector keysPtrs; + + for (const auto& o : info.outputs) { + keysPtrs.push_back(reinterpret_cast(&o.targetKey)); + } + + signatures.resize(keysPtrs.size()); + + generate_ring_signature( + reinterpret_cast(prefixHash), + reinterpret_cast(input.keyImage), + keysPtrs, + reinterpret_cast(ephKeys.secretKey), + info.realOutput.transactionIndex, + signatures.data()); + + getSignatures(index) = signatures; + } + + void Transaction::signInputMultisignature(size_t index, const PublicKey& sourceTransactionKey, size_t outputIndex, const AccountKeys& accountKeys) { + crypto::key_derivation derivation; + crypto::public_key ephemeralPublicKey; + crypto::secret_key ephemeralSecretKey; + + crypto::generate_key_derivation( + reinterpret_cast(sourceTransactionKey), + reinterpret_cast(accountKeys.viewSecretKey), + derivation); + + crypto::derive_public_key(derivation, outputIndex, + reinterpret_cast(accountKeys.address.spendPublicKey), ephemeralPublicKey); + crypto::derive_secret_key(derivation, outputIndex, + reinterpret_cast(accountKeys.spendSecretKey), ephemeralSecretKey); + + crypto::signature signature; + auto txPrefixHash = getTransactionPrefixHash(); + + crypto::generate_signature(reinterpret_cast(txPrefixHash), + ephemeralPublicKey, ephemeralSecretKey, signature); + + getSignatures(index).push_back(signature); + } + + std::vector& Transaction::getSignatures(size_t input) { + // update signatures container size if needed + if (transaction.signatures.size() < transaction.vin.size()) { + transaction.signatures.resize(transaction.vin.size()); + } + // check range + if (input >= transaction.signatures.size()) { + throw std::runtime_error("Invalid input index"); + } + + return transaction.signatures[input]; + } + + std::vector Transaction::getTransactionData() const { + return stringToVector(t_serializable_object_to_blob(constructFinalTransaction())); + } + + void Transaction::setPaymentId(const Hash& hash) { + checkIfSigning(); + blobdata paymentIdBlob; + set_payment_id_to_tx_extra_nonce(paymentIdBlob, reinterpret_cast(hash)); + setExtraNonce(paymentIdBlob); + } + + bool Transaction::getPaymentId(Hash& hash) const { + blobdata nonce; + if (getExtraNonce(nonce)) { + crypto::hash paymentId; + if (get_payment_id_from_tx_extra_nonce(nonce, paymentId)) { + hash = reinterpret_cast(paymentId); + return true; + } + } + return false; + } + + void Transaction::setExtraNonce(const std::string& nonce) { + checkIfSigning(); + tx_extra_nonce extraNonce = { nonce }; + extra.set(extraNonce); + } + + bool Transaction::getExtraNonce(std::string& nonce) const { + tx_extra_nonce extraNonce; + if (extra.get(extraNonce)) { + nonce = extraNonce.nonce; + return true; + } + return false; + } + + size_t Transaction::getInputCount() const { + return transaction.vin.size(); + } + + uint64_t Transaction::getInputTotalAmount() const { + return std::accumulate(transaction.vin.begin(), transaction.vin.end(), 0ULL, [](uint64_t val, const TransactionInput& in) { + return val + getTransactionInputAmount(in); }); + } + + TransactionTypes::InputType Transaction::getInputType(size_t index) const { + return getTransactionInputType(getInputChecked(transaction, index)); + } + + void Transaction::getInput(size_t index, InputKey& input) const { + const auto& k = boost::get(getInputChecked(transaction, index, InputType::Key)); + input.amount = k.amount; + input.keyImage = reinterpret_cast(k.keyImage); + input.keyOffsets = k.keyOffsets; + } + + void Transaction::getInput(size_t index, InputMultisignature& input) const { + const auto& m = boost::get(getInputChecked(transaction, index, InputType::Multisignature)); + input.amount = m.amount; + input.outputIndex = m.outputIndex; + input.signatures = m.signatures; + } + + size_t Transaction::getOutputCount() const { + return transaction.vout.size(); + } + + uint64_t Transaction::getOutputTotalAmount() const { + return std::accumulate(transaction.vout.begin(), transaction.vout.end(), 0ULL, [](uint64_t val, const TransactionOutput& out) { + return val + out.amount; }); + } + + TransactionTypes::OutputType Transaction::getOutputType(size_t index) const { + return getTransactionOutputType(getOutputChecked(transaction, index).target); + } + + void Transaction::getOutput(size_t index, OutputKey& output) const { + const auto& out = getOutputChecked(transaction, index, OutputType::Key); + const auto& k = boost::get(out.target); + output.amount = out.amount; + output.key = reinterpret_cast(k.key); + } + + void Transaction::getOutput(size_t index, OutputMultisignature& output) const { + const auto& out = getOutputChecked(transaction, index, OutputType::Multisignature); + const auto& m = boost::get(out.target); + output.amount = out.amount; + output.keys = reinterpret_cast&>(m.keys); + output.requiredSignatures = m.requiredSignatures; + } + + bool isOutToKey(const crypto::public_key& spendPublicKey, const crypto::public_key& outKey, const crypto::key_derivation& derivation, size_t keyIndex) { + crypto::public_key pk; + derive_public_key(derivation, keyIndex, spendPublicKey, pk); + return pk == outKey; + } + + bool Transaction::findOutputsToAccount(const AccountAddress& addr, const SecretKey& viewSecretKey, std::vector& out, uint64_t& amount) const { + account_keys keys; + keys.m_account_address = reinterpret_cast(addr); + // only view secret key is used, spend key is not needed + keys.m_view_secret_key = reinterpret_cast(viewSecretKey); + + auto pk = getTransactionPublicKey(); + crypto::public_key txPubKey = reinterpret_cast(pk); + + amount = 0; + size_t keyIndex = 0; + uint32_t outputIndex = 0; + + crypto::key_derivation derivation; + generate_key_derivation(txPubKey, keys.m_view_secret_key, derivation); + + for (const TransactionOutput& o : transaction.vout) { + assert(o.target.type() == typeid(TransactionOutputToKey) || o.target.type() == typeid(TransactionOutputMultisignature)); + if (o.target.type() == typeid(TransactionOutputToKey)) { + if (is_out_to_acc(keys, boost::get(o.target), derivation, keyIndex)) { + out.push_back(outputIndex); + amount += o.amount; + } + ++keyIndex; + } else if (o.target.type() == typeid(TransactionOutputMultisignature)) { + const auto& target = boost::get(o.target); + for (const auto& key : target.keys) { + if (isOutToKey(keys.m_account_address.m_spendPublicKey, key, derivation, static_cast(outputIndex))) { + out.push_back(outputIndex); + } + ++keyIndex; + } + } + ++outputIndex; + } + + return true; + } + + size_t Transaction::getRequiredSignaturesCount(size_t index) const { + return ::getRequiredSignaturesCount(getInputChecked(transaction, index)); + } + + bool Transaction::validateInputs() const { + return + check_inputs_types_supported(transaction) && + check_inputs_overflow(transaction) && + checkInputsKeyimagesDiff(transaction) && + checkMultisignatureInputsDiff(transaction); + } + + bool Transaction::validateOutputs() const { + return + check_outs_valid(transaction) && + check_outs_overflow(transaction); + } + + bool Transaction::validateSignatures() const { + if (transaction.signatures.size() < transaction.vin.size()) { + return false; + } + + for (size_t i = 0; i < transaction.vin.size(); ++i) { + if (getRequiredSignaturesCount(i) > transaction.signatures[i].size()) { + return false; + } + } + + return true; + } +} diff --git a/src/cryptonote_core/TransactionApi.h b/src/cryptonote_core/TransactionApi.h new file mode 100644 index 00000000..03414a7c --- /dev/null +++ b/src/cryptonote_core/TransactionApi.h @@ -0,0 +1,18 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include "ITransaction.h" + +namespace cryptonote { + struct Transaction; +} + +namespace CryptoNote { + std::unique_ptr createTransaction(); + std::unique_ptr createTransaction(const Blob& transactionBlob); + std::unique_ptr createTransaction(const cryptonote::Transaction& tx); +} diff --git a/src/cryptonote_core/TransactionExtra.h b/src/cryptonote_core/TransactionExtra.h new file mode 100644 index 00000000..b716a4b6 --- /dev/null +++ b/src/cryptonote_core/TransactionExtra.h @@ -0,0 +1,81 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include "cryptonote_format_utils.h" + +namespace CryptoNote { + + inline std::vector stringToVector(const std::string& s) { + std::vector vec( + reinterpret_cast(s.data()), + reinterpret_cast(s.data()) + s.size()); + return vec; + } + + class TransactionExtra { + public: + TransactionExtra() {} + TransactionExtra(const std::vector& extra) { + parse(extra); + } + + bool parse(const std::vector& extra) { + fields.clear(); + return cryptonote::parse_tx_extra(extra, fields); + } + + template + bool get(T& value) const { + auto it = find(typeid(T)); + if (it == fields.end()) { + return false; + } + value = boost::get(*it); + return true; + } + + template + void set(const T& value) { + auto it = find(typeid(T)); + if (it != fields.end()) { + *it = value; + } else { + fields.push_back(value); + } + } + + bool getPublicKey(crypto::public_key& pk) const { + cryptonote::tx_extra_pub_key extraPk; + if (!get(extraPk)) { + return false; + } + pk = extraPk.pub_key; + return true; + } + + std::vector serialize() const { + std::ostringstream out; + binary_archive ar(out); + for (const auto& f : fields) { + ::do_serialize(ar, const_cast(f)); + } + return stringToVector(out.str()); + } + + private: + + std::vector::const_iterator find(const std::type_info& t) const { + return std::find_if(fields.begin(), fields.end(), [&t](const cryptonote::tx_extra_field& f) { return t == f.type(); }); + } + + std::vector::iterator find(const std::type_info& t) { + return std::find_if(fields.begin(), fields.end(), [&t](const cryptonote::tx_extra_field& f) { return t == f.type(); }); + } + + std::vector fields; + }; + +} \ No newline at end of file diff --git a/src/cryptonote_core/account.cpp b/src/cryptonote_core/account.cpp index aa2efe80..e9f114ea 100644 --- a/src/cryptonote_core/account.cpp +++ b/src/cryptonote_core/account.cpp @@ -1,22 +1,19 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "account.h" + #include #include #include #include "include_base_utils.h" -#include "account.h" #include "warnings.h" -#include "crypto/crypto.h" -#include "cryptonote_core/cryptonote_basic_impl.h" -#include "cryptonote_core/cryptonote_format_utils.h" -using namespace std; DISABLE_VS_WARNINGS(4244 4345) - namespace cryptonote +namespace cryptonote { //----------------------------------------------------------------- account_base::account_base() @@ -31,8 +28,8 @@ DISABLE_VS_WARNINGS(4244 4345) //----------------------------------------------------------------- void account_base::generate() { - generate_keys(m_keys.m_account_address.m_spend_public_key, m_keys.m_spend_secret_key); - generate_keys(m_keys.m_account_address.m_view_public_key, m_keys.m_view_secret_key); + crypto::generate_keys(m_keys.m_account_address.m_spendPublicKey, m_keys.m_spend_secret_key); + crypto::generate_keys(m_keys.m_account_address.m_viewPublicKey, m_keys.m_view_secret_key); m_creation_timestamp = time(NULL); } //----------------------------------------------------------------- @@ -40,11 +37,9 @@ DISABLE_VS_WARNINGS(4244 4345) { return m_keys; } - //----------------------------------------------------------------- - std::string account_base::get_public_address_str() - { - //TODO: change this code into base 58 - return get_account_address_as_str(m_keys.m_account_address); + + void account_base::set_keys(const account_keys& keys) { + m_keys = keys; } //----------------------------------------------------------------- } diff --git a/src/cryptonote_core/account.h b/src/cryptonote_core/account.h index 1e4dcddd..c38b2055 100644 --- a/src/cryptonote_core/account.h +++ b/src/cryptonote_core/account.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,34 +6,26 @@ #include "cryptonote_core/cryptonote_basic.h" #include "crypto/crypto.h" -#include "serialization/keyvalue_serialization.h" -namespace cryptonote -{ +namespace cryptonote { + template struct AccountBaseSerializer; - struct account_keys - { - account_public_address m_account_address; + struct account_keys { + AccountPublicAddress m_account_address; crypto::secret_key m_spend_secret_key; crypto::secret_key m_view_secret_key; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(m_account_address) - KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_spend_secret_key) - KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_view_secret_key) - END_KV_SERIALIZE_MAP() }; /************************************************************************/ /* */ /************************************************************************/ - class account_base - { + class account_base { public: account_base(); void generate(); + const account_keys& get_keys() const; - std::string get_public_address_str(); + void set_keys(const account_keys& keys); uint64_t get_createtime() const { return m_creation_timestamp; } void set_createtime(uint64_t val) { m_creation_timestamp = val; } @@ -42,20 +34,17 @@ namespace cryptonote bool store(const std::string& file_path); template - inline void serialize(t_archive &a, const unsigned int /*ver*/) - { + inline void serialize(t_archive &a, const unsigned int /*ver*/) { a & m_keys; a & m_creation_timestamp; } - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(m_keys) - KV_SERIALIZE(m_creation_timestamp) - END_KV_SERIALIZE_MAP() - private: void set_null(); account_keys m_keys; uint64_t m_creation_timestamp; + + friend struct AccountBaseSerializer; + friend struct AccountBaseSerializer; }; } diff --git a/src/cryptonote_core/account_boost_serialization.h b/src/cryptonote_core/account_boost_serialization.h index 261cea51..43c27017 100644 --- a/src/cryptonote_core/account_boost_serialization.h +++ b/src/cryptonote_core/account_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,10 +21,10 @@ namespace boost } template - inline void serialize(Archive &a, cryptonote::account_public_address &x, const boost::serialization::version_type ver) + inline void serialize(Archive &a, cryptonote::AccountPublicAddress &x, const boost::serialization::version_type ver) { - a & x.m_spend_public_key; - a & x.m_view_public_key; + a & x.m_spendPublicKey; + a & x.m_viewPublicKey; } } diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index 87d53c02..9570b9a0 100755 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,15 +7,21 @@ #include #include + #include #include +// epee +#include "file_io_utils.h" +#include "misc_language.h" +#include "profile_tools.h" +#include "time_helper.h" + +#include "common/boost_serialization_helper.h" +#include "common/ShuffleGenerator.h" #include "cryptonote_format_utils.h" #include "cryptonote_boost_serialization.h" - -#include "profile_tools.h" -#include "file_io_utils.h" -#include "common/boost_serialization_helper.h" +#include "rpc/core_rpc_server_commands_defs.h" namespace { std::string appendPath(const std::string& path, const std::string& fileName) { @@ -45,14 +51,14 @@ DISABLE_VS_WARNINGS(4267) namespace cryptonote { struct transaction_chain_entry { - transaction tx; + Transaction tx; uint64_t m_keeper_block_height; size_t m_blob_size; std::vector m_global_output_indexes; }; struct block_extended_info { - block bl; + Block bl; uint64_t height; size_t block_cumulative_size; difficulty_type cumulative_difficulty; @@ -65,6 +71,144 @@ template void cryptonote::blockchain_storage::TransactionIndex::s archive & transaction; } +template void cryptonote::blockchain_storage::MultisignatureOutputUsage::serialize(Archive& archive, unsigned int version) { + archive & transactionIndex; + archive & outputIndex; + archive & isUsed; +} + +namespace cryptonote +{ + +#define CURRENT_BLOCKCACHE_STORAGE_ARCHIVE_VER 1 + + class BlockCacheSerializer { + + public: + BlockCacheSerializer(blockchain_storage& bs, const crypto::hash lastBlockHash) : + m_bs(bs), m_lastBlockHash(lastBlockHash), m_loaded(false) {} + + template void serialize(Archive& ar, unsigned int version) { + + // ignore old versions, do rebuild + if (version < CURRENT_BLOCKCACHE_STORAGE_ARCHIVE_VER) + return; + + std::string operation; + if (Archive::is_loading::value) { + operation = "- loading "; + crypto::hash blockHash; + ar & blockHash; + + if (blockHash != m_lastBlockHash) { + return; + } + + } else { + operation = "- saving "; + ar & m_lastBlockHash; + } + + LOG_PRINT_L0(operation << "block index..."); + ar & m_bs.m_blockIndex; + + LOG_PRINT_L0(operation << "transaction map..."); + ar & m_bs.m_transactionMap; + + LOG_PRINT_L0(operation << "spend keys..."); + ar & m_bs.m_spent_keys; + + LOG_PRINT_L0(operation << "outputs..."); + ar & m_bs.m_outputs; + + LOG_PRINT_L0(operation << "multi-signature outputs..."); + ar & m_bs.m_multisignatureOutputs; + + m_loaded = true; + } + + bool loaded() const { + return m_loaded; + } + + private: + + bool m_loaded; + blockchain_storage& m_bs; + crypto::hash m_lastBlockHash; + }; +} + +BOOST_CLASS_VERSION(cryptonote::BlockCacheSerializer, CURRENT_BLOCKCACHE_STORAGE_ARCHIVE_VER) + + +blockchain_storage::blockchain_storage(const Currency& currency, tx_memory_pool& tx_pool): + m_currency(currency), + m_tx_pool(tx_pool), + m_current_block_cumul_sz_limit(0), + m_is_in_checkpoint_zone(false), + m_is_blockchain_storing(false) { + m_outputs.set_deleted_key(0); + + crypto::key_image nullImage = AUTO_VAL_INIT(nullImage); + m_spent_keys.set_deleted_key(nullImage); +} + +bool blockchain_storage::addObserver(IBlockchainStorageObserver* observer) { + return m_observerManager.add(observer); +} + +bool blockchain_storage::removeObserver(IBlockchainStorageObserver* observer) { + return m_observerManager.remove(observer); +} + +bool blockchain_storage::checkTransactionInputs(const cryptonote::Transaction& tx, BlockInfo& maxUsedBlock) { + return check_tx_inputs(tx, maxUsedBlock.height, maxUsedBlock.id); +} + +bool blockchain_storage::checkTransactionInputs(const cryptonote::Transaction& tx, BlockInfo& maxUsedBlock, BlockInfo& lastFailed) { + + BlockInfo tail; + + //not the best implementation at this time, sorry :( + //check is ring_signature already checked ? + if (maxUsedBlock.empty()) { + //not checked, lets try to check + if (!lastFailed.empty() && get_current_blockchain_height() > lastFailed.height && get_block_id_by_height(lastFailed.height) == lastFailed.id) { + return false; //we already sure that this tx is broken for this height + } + + if (!check_tx_inputs(tx, maxUsedBlock.height, maxUsedBlock.id, &tail)) { + lastFailed = tail; + return false; + } + } + else { + if (maxUsedBlock.height >= get_current_blockchain_height()) { + return false; + } + + if (get_block_id_by_height(maxUsedBlock.height) != maxUsedBlock.id) { + //if we already failed on this height and id, skip actual ring signature check + if (lastFailed.id == get_block_id_by_height(lastFailed.height)) { + return false; + } + + //check ring signature again, it is possible (with very small chance) that this transaction become again valid + if (!check_tx_inputs(tx, maxUsedBlock.height, maxUsedBlock.id, &tail)) { + lastFailed = tail; + return false; + } + } + } + + return true; +} + +bool blockchain_storage::haveSpentKeyImages(const cryptonote::Transaction& tx) { + return this->have_tx_keyimges_as_spent(tx); +} + bool blockchain_storage::have_tx(const crypto::hash &id) { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_transactionMap.find(id) != m_transactionMap.end(); @@ -80,7 +224,7 @@ uint64_t blockchain_storage::get_current_blockchain_height() { return m_blocks.size(); } -bool blockchain_storage::init(const std::string& config_folder, bool load_existing, bool testnet) { +bool blockchain_storage::init(const std::string& config_folder, bool load_existing) { CRITICAL_REGION_LOCAL(m_blockchain_lock); if (!config_folder.empty() && !tools::create_directories_if_necessary(config_folder)) { LOG_ERROR("Failed to create data directory: " << m_config_folder); @@ -89,7 +233,7 @@ bool blockchain_storage::init(const std::string& config_folder, bool load_existi m_config_folder = config_folder; - if (!m_blocks.open(appendPath(config_folder, CRYPTONOTE_BLOCKS_FILENAME), appendPath(config_folder, CRYPTONOTE_BLOCKINDEXES_FILENAME), 1024)) { + if (!m_blocks.open(appendPath(config_folder, m_currency.blocksFileName()), appendPath(config_folder, m_currency.blockIndexesFileName()), 1024)) { return false; } @@ -99,42 +243,49 @@ bool blockchain_storage::init(const std::string& config_folder, bool load_existi if (m_blocks.empty()) { LOG_PRINT_L0("Can't load blockchain storage from file."); } else { - bool rebuild = true; - try { - std::ifstream file(appendPath(config_folder, CRYPTONOTE_BLOCKSCACHE_FILENAME), std::ios::binary); - boost::archive::binary_iarchive archive(file); - crypto::hash lastBlockHash; - archive & lastBlockHash; - if (lastBlockHash == get_block_hash(m_blocks.back().bl)) { - archive & m_blockMap; - archive & m_transactionMap; - archive & m_spent_keys; - archive & m_outputs; - rebuild = false; - } - } catch (std::exception&) { - } + BlockCacheSerializer loader(*this, get_block_hash(m_blocks.back().bl)); + tools::unserialize_obj_from_file(loader, appendPath(config_folder, m_currency.blocksCacheFileName())); - if (rebuild) { + if (!loader.loaded()) { LOG_PRINT_L0("No actual blockchain cache found, rebuilding internal structures..."); std::chrono::steady_clock::time_point timePoint = std::chrono::steady_clock::now(); + m_blockIndex.clear(); + m_transactionMap.clear(); + m_spent_keys.clear(); + m_outputs.clear(); + m_multisignatureOutputs.clear(); for (uint32_t b = 0; b < m_blocks.size(); ++b) { - const Block& block = m_blocks[b]; + if (b % 1000 == 0) { + std::cout << "Height " << b << " of " << m_blocks.size() << '\r'; + } + const BlockEntry& block = m_blocks[b]; crypto::hash blockHash = get_block_hash(block.bl); - m_blockMap.insert(std::make_pair(blockHash, b)); + m_blockIndex.push(blockHash); for (uint16_t t = 0; t < block.transactions.size(); ++t) { - const Transaction& transaction = block.transactions[t]; + const TransactionEntry& transaction = block.transactions[t]; crypto::hash transactionHash = get_transaction_hash(transaction.tx); TransactionIndex transactionIndex = { b, t }; m_transactionMap.insert(std::make_pair(transactionHash, transactionIndex)); + + // process inputs for (auto& i : transaction.tx.vin) { - if (i.type() == typeid(txin_to_key)) { - m_spent_keys.insert(::boost::get(i).k_image); + if (i.type() == typeid(TransactionInputToKey)) { + m_spent_keys.insert(::boost::get(i).keyImage); + } else if (i.type() == typeid(TransactionInputMultisignature)) { + auto out = ::boost::get(i); + m_multisignatureOutputs[out.amount][out.outputIndex].isUsed = true; } } + // process outputs for (uint16_t o = 0; o < transaction.tx.vout.size(); ++o) { - m_outputs[transaction.tx.vout[o].amount].push_back(std::make_pair<>(transactionIndex, o)); + const auto& out = transaction.tx.vout[o]; + if(out.target.type() == typeid(TransactionOutputToKey)) { + m_outputs[out.amount].push_back(std::make_pair<>(transactionIndex, o)); + } else if (out.target.type() == typeid(TransactionOutputMultisignature)) { + MultisignatureOutputUsage usage = { transactionIndex, o, false }; + m_multisignatureOutputs[out.amount].push_back(usage); + } } } } @@ -149,74 +300,48 @@ bool blockchain_storage::init(const std::string& config_folder, bool load_existi if (m_blocks.empty()) { LOG_PRINT_L0("Blockchain not loaded, generating genesis block."); - - if (!storeGenesisBlock(testnet)) { - return false; - } + block_verification_context bvc = boost::value_initialized(); + add_new_block(m_currency.genesisBlock(), bvc); + CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); } else { - cryptonote::block b; - if (testnet) { - generateTestnetGenesisBlock(b); - } else { - generateGenesisBlock(b); - } - - crypto::hash genesis_hash = get_block_hash(m_blocks[0].bl); - crypto::hash testnet_genesis_hash = get_block_hash(b); - if (genesis_hash != testnet_genesis_hash) { - LOG_ERROR("Failed to init: genesis block mismatch. Probably you set --testnet flag with data dir with non-test blockchain or another network."); - return false; - } + crypto::hash firstBlockHash = get_block_hash(m_blocks[0].bl); + CHECK_AND_ASSERT_MES(firstBlockHash == m_currency.genesisBlockHash(), false, + "Failed to init: genesis block mismatch. Probably you set --testnet flag with data dir with non-test blockchain or another network."); } + update_next_comulative_size_limit(); + uint64_t timestamp_diff = time(NULL) - m_blocks.back().bl.timestamp; - if (!m_blocks.back().bl.timestamp) + if (!m_blocks.back().bl.timestamp) { timestamp_diff = time(NULL) - 1341378000; + } + LOG_PRINT_GREEN("Blockchain initialized. last block: " << m_blocks.size() - 1 << ", " << epee::misc_utils::get_time_interval_string(timestamp_diff) << " time ago, current difficulty: " << get_difficulty_for_next_block(), LOG_LEVEL_0); return true; } -bool blockchain_storage::storeGenesisBlock(bool testnet) { - block bl = ::boost::value_initialized(); - block_verification_context bvc = boost::value_initialized(); +bool blockchain_storage::storeCache() { + CRITICAL_REGION_LOCAL(m_blockchain_lock); - if (testnet) { - generateTestnetGenesisBlock(bl); - } else { - generateGenesisBlock(bl); - } - - add_new_block(bl, bvc); - CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); - return true; -} - -bool blockchain_storage::store_blockchain() { - try { - std::ofstream file(appendPath(m_config_folder, CRYPTONOTE_BLOCKSCACHE_FILENAME), std::ios::binary); - boost::archive::binary_oarchive archive(file); - crypto::hash lastBlockHash = get_block_hash(m_blocks.back().bl); - archive & lastBlockHash; - archive & m_blockMap; - archive & m_transactionMap; - archive & m_spent_keys; - archive & m_outputs; - LOG_PRINT_L0("Saved blockchain cache."); - } catch (std::exception& e) { - LOG_ERROR("Failed to save blockchain cache, " << e.what()); + LOG_PRINT_L0("Saving blockchain..."); + BlockCacheSerializer ser(*this, get_tail_id()); + if (!tools::serialize_obj_to_file(ser, appendPath(m_config_folder, m_currency.blocksCacheFileName()))) { + LOG_ERROR("Failed to save blockchain cache"); + return false; } return true; } bool blockchain_storage::deinit() { - return store_blockchain(); + storeCache(); + return true; } -bool blockchain_storage::reset_and_set_genesis_block(const block& b) { +bool blockchain_storage::reset_and_set_genesis_block(const Block& b) { CRITICAL_REGION_LOCAL(m_blockchain_lock); m_blocks.clear(); - m_blockMap.clear(); + m_blockIndex.clear(); m_transactionMap.clear(); m_spent_keys.clear(); @@ -236,56 +361,42 @@ crypto::hash blockchain_storage::get_tail_id(uint64_t& height) { crypto::hash blockchain_storage::get_tail_id() { CRITICAL_REGION_LOCAL(m_blockchain_lock); - crypto::hash id = null_hash; - if (m_blocks.size()) { - get_block_hash(m_blocks.back().bl, id); + return m_blockIndex.getTailId(); +} + +bool blockchain_storage::getPoolSymmetricDifference(const std::vector& known_pool_tx_ids, const crypto::hash& known_block_id, std::vector& new_txs, std::vector& deleted_tx_ids) { + CRITICAL_REGION_LOCAL1(m_tx_pool); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if (known_block_id != get_tail_id()) { + return false; } - return id; + std::vector new_tx_ids; + m_tx_pool.get_difference(known_pool_tx_ids, new_tx_ids, deleted_tx_ids); + + std::vector misses; + get_transactions(new_tx_ids, new_txs, misses, true); + assert(misses.empty()); + return true; } bool blockchain_storage::get_short_chain_history(std::list& ids) { CRITICAL_REGION_LOCAL(m_blockchain_lock); - size_t i = 0; - size_t current_multiplier = 1; - size_t sz = m_blocks.size(); - if (!sz) - return true; - size_t current_back_offset = 1; - bool genesis_included = false; - while (current_back_offset < sz) - { - ids.push_back(get_block_hash(m_blocks[sz - current_back_offset].bl)); - if (sz - current_back_offset == 0) - genesis_included = true; - if (i < 10) - { - ++current_back_offset; - } else - { - current_back_offset += current_multiplier *= 2; - } - ++i; - } - if (!genesis_included) - ids.push_back(get_block_hash(m_blocks[0].bl)); - - return true; + return m_blockIndex.getShortChainHistory(ids); } crypto::hash blockchain_storage::get_block_id_by_height(uint64_t height) { CRITICAL_REGION_LOCAL(m_blockchain_lock); - if (height >= m_blocks.size()) - return null_hash; - - return get_block_hash(m_blocks[height].bl); + return m_blockIndex.getBlockId(height); } -bool blockchain_storage::get_block_by_hash(const crypto::hash& blockHash, block& b) { +bool blockchain_storage::get_block_by_hash(const crypto::hash& blockHash, Block& b) { CRITICAL_REGION_LOCAL(m_blockchain_lock); - auto blockIndexByHashIterator = m_blockMap.find(blockHash); - if (blockIndexByHashIterator != m_blockMap.end()) { - b = m_blocks[blockIndexByHashIterator->second].bl; + + uint64_t height = 0; + + if (m_blockIndex.getBlockHeight(blockHash, height)) { + b = m_blocks[height].bl; return true; } @@ -302,7 +413,7 @@ difficulty_type blockchain_storage::get_difficulty_for_next_block() { CRITICAL_REGION_LOCAL(m_blockchain_lock); std::vector timestamps; std::vector commulative_difficulties; - size_t offset = m_blocks.size() - std::min(m_blocks.size(), static_cast(DIFFICULTY_BLOCKS_COUNT)); + size_t offset = m_blocks.size() - std::min(m_blocks.size(), static_cast(m_currency.difficultyBlocksCount())); if (offset == 0) { ++offset; } @@ -312,10 +423,19 @@ difficulty_type blockchain_storage::get_difficulty_for_next_block() { commulative_difficulties.push_back(m_blocks[offset].cumulative_difficulty); } - return next_difficulty(timestamps, commulative_difficulties); + return m_currency.nextDifficulty(timestamps, commulative_difficulties); } -bool blockchain_storage::rollback_blockchain_switching(std::list& original_chain, size_t rollback_height) { +uint64_t blockchain_storage::getCoinsInCirculation() { + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if (m_blocks.empty()) { + return 0; + } else { + return m_blocks.back().already_generated_coins; + } +} + +bool blockchain_storage::rollback_blockchain_switching(std::list& original_chain, size_t rollback_height) { CRITICAL_REGION_LOCAL(m_blockchain_lock); //remove failed subchain for (size_t i = m_blocks.size() - 1; i >= rollback_height; i--) @@ -344,9 +464,9 @@ bool blockchain_storage::switch_to_alternative_blockchain(std::list split_height, false, "switch_to_alternative_blockchain: blockchain size is lower than split height"); //disconnecting old chain - std::list disconnected_chain; + std::list disconnected_chain; for (size_t i = m_blocks.size() - 1; i >= split_height; i--) { - block b = m_blocks[i].bl; + Block b = m_blocks[i].bl; popBlock(get_block_hash(b)); //CHECK_AND_ASSERT_MES(r, false, "failed to remove block on chain switching"); disconnected_chain.push_front(b); @@ -396,13 +516,13 @@ bool blockchain_storage::switch_to_alternative_blockchain(std::list& alt_chain, Block& bei) { +difficulty_type blockchain_storage::get_next_difficulty_for_alternative_chain(const std::list& alt_chain, BlockEntry& bei) { std::vector timestamps; std::vector commulative_difficulties; - if (alt_chain.size() < DIFFICULTY_BLOCKS_COUNT) { + if (alt_chain.size() < m_currency.difficultyBlocksCount()) { CRITICAL_REGION_LOCAL(m_blockchain_lock); size_t main_chain_stop_offset = alt_chain.size() ? alt_chain.front()->second.height : bei.height; - size_t main_chain_count = DIFFICULTY_BLOCKS_COUNT - std::min(static_cast(DIFFICULTY_BLOCKS_COUNT), alt_chain.size()); + size_t main_chain_count = m_currency.difficultyBlocksCount() - std::min(m_currency.difficultyBlocksCount(), alt_chain.size()); main_chain_count = std::min(main_chain_count, main_chain_stop_offset); size_t main_chain_start_offset = main_chain_stop_offset - main_chain_count; @@ -413,43 +533,46 @@ difficulty_type blockchain_storage::get_next_difficulty_for_alternative_chain(co commulative_difficulties.push_back(m_blocks[main_chain_start_offset].cumulative_difficulty); } - CHECK_AND_ASSERT_MES((alt_chain.size() + timestamps.size()) <= DIFFICULTY_BLOCKS_COUNT, false, "Internal error, alt_chain.size()[" << alt_chain.size() - << "] + vtimestampsec.size()[" << timestamps.size() << "] NOT <= DIFFICULTY_WINDOW[]" << DIFFICULTY_BLOCKS_COUNT); + CHECK_AND_ASSERT_MES((alt_chain.size() + timestamps.size()) <= m_currency.difficultyBlocksCount(), false, + "Internal error, alt_chain.size()[" << alt_chain.size() << "] + timestamps.size()[" << timestamps.size() << + "] NOT <= m_currency.difficultyBlocksCount()[" << m_currency.difficultyBlocksCount() << ']'); for (auto it : alt_chain) { timestamps.push_back(it->second.bl.timestamp); commulative_difficulties.push_back(it->second.cumulative_difficulty); } } else { - timestamps.resize(std::min(alt_chain.size(), static_cast(DIFFICULTY_BLOCKS_COUNT))); - commulative_difficulties.resize(std::min(alt_chain.size(), static_cast(DIFFICULTY_BLOCKS_COUNT))); + timestamps.resize(std::min(alt_chain.size(), m_currency.difficultyBlocksCount())); + commulative_difficulties.resize(std::min(alt_chain.size(), m_currency.difficultyBlocksCount())); size_t count = 0; size_t max_i = timestamps.size() - 1; BOOST_REVERSE_FOREACH(auto it, alt_chain) { timestamps[max_i - count] = it->second.bl.timestamp; commulative_difficulties[max_i - count] = it->second.cumulative_difficulty; count++; - if (count >= DIFFICULTY_BLOCKS_COUNT) { + if (count >= m_currency.difficultyBlocksCount()) { break; } } } - return next_difficulty(timestamps, commulative_difficulties); + return m_currency.nextDifficulty(timestamps, commulative_difficulties); } -bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t height) { - CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs"); - CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type"); - if (boost::get(b.miner_tx.vin[0]).height != height) { - LOG_PRINT_RED_L0("The miner transaction in block has invalid height: " << boost::get(b.miner_tx.vin[0]).height << ", expected: " << height); +bool blockchain_storage::prevalidate_miner_transaction(const Block& b, uint64_t height) { + CHECK_AND_ASSERT_MES(b.minerTx.vin.size() == 1, false, "coinbase transaction in the block has no inputs"); + CHECK_AND_ASSERT_MES(b.minerTx.vin[0].type() == typeid(TransactionInputGenerate), false, + "coinbase transaction in the block has the wrong type"); + if (boost::get(b.minerTx.vin[0]).height != height) { + LOG_PRINT_RED_L0("The miner transaction in block has invalid height: " << + boost::get(b.minerTx.vin[0]).height << ", expected: " << height); return false; } - CHECK_AND_ASSERT_MES(b.miner_tx.unlock_time == height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, + CHECK_AND_ASSERT_MES(b.minerTx.unlockTime == height + m_currency.minedMoneyUnlockWindow(), false, - "coinbase transaction transaction have wrong unlock time=" << b.miner_tx.unlock_time << ", expected " << height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW); + "coinbase transaction transaction have wrong unlock time=" << b.minerTx.unlockTime << ", expected " << height + m_currency.minedMoneyUnlockWindow()); - if (!check_outs_overflow(b.miner_tx)) { + if (!check_outs_overflow(b.minerTx)) { LOG_PRINT_RED_L0("miner transaction have money overflow in block " << get_block_hash(b)); return false; } @@ -457,27 +580,30 @@ bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t return true; } -bool blockchain_storage::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) { - uint64_t money_in_use = 0; - for (auto& o : b.miner_tx.vout) { - money_in_use += o.amount; +bool blockchain_storage::validate_miner_transaction(const Block& b, uint64_t height, size_t cumulativeBlockSize, + uint64_t alreadyGeneratedCoins, uint64_t fee, + uint64_t& reward, int64_t& emissionChange) { + uint64_t minerReward = 0; + for (auto& o : b.minerTx.vout) { + minerReward += o.amount; } - std::vector last_blocks_sizes; - get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward)) { - LOG_PRINT_L0("block size " << cumulative_block_size << " is bigger than allowed for this blockchain"); + std::vector lastBlocksSizes; + get_last_n_blocks_sizes(lastBlocksSizes, m_currency.rewardBlocksWindow()); + size_t blocksSizeMedian = epee::misc_utils::median(lastBlocksSizes); + + if (!m_currency.getBlockReward(blocksSizeMedian, cumulativeBlockSize, alreadyGeneratedCoins, fee, reward, emissionChange)) { + LOG_PRINT_L0("block size " << cumulativeBlockSize << " is bigger than allowed for this blockchain"); return false; } - if (base_reward + fee < money_in_use) { - LOG_ERROR("coinbase transaction spend too much money (" << print_money(money_in_use) << "). Block reward is " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << ")"); + if (minerReward > reward) { + LOG_ERROR("Coinbase transaction spend too much money: " << m_currency.formatAmount(minerReward) << + ", block reward is " << m_currency.formatAmount(reward)); return false; - } - - if (base_reward + fee != money_in_use) { - LOG_ERROR("coinbase transaction doesn't use full amount of block reward: spent: " - << print_money(money_in_use) << ", block reward " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << ")"); + } else if (minerReward < reward) { + LOG_ERROR("Coinbase transaction doesn't use full amount of block reward: spent " << + m_currency.formatAmount(minerReward) << ", block reward is " << m_currency.formatAmount(reward)); return false; } @@ -508,18 +634,21 @@ uint64_t blockchain_storage::get_current_comulative_blocksize_limit() { return m_current_block_cumul_sz_limit; } -bool blockchain_storage::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) { +bool blockchain_storage::create_block_template(Block& b, const AccountPublicAddress& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) { size_t median_size; uint64_t already_generated_coins; CRITICAL_REGION_BEGIN(m_blockchain_lock); - b.major_version = CURRENT_BLOCK_MAJOR_VERSION; - b.minor_version = CURRENT_BLOCK_MINOR_VERSION; - b.prev_id = get_tail_id(); - b.timestamp = time(NULL); height = m_blocks.size(); diffic = get_difficulty_for_next_block(); - CHECK_AND_ASSERT_MES(diffic, false, "difficulty owverhead."); + CHECK_AND_ASSERT_MES(diffic, false, "difficulty overhead."); + + b = boost::value_initialized(); + b.majorVersion = BLOCK_MAJOR_VERSION_1; + b.minorVersion = BLOCK_MINOR_VERSION_0; + + b.prevId = get_tail_id(); + b.timestamp = time(NULL); median_size = m_current_block_cumul_sz_limit / 2; already_generated_coins = m_blocks.back().already_generated_coins; @@ -528,7 +657,8 @@ bool blockchain_storage::create_block_template(block& b, const account_public_ad size_t txs_size; uint64_t fee; - if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee)) { + if (!m_tx_pool.fill_block_template(b, median_size, m_currency.maxBlockCumulativeSize(height), already_generated_coins, + txs_size, fee)) { return false; } @@ -536,7 +666,7 @@ bool blockchain_storage::create_block_template(block& b, const account_public_ad size_t real_txs_size = 0; uint64_t real_fee = 0; CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); - BOOST_FOREACH(crypto::hash &cur_hash, b.tx_hashes) { + for (crypto::hash &cur_hash : b.txHashes) { auto cur_res = m_tx_pool.m_transactions.find(cur_hash); if (cur_res == m_tx_pool.m_transactions.end()) { LOG_ERROR("Creating block template: error: transaction not found"); @@ -574,18 +704,18 @@ bool blockchain_storage::create_block_template(block& b, const account_public_ad block size, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block size */ //make blocks coin-base tx looks close to real coinbase tx to get truthful blob size - bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, 11); + bool r = m_currency.constructMinerTx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.minerTx, ex_nonce, 11); CHECK_AND_ASSERT_MES(r, false, "Failed to construc miner tx, first chance"); - size_t cumulative_size = txs_size + get_object_blobsize(b.miner_tx); + size_t cumulative_size = txs_size + get_object_blobsize(b.minerTx); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - LOG_PRINT_L1("Creating block template: miner tx size " << get_object_blobsize(b.miner_tx) << + LOG_PRINT_L1("Creating block template: miner tx size " << get_object_blobsize(b.minerTx) << ", cumulative size " << cumulative_size); #endif for (size_t try_count = 0; try_count != 10; ++try_count) { - r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, 11); + r = m_currency.constructMinerTx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.minerTx, ex_nonce, 11); CHECK_AND_ASSERT_MES(r, false, "Failed to construc miner tx, second chance"); - size_t coinbase_blob_size = get_object_blobsize(b.miner_tx); + size_t coinbase_blob_size = get_object_blobsize(b.minerTx); if (coinbase_blob_size > cumulative_size - txs_size) { cumulative_size = txs_size + coinbase_blob_size; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) @@ -602,21 +732,21 @@ bool blockchain_storage::create_block_template(block& b, const account_public_ad ", cumulative size " << txs_size + coinbase_blob_size << " is less then before, adding " << delta << " zero bytes"); #endif - b.miner_tx.extra.insert(b.miner_tx.extra.end(), delta, 0); + b.minerTx.extra.insert(b.minerTx.extra.end(), delta, 0); //here could be 1 byte difference, because of extra field counter is varint, and it can become from 1-byte len to 2-bytes len. - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) { - CHECK_AND_ASSERT_MES(cumulative_size + 1 == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " + 1 is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); - b.miner_tx.extra.resize(b.miner_tx.extra.size() - 1); - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) { + if (cumulative_size != txs_size + get_object_blobsize(b.minerTx)) { + CHECK_AND_ASSERT_MES(cumulative_size + 1 == txs_size + get_object_blobsize(b.minerTx), false, "unexpected case: cumulative_size=" << cumulative_size << " + 1 is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.minerTx)=" << get_object_blobsize(b.minerTx)); + b.minerTx.extra.resize(b.minerTx.extra.size() - 1); + if (cumulative_size != txs_size + get_object_blobsize(b.minerTx)) { //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_size LOG_PRINT_RED("Miner tx creation have no luck with delta_extra size = " << delta << " and " << delta - 1, LOG_LEVEL_2); cumulative_size += delta - 1; continue; } - LOG_PRINT_GREEN("Setting extra for block: " << b.miner_tx.extra.size() << ", try_count=" << try_count, LOG_LEVEL_1); + LOG_PRINT_GREEN("Setting extra for block: " << b.minerTx.extra.size() << ", try_count=" << try_count, LOG_LEVEL_1); } } - CHECK_AND_ASSERT_MES(cumulative_size == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + CHECK_AND_ASSERT_MES(cumulative_size == txs_size + get_object_blobsize(b.minerTx), false, "unexpected case: cumulative_size=" << cumulative_size << " is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.minerTx)=" << get_object_blobsize(b.minerTx)); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size << ", cumulative size " << cumulative_size << " is now good"); @@ -629,11 +759,11 @@ bool blockchain_storage::create_block_template(block& b, const account_public_ad } bool blockchain_storage::complete_timestamps_vector(uint64_t start_top_height, std::vector& timestamps) { - if (timestamps.size() >= BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) + if (timestamps.size() >= m_currency.timestampCheckWindow()) return true; CRITICAL_REGION_LOCAL(m_blockchain_lock); - size_t need_elements = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - timestamps.size(); + size_t need_elements = m_currency.timestampCheckWindow() - timestamps.size(); CHECK_AND_ASSERT_MES(start_top_height < m_blocks.size(), false, "internal error: passed start_height = " << start_top_height << " not less then m_blocks.size()=" << m_blocks.size()); size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0; do @@ -646,7 +776,7 @@ bool blockchain_storage::complete_timestamps_vector(uint64_t start_top_height, s return true; } -bool blockchain_storage::handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc) { +bool blockchain_storage::handle_alternative_block(const Block& b, const crypto::hash& id, block_verification_context& bvc) { CRITICAL_REGION_LOCAL(m_blockchain_lock); uint64_t block_height = get_block_height(b); @@ -657,18 +787,30 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: } if (!m_checkpoints.is_alternative_block_allowed(get_current_blockchain_height(), block_height)) { - LOG_PRINT_RED_L0("Block with id: " << id - << ENDL << " can't be accepted for alternative chain, block height: " << block_height - << ENDL << " blockchain height: " << get_current_blockchain_height()); + LOG_PRINT_L2("Block with id: " << id << std::endl << + " can't be accepted for alternative chain, block height: " << block_height << std::endl << + " blockchain height: " << get_current_blockchain_height()); + bvc.m_verifivation_failed = true; + return false; + } + + size_t cumulativeSize; + if (!getBlockCumulativeSize(b, cumulativeSize)) { + LOG_PRINT_L2("Block with id: " << id << " has at least one unknown transaction. Cumulative size is calculated imprecisely"); + } + + if (!checkCumulativeBlockSize(id, cumulativeSize, block_height)) { bvc.m_verifivation_failed = true; return false; } //block is not related with head of main chain //first of all - look in alternative chains container - auto it_main_prev = m_blockMap.find(b.prev_id); - auto it_prev = m_alternative_chains.find(b.prev_id); - if (it_prev != m_alternative_chains.end() || it_main_prev != m_blockMap.end()) { + uint64_t mainPrevHeight = 0; + const bool mainPrev = m_blockIndex.getBlockHeight(b.prevId, mainPrevHeight); + const auto it_prev = m_alternative_chains.find(b.prevId); + + if (it_prev != m_alternative_chains.end() || mainPrev) { //we have new block in alternative chain //build alternative subchain, front -> mainchain, back -> alternative head @@ -678,7 +820,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: while (alt_it != m_alternative_chains.end()) { alt_chain.push_front(alt_it); timestamps.push_back(alt_it->second.bl.timestamp); - alt_it = m_alternative_chains.find(alt_it->second.bl.prev_id); + alt_it = m_alternative_chains.find(alt_it->second.bl.prevId); } if (alt_chain.size()) { @@ -686,11 +828,11 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: CHECK_AND_ASSERT_MES(m_blocks.size() > alt_chain.front()->second.height, false, "main blockchain wrong height"); crypto::hash h = null_hash; get_block_hash(m_blocks[alt_chain.front()->second.height - 1].bl, h); - CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain have wrong connection to main chain"); + CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prevId, false, "alternative chain have wrong connection to main chain"); complete_timestamps_vector(alt_chain.front()->second.height - 1, timestamps); } else { - CHECK_AND_ASSERT_MES(it_main_prev != m_blockMap.end(), false, "internal error: broken imperative condition it_main_prev != m_blocks_index.end()"); - complete_timestamps_vector(it_main_prev->second, timestamps); + CHECK_AND_ASSERT_MES(mainPrev, false, "internal error: broken imperative condition it_main_prev != m_blocks_index.end()"); + complete_timestamps_vector(mainPrevHeight, timestamps); } //check timestamp correct @@ -702,9 +844,9 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: return false; } - Block bei = boost::value_initialized(); + BlockEntry bei = boost::value_initialized(); bei.bl = b; - bei.height = alt_chain.size() ? it_prev->second.height + 1 : it_main_prev->second + 1; + bei.height = static_cast(alt_chain.size() ? it_prev->second.height + 1 : mainPrevHeight + 1); bool is_a_checkpoint; if (!m_checkpoints.check_block(bei.height, id, is_a_checkpoint)) { @@ -718,8 +860,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: difficulty_type current_diff = get_next_difficulty_for_alternative_chain(alt_chain, bei); CHECK_AND_ASSERT_MES(current_diff, false, "!!!!!!! DIFFICULTY OVERHEAD !!!!!!!"); crypto::hash proof_of_work = null_hash; - get_block_longhash(m_cn_context, bei.bl, proof_of_work, bei.height); - if (!check_hash(proof_of_work, current_diff)) { + if (!m_currency.checkProofOfWork(m_cn_context, bei.bl, current_diff, proof_of_work)) { LOG_PRINT_RED_L0("Block with id: " << id << ENDL << " for alternative chain, have not enough proof of work: " << proof_of_work << ENDL << " expected difficulty: " << current_diff); @@ -733,7 +874,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: return false; } - bei.cumulative_difficulty = alt_chain.size() ? it_prev->second.cumulative_difficulty : m_blocks[it_main_prev->second].cumulative_difficulty; + bei.cumulative_difficulty = alt_chain.size() ? it_prev->second.cumulative_difficulty : m_blocks[mainPrevHeight].cumulative_difficulty; bei.cumulative_difficulty += current_diff; #ifdef _DEBUG @@ -778,7 +919,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: return true; } -bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) { +bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) { CRITICAL_REGION_LOCAL(m_blockchain_lock); if (start_offset >= m_blocks.size()) return false; @@ -786,14 +927,14 @@ bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::li { blocks.push_back(m_blocks[i].bl); std::list missed_ids; - get_transactions(m_blocks[i].bl.tx_hashes, txs, missed_ids); + get_transactions(m_blocks[i].bl.txHashes, txs, missed_ids); CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "have missed transactions in own block in main blockchain"); } return true; } -bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) { +bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) { CRITICAL_REGION_LOCAL(m_blockchain_lock); if (start_offset >= m_blocks.size()) { return false; @@ -809,26 +950,26 @@ bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::li bool blockchain_storage::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp) { CRITICAL_REGION_LOCAL(m_blockchain_lock); rsp.current_blockchain_height = get_current_blockchain_height(); - std::list blocks; + std::list blocks; get_blocks(arg.blocks, blocks, rsp.missed_ids); for (const auto& bl : blocks) { std::list missed_tx_id; - std::list txs; - get_transactions(bl.tx_hashes, txs, rsp.missed_ids); + std::list txs; + get_transactions(bl.txHashes, txs, rsp.missed_ids); CHECK_AND_ASSERT_MES(!missed_tx_id.size(), false, "Internal error: have missed missed_tx_id.size()=" << missed_tx_id.size() << ENDL << "for block id = " << get_block_hash(bl)); rsp.blocks.push_back(block_complete_entry()); block_complete_entry& e = rsp.blocks.back(); //pack block e.block = t_serializable_object_to_blob(bl); //pack transactions - for (transaction& tx : txs) { + for (Transaction& tx : txs) { e.txs.push_back(t_serializable_object_to_blob(tx)); } } //get another transactions, if need - std::list txs; + std::list txs; get_transactions(arg.txs, txs, rsp.missed_ids); //pack aside transactions for (const auto& tx : txs) { @@ -838,7 +979,7 @@ bool blockchain_storage::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& return true; } -bool blockchain_storage::get_alternative_blocks(std::list& blocks) { +bool blockchain_storage::get_alternative_blocks(std::list& blocks) { CRITICAL_REGION_LOCAL(m_blockchain_lock); for (auto& alt_bl : m_alternative_chains) { blocks.push_back(alt_bl.second.bl); @@ -854,18 +995,18 @@ size_t blockchain_storage::get_alternative_blocks_count() { bool blockchain_storage::add_out_to_get_random_outs(std::vector>& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) { CRITICAL_REGION_LOCAL(m_blockchain_lock); - const transaction& tx = transactionByIndex(amount_outs[i].first).tx; + const Transaction& tx = transactionByIndex(amount_outs[i].first).tx; CHECK_AND_ASSERT_MES(tx.vout.size() > amount_outs[i].second, false, "internal error: in global outs index, transaction out index=" << amount_outs[i].second << " more than transaction outputs = " << tx.vout.size() << ", for tx id = " << get_transaction_hash(tx)); - CHECK_AND_ASSERT_MES(tx.vout[amount_outs[i].second].target.type() == typeid(txout_to_key), false, "unknown tx out type"); + CHECK_AND_ASSERT_MES(tx.vout[amount_outs[i].second].target.type() == typeid(TransactionOutputToKey), false, "unknown tx out type"); //check if transaction is unlocked - if (!is_tx_spendtime_unlocked(tx.unlock_time)) + if (!is_tx_spendtime_unlocked(tx.unlockTime)) return false; COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); oen.global_amount_index = i; - oen.out_key = boost::get(tx.vout[amount_outs[i].second].target).key; + oen.out_key = boost::get(tx.vout[amount_outs[i].second].target).key; return true; } @@ -878,7 +1019,7 @@ size_t blockchain_storage::find_end_of_allowed_index(const std::vector(time(NULL))); CRITICAL_REGION_LOCAL(m_blockchain_lock); + for (uint64_t amount : req.amounts) { COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs = *res.outs.insert(res.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount()); result_outs.amount = amount; @@ -903,22 +1044,11 @@ bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDO //lets find upper bound of not fresh outs size_t up_index_limit = find_end_of_allowed_index(amount_outs); CHECK_AND_ASSERT_MES(up_index_limit <= amount_outs.size(), false, "internal error: find_end_of_allowed_index returned wrong index=" << up_index_limit << ", with amount_outs.size = " << amount_outs.size()); - if (amount_outs.size() > req.outs_count) { - std::set used; - size_t try_count = 0; - for (uint64_t j = 0; j != req.outs_count && try_count < up_index_limit;) { - size_t i = rand() % up_index_limit; - if (used.count(i)) - continue; - bool added = add_out_to_get_random_outs(amount_outs, result_outs, amount, i); - used.insert(i); - if (added) - ++j; - ++try_count; - } - } else { - for (size_t i = 0; i != up_index_limit; i++) { - add_out_to_get_random_outs(amount_outs, result_outs, amount, i); + + if (up_index_limit > 0) { + ShuffleGenerator> generator(up_index_limit); + for (uint64_t j = 0; j < up_index_limit && result_outs.outs.size() < req.outs_count; ++j) { + add_out_to_get_random_outs(amount_outs, result_outs, amount, generator()); } } } @@ -944,32 +1074,13 @@ bool blockchain_storage::find_blockchain_supplement(const std::listsecond; - return true; + //this should NEVER happen, but, dose of paranoia in such cases is not too bad + LOG_ERROR("Internal error handling connection, can't find split point"); + return false; } uint64_t blockchain_storage::block_difficulty(size_t i) @@ -996,7 +1107,7 @@ void blockchain_storage::print_blockchain(uint64_t start_index, uint64_t end_ind { ss << "height " << i << ", timestamp " << m_blocks[i].bl.timestamp << ", cumul_dif " << m_blocks[i].cumulative_difficulty << ", cumul_size " << m_blocks[i].block_cumulative_size << "\nid\t\t" << get_block_hash(m_blocks[i].bl) - << "\ndifficulty\t\t" << block_difficulty(i) << ", nonce " << m_blocks[i].bl.nonce << ", tx_count " << m_blocks[i].bl.tx_hashes.size() << ENDL; + << "\ndifficulty\t\t" << block_difficulty(i) << ", nonce " << m_blocks[i].bl.nonce << ", tx_count " << m_blocks[i].bl.txHashes.size() << ENDL; } LOG_PRINT_L1("Current blockchain:" << ENDL << ss.str()); LOG_PRINT_L0("Blockchain printed with log level 1"); @@ -1005,11 +1116,17 @@ void blockchain_storage::print_blockchain(uint64_t start_index, uint64_t end_ind void blockchain_storage::print_blockchain_index() { std::stringstream ss; CRITICAL_REGION_LOCAL(m_blockchain_lock); - for (auto& i : m_blockMap) { - ss << "id\t\t" << i.first << " height" << i.second << ENDL << ""; + + std::list blockIds; + m_blockIndex.getBlockIds(0, std::numeric_limits::max(), blockIds); + + LOG_PRINT_L0("Current blockchain index:" << ENDL); + + size_t height = 0; + for (auto i = blockIds.begin(); i != blockIds.end(); ++i, ++height) { + LOG_PRINT_L0("id\t\t" << *i << " height" << height); } - LOG_PRINT_L0("Current blockchain index:" << ENDL << ss.str()); } void blockchain_storage::print_blockchain_outs(const std::string& file) { @@ -1039,19 +1156,11 @@ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) { +bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) { CRITICAL_REGION_LOCAL(m_blockchain_lock); if (!find_blockchain_supplement(qblock_ids, start_height)) { return false; @@ -1063,7 +1172,7 @@ bool blockchain_storage::find_blockchain_supplement(const std::list mis; - get_transactions(m_blocks[i].bl.tx_hashes, blocks.back().second, mis); + get_transactions(m_blocks[i].bl.txHashes, blocks.back().second, mis); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); } @@ -1073,7 +1182,7 @@ bool blockchain_storage::find_blockchain_supplement(const std::listsecond); + const TransactionEntry& tx = transactionByIndex(it->second); CHECK_AND_ASSERT_MES(tx.m_global_output_indexes.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty"); indexs.resize(tx.m_global_output_indexes.size()); for (size_t i = 0; i < tx.m_global_output_indexes.size(); ++i) { @@ -1105,8 +1214,12 @@ bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std:: return true; } -bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) { +bool blockchain_storage::check_tx_inputs(const Transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id, BlockInfo* tail) { CRITICAL_REGION_LOCAL(m_blockchain_lock); + + if (tail) + tail->id = get_tail_id(tail->height); + bool res = check_tx_inputs(tx, &max_used_block_height); if (!res) return false; CHECK_AND_ASSERT_MES(max_used_block_height < m_blocks.size(), false, "internal error: max used block index=" << max_used_block_height << " is not less then blockchain size = " << m_blocks.size()); @@ -1114,63 +1227,73 @@ bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t& max_us return true; } -bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) { - for(const txin_v& in : tx.vin) { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, in_to_key, true); - if (have_tx_keyimg_as_spent(in_to_key.k_image)) { - return true; +bool blockchain_storage::have_tx_keyimges_as_spent(const Transaction &tx) { + for (const auto& in : tx.vin) { + if (in.type() == typeid(TransactionInputToKey)) { + if (have_tx_keyimg_as_spent(boost::get(in).keyImage)) { + return true; + } } } return false; } -bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) { +bool blockchain_storage::check_tx_inputs(const Transaction& tx, uint64_t* pmax_used_block_height) { crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); return check_tx_inputs(tx, tx_prefix_hash, pmax_used_block_height); } -bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height) { - size_t sig_index = 0; +bool blockchain_storage::check_tx_inputs(const Transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height) { + size_t inputIndex = 0; if (pmax_used_block_height) { *pmax_used_block_height = 0; } + crypto::hash transactionHash = get_transaction_hash(tx); for (const auto& txin : tx.vin) { - CHECK_AND_ASSERT_MES(txin.type() == typeid(txin_to_key), false, "wrong type id in tx input at blockchain_storage::check_tx_inputs"); - const txin_to_key& in_to_key = boost::get(txin); + assert(inputIndex < tx.signatures.size()); + if (txin.type() == typeid(TransactionInputToKey)) { + const TransactionInputToKey& in_to_key = boost::get(txin); + CHECK_AND_ASSERT_MES(!in_to_key.keyOffsets.empty(), false, "empty in_to_key.keyOffsets in transaction with id " << get_transaction_hash(tx)); - CHECK_AND_ASSERT_MES(in_to_key.key_offsets.size(), false, "empty in_to_key.key_offsets in transaction with id " << get_transaction_hash(tx)); + if (have_tx_keyimg_as_spent(in_to_key.keyImage)) { + LOG_PRINT_L1("Key image already spent in blockchain: " << epee::string_tools::pod_to_hex(in_to_key.keyImage)); + return false; + } - if (have_tx_keyimg_as_spent(in_to_key.k_image)) - { - LOG_PRINT_L1("Key image already spent in blockchain: " << epee::string_tools::pod_to_hex(in_to_key.k_image)); + if (!check_tx_input(in_to_key, tx_prefix_hash, tx.signatures[inputIndex], pmax_used_block_height)) { + LOG_PRINT_L0("Failed to check ring signature for tx " << transactionHash); + return false; + } + + ++inputIndex; + } else if (txin.type() == typeid(TransactionInputMultisignature)) { + if (!validateInput(::boost::get(txin), transactionHash, tx_prefix_hash, tx.signatures[inputIndex])) { + return false; + } + + ++inputIndex; + } else { + LOG_PRINT_L0("Transaction << " << transactionHash << " contains input of unsupported type."); return false; } - - CHECK_AND_ASSERT_MES(sig_index < tx.signatures.size(), false, "wrong transaction: not signature entry for input with index= " << sig_index); - if (!check_tx_input(in_to_key, tx_prefix_hash, tx.signatures[sig_index], pmax_used_block_height)) { - LOG_PRINT_L0("Failed to check ring signature for tx " << get_transaction_hash(tx)); - return false; - } - - sig_index++; } return true; } bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) { - if (unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { + if (unlock_time < m_currency.maxBlockHeight()) { //interpret as block index - if (get_current_blockchain_height() - 1 + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) + if (get_current_blockchain_height() - 1 + m_currency.lockedTxAllowedDeltaBlocks() >= unlock_time) return true; else return false; } else { //interpret as time uint64_t current_time = static_cast(time(NULL)); - if (current_time + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS >= unlock_time) + if (current_time + m_currency.lockedTxAllowedDeltaSeconds() >= unlock_time) return true; else return false; @@ -1179,7 +1302,7 @@ bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) { return false; } -bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height) { +bool blockchain_storage::check_tx_input(const TransactionInputToKey& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height) { CRITICAL_REGION_LOCAL(m_blockchain_lock); struct outputs_visitor @@ -1188,20 +1311,20 @@ bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::h blockchain_storage& m_bch; outputs_visitor(std::vector& results_collector, blockchain_storage& bch) :m_results_collector(results_collector), m_bch(bch) {} - bool handle_output(const transaction& tx, const tx_out& out) { + bool handle_output(const Transaction& tx, const TransactionOutput& out) { //check tx unlock time - if (!m_bch.is_tx_spendtime_unlocked(tx.unlock_time)) { - LOG_PRINT_L0("One of outputs for one of inputs have wrong tx.unlock_time = " << tx.unlock_time); + if (!m_bch.is_tx_spendtime_unlocked(tx.unlockTime)) { + LOG_PRINT_L0("One of outputs for one of inputs have wrong tx.unlockTime = " << tx.unlockTime); return false; } - if (out.target.type() != typeid(txout_to_key)) + if (out.target.type() != typeid(TransactionOutputToKey)) { LOG_PRINT_L0("Output have wrong type id, which=" << out.target.which()); return false; } - m_results_collector.push_back(&boost::get(out.target).key); + m_results_collector.push_back(&boost::get(out.target).key); return true; } }; @@ -1210,12 +1333,13 @@ bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::h std::vector output_keys; outputs_visitor vi(output_keys, *this); if (!scan_outputkeys_for_indexes(txin, vi, pmax_related_block_height)) { - LOG_PRINT_L0("Failed to get output keys for tx with amount = " << print_money(txin.amount) << " and count indexes " << txin.key_offsets.size()); + LOG_PRINT_L0("Failed to get output keys for tx with amount = " << m_currency.formatAmount(txin.amount) << + " and count indexes " << txin.keyOffsets.size()); return false; } - if (txin.key_offsets.size() != output_keys.size()) { - LOG_PRINT_L0("Output keys for tx with amount = " << txin.amount << " and count indexes " << txin.key_offsets.size() << " returned wrong keys count " << output_keys.size()); + if (txin.keyOffsets.size() != output_keys.size()) { + LOG_PRINT_L0("Output keys for tx with amount = " << txin.amount << " and count indexes " << txin.keyOffsets.size() << " returned wrong keys count " << output_keys.size()); return false; } @@ -1224,7 +1348,7 @@ bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::h return true; } - return crypto::check_ring_signature(tx_prefix_hash, txin.k_image, output_keys, sig.data()); + return crypto::check_ring_signature(tx_prefix_hash, txin.keyImage, output_keys, sig.data()); } uint64_t blockchain_storage::get_adjusted_time() { @@ -1232,14 +1356,14 @@ uint64_t blockchain_storage::get_adjusted_time() { return time(NULL); } -bool blockchain_storage::check_block_timestamp_main(const block& b) { - if (b.timestamp > get_adjusted_time() + CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT) { +bool blockchain_storage::check_block_timestamp_main(const Block& b) { + if (b.timestamp > get_adjusted_time() + m_currency.blockFutureTimeLimit()) { LOG_PRINT_L0("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", bigger than adjusted time + 2 hours"); return false; } std::vector timestamps; - size_t offset = m_blocks.size() <= BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW ? 0 : m_blocks.size() - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW; + size_t offset = m_blocks.size() <= m_currency.timestampCheckWindow() ? 0 : m_blocks.size() - m_currency.timestampCheckWindow(); for (; offset != m_blocks.size(); ++offset) { timestamps.push_back(m_blocks[offset].bl.timestamp); } @@ -1247,39 +1371,74 @@ bool blockchain_storage::check_block_timestamp_main(const block& b) { return check_block_timestamp(std::move(timestamps), b); } -bool blockchain_storage::check_block_timestamp(std::vector timestamps, const block& b) { - if (timestamps.size() < BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) { +bool blockchain_storage::check_block_timestamp(std::vector timestamps, const Block& b) { + if (timestamps.size() < m_currency.timestampCheckWindow()) { return true; } uint64_t median_ts = epee::misc_utils::median(timestamps); if (b.timestamp < median_ts) { - LOG_PRINT_L0("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", less than median of last " << BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW << " blocks, " << median_ts); + LOG_PRINT_L0("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << + ", less than median of last " << m_currency.timestampCheckWindow() << " blocks, " << median_ts); return false; } return true; } +bool blockchain_storage::checkCumulativeBlockSize(const crypto::hash& blockId, size_t cumulativeBlockSize, uint64_t height) { + size_t maxBlockCumulativeSize = m_currency.maxBlockCumulativeSize(height); + if (cumulativeBlockSize > maxBlockCumulativeSize) { + LOG_PRINT_L0("Block " << blockId << " is too big: " << cumulativeBlockSize << " bytes, " << + "exptected no more than " << maxBlockCumulativeSize << " bytes"); + return false; + } + + return true; +} + +// Returns true, if cumulativeSize is calculated precisely, else returns false. +bool blockchain_storage::getBlockCumulativeSize(const Block& block, size_t& cumulativeSize) { + std::vector blockTxs; + std::vector missedTxs; + get_transactions(block.txHashes, blockTxs, missedTxs, true); + + cumulativeSize = get_object_blobsize(block.minerTx); + for (const Transaction& tx : blockTxs) { + cumulativeSize += get_object_blobsize(tx); + } + + return missedTxs.empty(); +} + +// Precondition: m_blockchain_lock is locked. bool blockchain_storage::update_next_comulative_size_limit() { std::vector sz; - get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + get_last_n_blocks_sizes(sz, m_currency.rewardBlocksWindow()); uint64_t median = epee::misc_utils::median(sz); - if (median <= CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE) - median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE; + if (median <= m_currency.blockGrantedFullRewardZone()) { + median = m_currency.blockGrantedFullRewardZone(); + } m_current_block_cumul_sz_limit = median * 2; return true; } -bool blockchain_storage::add_new_block(const block& bl_, block_verification_context& bvc) { +bool blockchain_storage::add_new_block(const Block& bl_, block_verification_context& bvc) { //copy block here to let modify block.target - block bl = bl_; - crypto::hash id = get_block_hash(bl); - CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process - CRITICAL_REGION_LOCAL1(m_blockchain_lock); + Block bl = bl_; + crypto::hash id; + if (!get_block_hash(bl, id)) { + LOG_ERROR("Failed to get block hash, possible block has invalid format"); + bvc.m_verifivation_failed = true; + return false; + } + + bool add_result; + CRITICAL_REGION_BEGIN(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process + CRITICAL_REGION_BEGIN1(m_blockchain_lock); if (have_block(id)) { LOG_PRINT_L3("block with id = " << id << " already exists"); bvc.m_already_exists = true; @@ -1287,34 +1446,41 @@ bool blockchain_storage::add_new_block(const block& bl_, block_verification_cont } //check that block refers to chain tail - if (!(bl.prev_id == get_tail_id())) { + if (!(bl.prevId == get_tail_id())) { //chain switching or wrong block bvc.m_added_to_main_chain = false; - return handle_alternative_block(bl, id, bvc); - //never relay alternative blocks + add_result = handle_alternative_block(bl, id, bvc); + } else { + add_result = pushBlock(bl, bvc); + } + CRITICAL_REGION_END(); + CRITICAL_REGION_END(); + + if (add_result && bvc.m_added_to_main_chain) { + m_observerManager.notify(&IBlockchainStorageObserver::blockchainUpdated); } - return pushBlock(bl, bvc); + return add_result; } -const blockchain_storage::Transaction& blockchain_storage::transactionByIndex(TransactionIndex index) { +const blockchain_storage::TransactionEntry& blockchain_storage::transactionByIndex(TransactionIndex index) { return m_blocks[index.block].transactions[index.transaction]; } -bool blockchain_storage::pushBlock(const block& blockData, block_verification_context& bvc) { +bool blockchain_storage::pushBlock(const Block& blockData, block_verification_context& bvc) { CRITICAL_REGION_LOCAL(m_blockchain_lock); TIME_MEASURE_START(block_processing_time); crypto::hash blockHash = get_block_hash(blockData); - if (m_blockMap.count(blockHash) != 0) { + if (m_blockIndex.hasBlock(blockHash)) { LOG_ERROR("Block " << blockHash << " already exists in blockchain."); bvc.m_verifivation_failed = true; return false; } - if (blockData.prev_id != get_tail_id()) { - LOG_PRINT_L0("Block " << blockHash << " has wrong prev_id: " << blockData.prev_id << ", expected: " << get_tail_id()); + if (blockData.prevId != get_tail_id()) { + LOG_PRINT_L0("Block " << blockHash << " has wrong prevId: " << blockData.prevId << ", expected: " << get_tail_id()); bvc.m_verifivation_failed = true; return false; } @@ -1339,8 +1505,7 @@ bool blockchain_storage::pushBlock(const block& blockData, block_verification_co return false; } } else { - proof_of_work = get_block_longhash(m_cn_context, blockData, m_blocks.size()); - if (!check_hash(proof_of_work, currentDifficulty)) { + if (!m_currency.checkProofOfWork(m_cn_context, blockData, currentDifficulty, proof_of_work)) { LOG_PRINT_L0("Block " << blockHash << ", has too weak proof of work: " << proof_of_work << ", expected difficulty: " << currentDifficulty); bvc.m_verifivation_failed = true; return false; @@ -1355,19 +1520,19 @@ bool blockchain_storage::pushBlock(const block& blockData, block_verification_co return false; } - crypto::hash minerTransactionHash = get_transaction_hash(blockData.miner_tx); + crypto::hash minerTransactionHash = get_transaction_hash(blockData.minerTx); - Block block; + BlockEntry block; block.bl = blockData; block.transactions.resize(1); - block.transactions[0].tx = blockData.miner_tx; + block.transactions[0].tx = blockData.minerTx; TransactionIndex transactionIndex = { static_cast(m_blocks.size()), static_cast(0) }; pushTransaction(block, minerTransactionHash, transactionIndex); - size_t coinbase_blob_size = get_object_blobsize(blockData.miner_tx); + size_t coinbase_blob_size = get_object_blobsize(blockData.minerTx); size_t cumulative_block_size = coinbase_blob_size; uint64_t fee_summary = 0; - for (const crypto::hash& tx_id : blockData.tx_hashes) { + for (const crypto::hash& tx_id : blockData.txHashes) { block.transactions.resize(block.transactions.size() + 1); size_t blob_size = 0; uint64_t fee = 0; @@ -1400,9 +1565,15 @@ bool blockchain_storage::pushBlock(const block& blockData, block_verification_co fee_summary += fee; } - uint64_t base_reward = 0; - uint64_t already_generated_coins = m_blocks.size() ? m_blocks.back().already_generated_coins : 0; - if (!validate_miner_transaction(blockData, cumulative_block_size, fee_summary, base_reward, already_generated_coins)) { + if (!checkCumulativeBlockSize(blockHash, cumulative_block_size, m_blocks.size())) { + bvc.m_verifivation_failed = true; + return false; + } + + int64_t emissionChange = 0; + uint64_t reward = 0; + uint64_t already_generated_coins = m_blocks.empty() ? 0 : m_blocks.back().already_generated_coins; + if (!validate_miner_transaction(blockData, m_blocks.size(), cumulative_block_size, already_generated_coins, fee_summary, reward, emissionChange)) { LOG_PRINT_L0("Block " << blockHash << " has invalid miner transaction"); bvc.m_verifivation_failed = true; popTransactions(block, minerTransactionHash); @@ -1412,34 +1583,35 @@ bool blockchain_storage::pushBlock(const block& blockData, block_verification_co block.height = static_cast(m_blocks.size()); block.block_cumulative_size = cumulative_block_size; block.cumulative_difficulty = currentDifficulty; - block.already_generated_coins = already_generated_coins + base_reward; + block.already_generated_coins = already_generated_coins + emissionChange; if (m_blocks.size() > 0) { block.cumulative_difficulty += m_blocks.back().cumulative_difficulty; } pushBlock(block); - update_next_comulative_size_limit(); TIME_MEASURE_FINISH(block_processing_time); LOG_PRINT_L1("+++++ BLOCK SUCCESSFULLY ADDED" << ENDL << "id:\t" << blockHash << ENDL << "PoW:\t" << proof_of_work << ENDL << "HEIGHT " << block.height << ", difficulty:\t" << currentDifficulty - << ENDL << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) - << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size + << ENDL << "block reward: " << m_currency.formatAmount(reward) << ", fee = " << m_currency.formatAmount(fee_summary) + << ", coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); bvc.m_added_to_main_chain = true; + + update_next_comulative_size_limit(); + return true; } -bool blockchain_storage::pushBlock(Block& block) { +bool blockchain_storage::pushBlock(BlockEntry& block) { crypto::hash blockHash = get_block_hash(block.bl); - auto result = m_blockMap.insert(std::make_pair(blockHash, static_cast(m_blocks.size()))); - if (!result.second) { - LOG_ERROR("Duplicate block was pushed to blockchain."); - return false; - } m_blocks.push_back(block); + m_blockIndex.push(blockHash); + + assert(m_blockIndex.size() == m_blocks.size()); + return true; } @@ -1449,29 +1621,35 @@ void blockchain_storage::popBlock(const crypto::hash& blockHash) { return; } - popTransactions(m_blocks.back(), get_transaction_hash(m_blocks.back().bl.miner_tx)); + popTransactions(m_blocks.back(), get_transaction_hash(m_blocks.back().bl.minerTx)); m_blocks.pop_back(); - size_t count = m_blockMap.erase(blockHash); - if (count != 1) { - LOG_ERROR("Blockchain consistency broken - cannot find block by hash."); - } + m_blockIndex.pop(); + + assert(m_blockIndex.size() == m_blocks.size()); } -bool blockchain_storage::pushTransaction(Block& block, const crypto::hash& transactionHash, TransactionIndex transactionIndex) { +bool blockchain_storage::pushTransaction(BlockEntry& block, const crypto::hash& transactionHash, TransactionIndex transactionIndex) { auto result = m_transactionMap.insert(std::make_pair(transactionHash, transactionIndex)); if (!result.second) { LOG_ERROR("Duplicate transaction was pushed to blockchain."); return false; } - Transaction& transaction = block.transactions[transactionIndex.transaction]; + TransactionEntry& transaction = block.transactions[transactionIndex.transaction]; + + if (!checkMultisignatureInputsDiff(transaction.tx)) { + LOG_ERROR("Double spending transaction was pushed to blockchain."); + m_transactionMap.erase(transactionHash); + return false; + } + for (size_t i = 0; i < transaction.tx.vin.size(); ++i) { - if (transaction.tx.vin[i].type() == typeid(txin_to_key)) { - auto result = m_spent_keys.insert(::boost::get(transaction.tx.vin[i]).k_image); + if (transaction.tx.vin[i].type() == typeid(TransactionInputToKey)) { + auto result = m_spent_keys.insert(::boost::get(transaction.tx.vin[i]).keyImage); if (!result.second) { LOG_ERROR("Double spending transaction was pushed to blockchain."); for (size_t j = 0; j < i; ++j) { - m_spent_keys.erase(::boost::get(transaction.tx.vin[i - 1 - j]).k_image); + m_spent_keys.erase(::boost::get(transaction.tx.vin[i - 1 - j]).keyImage); } m_transactionMap.erase(transactionHash); @@ -1480,52 +1658,109 @@ bool blockchain_storage::pushTransaction(Block& block, const crypto::hash& trans } } + for (const auto& inv : transaction.tx.vin) { + if (inv.type() == typeid(TransactionInputMultisignature)) { + const TransactionInputMultisignature& in = ::boost::get(inv); + auto& amountOutputs = m_multisignatureOutputs[in.amount]; + amountOutputs[in.outputIndex].isUsed = true; + } + } + transaction.m_global_output_indexes.resize(transaction.tx.vout.size()); for (uint16_t output = 0; output < transaction.tx.vout.size(); ++output) { - auto& amountOutputs = m_outputs[transaction.tx.vout[output].amount]; - transaction.m_global_output_indexes[output] = amountOutputs.size(); - amountOutputs.push_back(std::make_pair<>(transactionIndex, output)); + if (transaction.tx.vout[output].target.type() == typeid(TransactionOutputToKey)) { + auto& amountOutputs = m_outputs[transaction.tx.vout[output].amount]; + transaction.m_global_output_indexes[output] = amountOutputs.size(); + amountOutputs.push_back(std::make_pair<>(transactionIndex, output)); + } else if (transaction.tx.vout[output].target.type() == typeid(TransactionOutputMultisignature)) { + auto& amountOutputs = m_multisignatureOutputs[transaction.tx.vout[output].amount]; + transaction.m_global_output_indexes[output] = amountOutputs.size(); + MultisignatureOutputUsage outputUsage = { transactionIndex, output, false }; + amountOutputs.push_back(outputUsage); + } } return true; } -void blockchain_storage::popTransaction(const transaction& transaction, const crypto::hash& transactionHash) { +void blockchain_storage::popTransaction(const Transaction& transaction, const crypto::hash& transactionHash) { TransactionIndex transactionIndex = m_transactionMap.at(transactionHash); - for (size_t output = 0; output < transaction.vout.size(); ++output) { - auto amountOutputs = m_outputs.find(transaction.vout[transaction.vout.size() - 1 - output].amount); - if (amountOutputs == m_outputs.end()) { - LOG_ERROR("Blockchain consistency broken - cannot find specific amount in outputs map."); - continue; - } + for (size_t outputIndex = 0; outputIndex < transaction.vout.size(); ++outputIndex) { + const TransactionOutput& output = transaction.vout[transaction.vout.size() - 1 - outputIndex]; + if (output.target.type() == typeid(TransactionOutputToKey)) { + auto amountOutputs = m_outputs.find(output.amount); + if (amountOutputs == m_outputs.end()) { + LOG_ERROR("Blockchain consistency broken - cannot find specific amount in outputs map."); + continue; + } - if (amountOutputs->second.empty()) { - LOG_ERROR("Blockchain consistency broken - output array for specific amount is empty."); - continue; - } + if (amountOutputs->second.empty()) { + LOG_ERROR("Blockchain consistency broken - output array for specific amount is empty."); + continue; + } - if (amountOutputs->second.back().first.block != transactionIndex.block || amountOutputs->second.back().first.transaction != transactionIndex.transaction) { - LOG_ERROR("Blockchain consistency broken - invalid transaction index."); - continue; - } + if (amountOutputs->second.back().first.block != transactionIndex.block || amountOutputs->second.back().first.transaction != transactionIndex.transaction) { + LOG_ERROR("Blockchain consistency broken - invalid transaction index."); + continue; + } - if (amountOutputs->second.back().second != transaction.vout.size() - 1 - output) { - LOG_ERROR("Blockchain consistency broken - invalid output index."); - continue; - } + if (amountOutputs->second.back().second != transaction.vout.size() - 1 - outputIndex) { + LOG_ERROR("Blockchain consistency broken - invalid output index."); + continue; + } - amountOutputs->second.pop_back(); - if (amountOutputs->second.empty()) { - m_outputs.erase(amountOutputs); + amountOutputs->second.pop_back(); + if (amountOutputs->second.empty()) { + m_outputs.erase(amountOutputs); + } + } else if (output.target.type() == typeid(TransactionOutputMultisignature)) { + auto amountOutputs = m_multisignatureOutputs.find(output.amount); + if (amountOutputs == m_multisignatureOutputs.end()) { + LOG_ERROR("Blockchain consistency broken - cannot find specific amount in outputs map."); + continue; + } + + if (amountOutputs->second.empty()) { + LOG_ERROR("Blockchain consistency broken - output array for specific amount is empty."); + continue; + } + + if (amountOutputs->second.back().isUsed) { + LOG_ERROR("Blockchain consistency broken - attempting to remove used output."); + continue; + } + + if (amountOutputs->second.back().transactionIndex.block != transactionIndex.block || amountOutputs->second.back().transactionIndex.transaction != transactionIndex.transaction) { + LOG_ERROR("Blockchain consistency broken - invalid transaction index."); + continue; + } + + if (amountOutputs->second.back().outputIndex != transaction.vout.size() - 1 - outputIndex) { + LOG_ERROR("Blockchain consistency broken - invalid output index."); + continue; + } + + amountOutputs->second.pop_back(); + if (amountOutputs->second.empty()) { + m_multisignatureOutputs.erase(amountOutputs); + } } } for (auto& input : transaction.vin) { - if (input.type() == typeid(txin_to_key)) { - size_t count = m_spent_keys.erase(::boost::get(input).k_image); + if (input.type() == typeid(TransactionInputToKey)) { + size_t count = m_spent_keys.erase(::boost::get(input).keyImage); if (count != 1) { LOG_ERROR("Blockchain consistency broken - cannot find spent key."); } + } else if (input.type() == typeid(TransactionInputMultisignature)) { + const TransactionInputMultisignature& in = ::boost::get(input); + auto& amountOutputs = m_multisignatureOutputs[in.amount]; + if (!amountOutputs[in.outputIndex].isUsed) { + LOG_ERROR("Blockchain consistency broken - multisignature output not marked as used."); + } + + amountOutputs[in.outputIndex].isUsed = false; } } @@ -1535,14 +1770,88 @@ void blockchain_storage::popTransaction(const transaction& transaction, const cr } } -void blockchain_storage::popTransactions(const Block& block, const crypto::hash& minerTransactionHash) { +void blockchain_storage::popTransactions(const BlockEntry& block, const crypto::hash& minerTransactionHash) { for (size_t i = 0; i < block.transactions.size() - 1; ++i) { - popTransaction(block.transactions[block.transactions.size() - 1 - i].tx, block.bl.tx_hashes[block.transactions.size() - 2 - i]); + popTransaction(block.transactions[block.transactions.size() - 1 - i].tx, block.bl.txHashes[block.transactions.size() - 2 - i]); tx_verification_context tvc = ::AUTO_VAL_INIT(tvc); if (!m_tx_pool.add_tx(block.transactions[block.transactions.size() - 1 - i].tx, tvc, true)) { LOG_ERROR("Cannot move transaction from blockchain to transaction pool."); } } - popTransaction(block.bl.miner_tx, minerTransactionHash); + popTransaction(block.bl.minerTx, minerTransactionHash); +} + +bool blockchain_storage::validateInput(const TransactionInputMultisignature& input, const crypto::hash& transactionHash, const crypto::hash& transactionPrefixHash, const std::vector& transactionSignatures) { + assert(input.signatures == transactionSignatures.size()); + MultisignatureOutputsContainer::const_iterator amountOutputs = m_multisignatureOutputs.find(input.amount); + if (amountOutputs == m_multisignatureOutputs.end()) { + LOG_PRINT_L1("Transaction << " << transactionHash << " contains multisignature input with invalid amount."); + return false; + } + + if (input.outputIndex >= amountOutputs->second.size()) { + LOG_PRINT_L1("Transaction << " << transactionHash << " contains multisignature input with invalid outputIndex."); + return false; + } + + const MultisignatureOutputUsage& outputIndex = amountOutputs->second[input.outputIndex]; + if (outputIndex.isUsed) { + LOG_PRINT_L1("Transaction << " << transactionHash << " contains double spending multisignature input."); + return false; + } + + const Transaction& outputTransaction = m_blocks[outputIndex.transactionIndex.block].transactions[outputIndex.transactionIndex.transaction].tx; + if (!is_tx_spendtime_unlocked(outputTransaction.unlockTime)) { + LOG_PRINT_L1("Transaction << " << transactionHash << " contains multisignature input which points to a locked transaction."); + return false; + } + + assert(outputTransaction.vout[outputIndex.outputIndex].amount == input.amount); + assert(outputTransaction.vout[outputIndex.outputIndex].target.type() == typeid(TransactionOutputMultisignature)); + const TransactionOutputMultisignature& output = ::boost::get(outputTransaction.vout[outputIndex.outputIndex].target); + if (input.signatures != output.requiredSignatures) { + LOG_PRINT_L1("Transaction << " << transactionHash << " contains multisignature input with invalid signature count."); + return false; + } + + std::size_t inputSignatureIndex = 0; + std::size_t outputKeyIndex = 0; + while (inputSignatureIndex < input.signatures) { + if (outputKeyIndex == output.keys.size()) { + LOG_PRINT_L1("Transaction << " << transactionHash << " contains multisignature input with invalid signatures."); + return false; + } + + if (crypto::check_signature(transactionPrefixHash, output.keys[outputKeyIndex], transactionSignatures[inputSignatureIndex])) { + ++inputSignatureIndex; + } + + ++outputKeyIndex; + } + + return true; +} + +bool blockchain_storage::getLowerBound(uint64_t timestamp, uint64_t startOffset, uint64_t& height) { + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + if (startOffset >= m_blocks.size()) { + return false; + } + + auto bound = std::lower_bound(m_blocks.begin() + startOffset, m_blocks.end(), timestamp - m_currency.blockFutureTimeLimit(), + [](const BlockEntry& b, uint64_t timestamp) { return b.bl.timestamp < timestamp; }); + + if (bound == m_blocks.end()) { + return false; + } + + height = std::distance(m_blocks.begin(), bound); + return true; +} + +bool blockchain_storage::getBlockIds(uint64_t startHeight, size_t maxCount, std::list& items) { + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_blockIndex.getBlockIds(startHeight, maxCount, items); } diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index e8946865..33292fec 100755 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -1,71 +1,102 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once -#include "SwappedVector.h" -#include "cryptonote_format_utils.h" -#include "tx_pool.h" +#include + +#include "google/sparse_hash_set" +#include "google/sparse_hash_map" + +#include "common/ObserverManager.h" #include "common/util.h" -#include "rpc/core_rpc_server_commands_defs.h" -#include "checkpoints.h" +#include "cryptonote_core/BlockIndex.h" +#include "cryptonote_core/checkpoints.h" +#include "cryptonote_core/Currency.h" +#include "cryptonote_core/IBlockchainStorageObserver.h" +#include "cryptonote_core/ITransactionValidator.h" +#include "cryptonote_core/SwappedVector.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "cryptonote_core/tx_pool.h" + namespace cryptonote { - class blockchain_storage { + struct NOTIFY_RESPONSE_CHAIN_ENTRY_request; + struct NOTIFY_REQUEST_GET_OBJECTS_request; + struct NOTIFY_RESPONSE_GET_OBJECTS_request; + struct COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_request; + struct COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_response; + struct COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_outs_for_amount; + + using CryptoNote::BlockInfo; + class blockchain_storage : public CryptoNote::ITransactionValidator { public: - blockchain_storage(tx_memory_pool& tx_pool):m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false) - {}; + blockchain_storage(const Currency& currency, tx_memory_pool& tx_pool); + + bool addObserver(IBlockchainStorageObserver* observer); + bool removeObserver(IBlockchainStorageObserver* observer); + + // ITransactionValidator + virtual bool checkTransactionInputs(const cryptonote::Transaction& tx, BlockInfo& maxUsedBlock); + virtual bool checkTransactionInputs(const cryptonote::Transaction& tx, BlockInfo& maxUsedBlock, BlockInfo& lastFailed); + virtual bool haveSpentKeyImages(const cryptonote::Transaction& tx); bool init() { return init(tools::get_default_data_dir(), true); } - bool init(const std::string& config_folder, bool load_existing, bool testnet = false); + bool init(const std::string& config_folder, bool load_existing); bool deinit(); + bool getLowerBound(uint64_t timestamp, uint64_t startOffset, uint64_t& height); + bool getBlockIds(uint64_t startHeight, size_t maxCount, std::list& items); + void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } - bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs); - bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks); - bool get_alternative_blocks(std::list& blocks); + bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs); + bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks); + bool get_alternative_blocks(std::list& blocks); size_t get_alternative_blocks_count(); crypto::hash get_block_id_by_height(uint64_t height); - bool get_block_by_hash(const crypto::hash &h, block &blk); + bool get_block_by_hash(const crypto::hash &h, Block &blk); bool have_tx(const crypto::hash &id); - bool have_tx_keyimges_as_spent(const transaction &tx); + bool have_tx_keyimges_as_spent(const Transaction &tx); uint64_t get_current_blockchain_height(); crypto::hash get_tail_id(); crypto::hash get_tail_id(uint64_t& height); difficulty_type get_difficulty_for_next_block(); - bool add_new_block(const block& bl_, block_verification_context& bvc); - bool reset_and_set_genesis_block(const block& b); - bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce); + uint64_t getCoinsInCirculation(); + bool add_new_block(const Block& bl_, block_verification_context& bvc); + bool reset_and_set_genesis_block(const Block& b); + bool create_block_template(Block& b, const AccountPublicAddress& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce); bool have_block(const crypto::hash& id); size_t get_total_transactions(); bool get_short_chain_history(std::list& ids); - bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp); - bool find_blockchain_supplement(const std::list& qblock_ids, std::list>>& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count); - bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp); - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); + bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset); // !!!! + bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY_request& resp); + bool find_blockchain_supplement(const std::list& qblock_ids, std::list>>& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count); + bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS_request& arg, NOTIFY_RESPONSE_GET_OBJECTS_request& rsp); + bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_response& res); bool get_backward_blocks_sizes(size_t from_height, std::vector& sz, size_t count); bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs); - bool store_blockchain(); - bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id); + bool check_tx_inputs(const Transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, BlockInfo* tail = 0); uint64_t get_current_comulative_blocksize_limit(); bool is_storing_blockchain(){return m_is_blockchain_storing;} uint64_t block_difficulty(size_t i); + bool getPoolSymmetricDifference(const std::vector& known_pool_tx_ids, const crypto::hash& known_block_id, std::vector& new_txs, std::vector& deleted_tx_ids); + template bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) { CRITICAL_REGION_LOCAL(m_blockchain_lock); for (const auto& bl_id : block_ids) { - auto it = m_blockMap.find(bl_id); - if (it == m_blockMap.end()) { + uint64_t height = 0; + if (!m_blockIndex.getBlockHeight(bl_id, height)) { missed_bs.push_back(bl_id); } else { - CHECK_AND_ASSERT_MES(it->second < m_blocks.size(), false, "Internal error: bl_id=" << epee::string_tools::pod_to_hex(bl_id) - << " have index record with offset="<second<< ", bigger then m_blocks.size()=" << m_blocks.size()); - blocks.push_back(m_blocks[it->second].bl); + CHECK_AND_ASSERT_MES(height < m_blocks.size(), false, "Internal error: bl_id=" << epee::string_tools::pod_to_hex(bl_id) + << " have index record with offset=" << height << ", bigger then m_blocks.size()=" << m_blocks.size()); + blocks.push_back(m_blocks[height].bl); } } @@ -73,7 +104,7 @@ namespace cryptonote { } template - void get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) { + void get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool checkTxPool = false) { CRITICAL_REGION_LOCAL(m_blockchain_lock); for (const auto& tx_id : txs_ids) { @@ -84,6 +115,12 @@ namespace cryptonote { txs.push_back(transactionByIndex(it->second).tx); } } + + if (checkTxPool) { + auto poolTxIds = std::move(missed_txs); + missed_txs.clear(); + m_tx_pool.getTransactions(poolTxIds, txs, missed_txs); + } } //debug functions @@ -92,8 +129,8 @@ namespace cryptonote { void print_blockchain_outs(const std::string& file); private: - struct Transaction { - transaction tx; + struct TransactionEntry { + Transaction tx; std::vector m_global_output_indexes; BEGIN_SERIALIZE_OBJECT() @@ -102,13 +139,13 @@ namespace cryptonote { END_SERIALIZE() }; - struct Block { - block bl; + struct BlockEntry { + Block bl; uint32_t height; uint64_t block_cumulative_size; difficulty_type cumulative_difficulty; uint64_t already_generated_coins; - std::vector transactions; + std::vector transactions; BEGIN_SERIALIZE_OBJECT() FIELD(bl) @@ -127,13 +164,24 @@ namespace cryptonote { template void serialize(Archive& archive, unsigned int version); }; - typedef std::unordered_set key_images_container; - typedef std::unordered_map blocks_ext_by_hash; - typedef std::map>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction + struct MultisignatureOutputUsage { + TransactionIndex transactionIndex; + uint16_t outputIndex; + bool isUsed; + template void serialize(Archive& archive, unsigned int version); + }; + + typedef google::sparse_hash_set key_images_container; + typedef std::unordered_map blocks_ext_by_hash; + typedef google::sparse_hash_map>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction + typedef std::map> MultisignatureOutputsContainer; + + const Currency& m_currency; tx_memory_pool& m_tx_pool; epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock crypto::cn_context m_cn_context; + tools::ObserverManager m_observerManager; key_images_container m_spent_keys; size_t m_current_block_cumul_sz_limit; @@ -145,54 +193,76 @@ namespace cryptonote { std::atomic m_is_in_checkpoint_zone; std::atomic m_is_blockchain_storing; - typedef SwappedVector Blocks; + typedef SwappedVector Blocks; typedef std::unordered_map BlockMap; typedef std::unordered_map TransactionMap; - Blocks m_blocks; - BlockMap m_blockMap; - TransactionMap m_transactionMap; + friend class BlockCacheSerializer; - template bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL); + Blocks m_blocks; + CryptoNote::BlockIndex m_blockIndex; + TransactionMap m_transactionMap; + MultisignatureOutputsContainer m_multisignatureOutputs; + + bool storeCache(); + template bool scan_outputkeys_for_indexes(const TransactionInputToKey& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL); bool switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain); - bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc); - difficulty_type get_next_difficulty_for_alternative_chain(const std::list& alt_chain, Block& bei); - bool prevalidate_miner_transaction(const block& b, uint64_t height); - bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins); - bool validate_transaction(const block& b, uint64_t height, const transaction& tx); - bool rollback_blockchain_switching(std::list& original_chain, size_t rollback_height); + bool handle_alternative_block(const Block& b, const crypto::hash& id, block_verification_context& bvc); + difficulty_type get_next_difficulty_for_alternative_chain(const std::list& alt_chain, BlockEntry& bei); + bool prevalidate_miner_transaction(const Block& b, uint64_t height); + bool validate_miner_transaction(const Block& b, uint64_t height, size_t cumulativeBlockSize, uint64_t alreadyGeneratedCoins, uint64_t fee, uint64_t& reward, int64_t& emissionChange); + bool validate_transaction(const Block& b, uint64_t height, const Transaction& tx); + bool rollback_blockchain_switching(std::list& original_chain, size_t rollback_height); bool get_last_n_blocks_sizes(std::vector& sz, size_t count); - bool add_out_to_get_random_outs(std::vector>& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i); + bool add_out_to_get_random_outs(std::vector>& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_outs_for_amount& result_outs, uint64_t amount, size_t i); bool is_tx_spendtime_unlocked(uint64_t unlock_time); size_t find_end_of_allowed_index(const std::vector>& amount_outs); - bool check_block_timestamp_main(const block& b); - bool check_block_timestamp(std::vector timestamps, const block& b); + bool check_block_timestamp_main(const Block& b); + bool check_block_timestamp(std::vector timestamps, const Block& b); uint64_t get_adjusted_time(); bool complete_timestamps_vector(uint64_t start_height, std::vector& timestamps); + bool checkCumulativeBlockSize(const crypto::hash& blockId, size_t cumulativeBlockSize, uint64_t height); + bool getBlockCumulativeSize(const Block& block, size_t& cumulativeSize); bool update_next_comulative_size_limit(); - bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset); - bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height = NULL); - bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height = NULL); - bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL); + bool check_tx_input(const TransactionInputToKey& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height = NULL); + bool check_tx_inputs(const Transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height = NULL); + bool check_tx_inputs(const Transaction& tx, uint64_t* pmax_used_block_height = NULL); bool have_tx_keyimg_as_spent(const crypto::key_image &key_im); - const Transaction& transactionByIndex(TransactionIndex index); - bool pushBlock(const block& blockData, block_verification_context& bvc); - bool pushBlock(Block& block); + const TransactionEntry& transactionByIndex(TransactionIndex index); + bool pushBlock(const Block& blockData, block_verification_context& bvc); + bool pushBlock(BlockEntry& block); void popBlock(const crypto::hash& blockHash); - bool pushTransaction(Block& block, const crypto::hash& transactionHash, TransactionIndex transactionIndex); - void popTransaction(const transaction& transaction, const crypto::hash& transactionHash); - void popTransactions(const Block& block, const crypto::hash& minerTransactionHash); - bool storeGenesisBlock(bool testnet); + bool pushTransaction(BlockEntry& block, const crypto::hash& transactionHash, TransactionIndex transactionIndex); + void popTransaction(const Transaction& transaction, const crypto::hash& transactionHash); + void popTransactions(const BlockEntry& block, const crypto::hash& minerTransactionHash); + bool validateInput(const TransactionInputMultisignature& input, const crypto::hash& transactionHash, const crypto::hash& transactionPrefixHash, const std::vector& transactionSignatures); + + friend class LockedBlockchainStorage; }; + class LockedBlockchainStorage: boost::noncopyable { + public: - template bool blockchain_storage::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) { + LockedBlockchainStorage(blockchain_storage& bc) + : m_bc(bc), m_lock(bc.m_blockchain_lock) {} + + blockchain_storage* operator -> () { + return &m_bc; + } + + private: + + blockchain_storage& m_bc; + epee::critical_region_t m_lock; + }; + + template bool blockchain_storage::scan_outputkeys_for_indexes(const TransactionInputToKey& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_outputs.find(tx_in_to_key.amount); - if (it == m_outputs.end() || !tx_in_to_key.key_offsets.size()) + if (it == m_outputs.end() || !tx_in_to_key.keyOffsets.size()) return false; - std::vector absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.key_offsets); + std::vector absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.keyOffsets); std::vector>& amount_outs_vec = it->second; size_t count = 0; for (uint64_t i : absolute_offsets) { @@ -204,7 +274,7 @@ namespace cryptonote { //auto tx_it = m_transactionMap.find(amount_outs_vec[i].first); //CHECK_AND_ASSERT_MES(tx_it != m_transactionMap.end(), false, "Wrong transaction id in output indexes: " << epee::string_tools::pod_to_hex(amount_outs_vec[i].first)); - const Transaction& tx = transactionByIndex(amount_outs_vec[i].first); + const TransactionEntry& tx = transactionByIndex(amount_outs_vec[i].first); CHECK_AND_ASSERT_MES(amount_outs_vec[i].second < tx.tx.vout.size(), false, "Wrong index in transaction outputs: " << amount_outs_vec[i].second << ", expected less then " << tx.tx.vout.size()); if (!vis.handle_output(tx.tx, tx.tx.vout[amount_outs_vec[i].second])) { diff --git a/src/cryptonote_core/blockchain_storage_boost_serialization.h b/src/cryptonote_core/blockchain_storage_boost_serialization.h index 77e4c726..211ad0f4 100644 --- a/src/cryptonote_core/blockchain_storage_boost_serialization.h +++ b/src/cryptonote_core/blockchain_storage_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/cryptonote_core/checkpoints.cpp b/src/cryptonote_core/checkpoints.cpp index 532e23cf..9a62e71b 100644 --- a/src/cryptonote_core/checkpoints.cpp +++ b/src/cryptonote_core/checkpoints.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/cryptonote_core/checkpoints.h b/src/cryptonote_core/checkpoints.h index 8db10bf9..e1b459b6 100644 --- a/src/cryptonote_core/checkpoints.h +++ b/src/cryptonote_core/checkpoints.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/cryptonote_core/checkpoints_create.h b/src/cryptonote_core/checkpoints_create.h deleted file mode 100644 index 85437580..00000000 --- a/src/cryptonote_core/checkpoints_create.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2011-2014 The Cryptonote developers -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#pragma once - -#include "checkpoints.h" -#include "misc_log_ex.h" - -#define ADD_CHECKPOINT(h, hash) CHECK_AND_ASSERT(checkpoints.add_checkpoint(h, hash), false); - -namespace cryptonote { - inline bool create_checkpoints(cryptonote::checkpoints& checkpoints) - { - return true; - } -} diff --git a/src/cryptonote_core/connection_context.h b/src/cryptonote_core/connection_context.h index f334b14c..7d3e81f2 100644 --- a/src/cryptonote_core/connection_context.h +++ b/src/cryptonote_core/connection_context.h @@ -1,13 +1,18 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once -#include + #include +#include +#include + #include "net/net_utils_base.h" #include "copyable_atomic.h" +#include "crypto/hash.h" + namespace cryptonote { diff --git a/src/cryptonote_core/cryptonote_basic.h b/src/cryptonote_core/cryptonote_basic.h index 8546329c..46157c87 100644 --- a/src/cryptonote_core/cryptonote_basic.h +++ b/src/cryptonote_core/cryptonote_basic.h @@ -1,347 +1,309 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once -#include -#include +#include #include -#include // memcmp -#include -#include "serialization/serialization.h" -#include "serialization/variant.h" -#include "serialization/vector.h" -#include "serialization/binary_archive.h" -#include "serialization/json_archive.h" -#include "serialization/debug_archive.h" -#include "serialization/crypto.h" -#include "serialization/keyvalue_serialization.h" // eepe named serialization -#include "string_tools.h" -#include "cryptonote_config.h" + +#include +#include + #include "crypto/crypto.h" #include "crypto/hash.h" -#include "misc_language.h" -#include "tx_extra.h" +#include "cryptonote_core/tx_extra.h" +#include "serialization/binary_archive.h" +#include "serialization/crypto.h" +#include "serialization/keyvalue_serialization.h" // eepe named serialization +#include "serialization/debug_archive.h" +#include "serialization/json_archive.h" +#include "serialization/serialization.h" +#include "serialization/variant.h" +#include "cryptonote_config.h" +namespace cryptonote { + class account_base; + struct account_keys; + struct Block; + struct Transaction; -namespace cryptonote -{ - - const static crypto::hash null_hash = AUTO_VAL_INIT(null_hash); - const static crypto::public_key null_pkey = AUTO_VAL_INIT(null_pkey); - - typedef std::vector ring_signature; - - - /* outputs */ - - struct txout_to_script - { - std::vector keys; - std::vector script; - - BEGIN_SERIALIZE_OBJECT() - FIELD(keys) - FIELD(script) - END_SERIALIZE() - }; - - struct txout_to_scripthash - { - crypto::hash hash; - }; - - struct txout_to_key - { - txout_to_key() { } - txout_to_key(const crypto::public_key &_key) : key(_key) { } - crypto::public_key key; - }; + // Implemented in cryptonote_format_utils.cpp + bool get_transaction_hash(const Transaction& t, crypto::hash& res); + const static crypto::hash null_hash = boost::value_initialized(); + const static crypto::public_key null_pkey = boost::value_initialized(); /* inputs */ - struct txin_gen - { + struct TransactionInputGenerate { size_t height; BEGIN_SERIALIZE_OBJECT() - VARINT_FIELD(height) + VARINT_FIELD(height); END_SERIALIZE() }; - struct txin_to_script - { - crypto::hash prev; - size_t prevout; - std::vector sigset; - - BEGIN_SERIALIZE_OBJECT() - FIELD(prev) - VARINT_FIELD(prevout) - FIELD(sigset) - END_SERIALIZE() - }; - - struct txin_to_scripthash - { - crypto::hash prev; - size_t prevout; - txout_to_script script; - std::vector sigset; - - BEGIN_SERIALIZE_OBJECT() - FIELD(prev) - VARINT_FIELD(prevout) - FIELD(script) - FIELD(sigset) - END_SERIALIZE() - }; - - struct txin_to_key - { + struct TransactionInputToKey { uint64_t amount; - std::vector key_offsets; - crypto::key_image k_image; // double spending protection + std::vector keyOffsets; + crypto::key_image keyImage; // double spending protection BEGIN_SERIALIZE_OBJECT() - VARINT_FIELD(amount) - FIELD(key_offsets) - FIELD(k_image) + VARINT_FIELD(amount); + FIELD(keyOffsets); + FIELD(keyImage); END_SERIALIZE() }; - - typedef boost::variant txin_v; - - typedef boost::variant txout_target_v; - - //typedef std::pair out_t; - struct tx_out - { + struct TransactionInputMultisignature { uint64_t amount; - txout_target_v target; + uint32_t signatures; + uint64_t outputIndex; BEGIN_SERIALIZE_OBJECT() - VARINT_FIELD(amount) - FIELD(target) + VARINT_FIELD(amount); + VARINT_FIELD(signatures); + VARINT_FIELD(outputIndex); END_SERIALIZE() - - }; - class transaction_prefix - { + /* outputs */ - public: + struct TransactionOutputToKey { + TransactionOutputToKey() { } + TransactionOutputToKey(const crypto::public_key &_key) : key(_key) { } + crypto::public_key key; + }; + + struct TransactionOutputMultisignature { + std::vector keys; + uint32_t requiredSignatures; + + BEGIN_SERIALIZE_OBJECT() + FIELD(keys); + VARINT_FIELD(requiredSignatures); + END_SERIALIZE() + }; + + struct TransactionInputToScript { + BEGIN_SERIALIZE_OBJECT() + END_SERIALIZE() + }; + + struct TransactionInputToScriptHash { + BEGIN_SERIALIZE_OBJECT() + END_SERIALIZE() + }; + + struct TransactionOutputToScript { + BEGIN_SERIALIZE_OBJECT() + END_SERIALIZE() + }; + + struct TransactionOutputToScriptHash { + BEGIN_SERIALIZE_OBJECT() + END_SERIALIZE() + }; + + typedef boost::variant< + TransactionInputGenerate, + TransactionInputToScript, + TransactionInputToScriptHash, + TransactionInputToKey, + TransactionInputMultisignature> TransactionInput; + + typedef boost::variant< + TransactionOutputToScript, + TransactionOutputToScriptHash, + TransactionOutputToKey, + TransactionOutputMultisignature> TransactionOutputTarget; + + struct TransactionOutput { + uint64_t amount; + TransactionOutputTarget target; + + BEGIN_SERIALIZE_OBJECT() + VARINT_FIELD(amount); + FIELD(target); + END_SERIALIZE() + }; + + struct TransactionPrefix { // tx information size_t version; - uint64_t unlock_time; //number of block (or time), used as a limitation like: spend this tx not early then block/time + uint64_t unlockTime; //number of block (or time), used as a limitation like: spend this tx not early then block/time - std::vector vin; - std::vector vout; + std::vector vin; + std::vector vout; //extra std::vector extra; BEGIN_SERIALIZE() - VARINT_FIELD(version) - if(CURRENT_TRANSACTION_VERSION < version) return false; - VARINT_FIELD(unlock_time) - FIELD(vin) - FIELD(vout) - FIELD(extra) + VARINT_FIELD(version); + if(CURRENT_TRANSACTION_VERSION < version) { + return false; + } + VARINT_FIELD(unlockTime); + FIELD(vin); + FIELD(vout); + FIELD(extra); END_SERIALIZE() - protected: - transaction_prefix(){} + TransactionPrefix() {} }; - class transaction: public transaction_prefix - { - public: + struct Transaction: public TransactionPrefix { std::vector > signatures; //count signatures always the same as inputs count - transaction(); - virtual ~transaction(); - void set_null(); + Transaction() { + clear(); + } + + void clear() { + version = 0; + unlockTime = 0; + vin.clear(); + vout.clear(); + extra.clear(); + signatures.clear(); + } BEGIN_SERIALIZE_OBJECT() - FIELDS(*static_cast(this)) + FIELDS(*static_cast(this)) ar.tag("signatures"); ar.begin_array(); PREPARE_CUSTOM_VECTOR_SERIALIZATION(vin.size(), signatures); bool signatures_not_expected = signatures.empty(); - if (!signatures_not_expected && vin.size() != signatures.size()) + if (!signatures_not_expected && vin.size() != signatures.size()) { return false; + } - for (size_t i = 0; i < vin.size(); ++i) - { - size_t signature_size = get_signature_size(vin[i]); - if (signatures_not_expected) - { - if (0 == signature_size) + for (size_t i = 0; i < vin.size(); ++i) { + size_t signatureSize = getSignatureSize(vin[i]); + if (signatures_not_expected) { + if (0 == signatureSize) { continue; - else + } else { return false; + } } - PREPARE_CUSTOM_VECTOR_SERIALIZATION(signature_size, signatures[i]); - if (signature_size != signatures[i].size()) + PREPARE_CUSTOM_VECTOR_SERIALIZATION(signatureSize, signatures[i]); + if (signatureSize != signatures[i].size()) { return false; + } FIELDS(signatures[i]); - if (vin.size() - i > 1) + if (vin.size() - i > 1) { ar.delimit_array(); + } } ar.end_array(); END_SERIALIZE() - private: - static size_t get_signature_size(const txin_v& tx_in); + static size_t getSignatureSize(const TransactionInput& input) { + struct txin_signature_size_visitor : public boost::static_visitor { + size_t operator()(const TransactionInputGenerate& txin) const { return 0; } + size_t operator()(const TransactionInputToScript& txin) const { assert(false); return 0; } + size_t operator()(const TransactionInputToScriptHash& txin) const { assert(false); return 0; } + size_t operator()(const TransactionInputToKey& txin) const { return txin.keyOffsets.size();} + size_t operator()(const TransactionInputMultisignature& txin) const { return txin.signatures; } + }; + + return boost::apply_visitor(txin_signature_size_visitor(), input); + } }; - - inline - transaction::transaction() - { - set_null(); - } - - inline - transaction::~transaction() - { - //set_null(); - } - - inline - void transaction::set_null() - { - version = 0; - unlock_time = 0; - vin.clear(); - vout.clear(); - extra.clear(); - signatures.clear(); - } - - inline - size_t transaction::get_signature_size(const txin_v& tx_in) - { - struct txin_signature_size_visitor : public boost::static_visitor - { - size_t operator()(const txin_gen& txin) const{return 0;} - size_t operator()(const txin_to_script& txin) const{return 0;} - size_t operator()(const txin_to_scripthash& txin) const{return 0;} - size_t operator()(const txin_to_key& txin) const {return txin.key_offsets.size();} - }; - - return boost::apply_visitor(txin_signature_size_visitor(), tx_in); - } - - - - /************************************************************************/ - /* */ - /************************************************************************/ - struct block_header - { - uint8_t major_version; - uint8_t minor_version; - uint64_t timestamp; - crypto::hash prev_id; + struct BlockHeader { + uint8_t majorVersion; + uint8_t minorVersion; uint32_t nonce; + uint64_t timestamp; + crypto::hash prevId; BEGIN_SERIALIZE() - VARINT_FIELD(major_version) - if(major_version > CURRENT_BLOCK_MAJOR_VERSION) return false; - VARINT_FIELD(minor_version) - VARINT_FIELD(timestamp) - FIELD(prev_id) - FIELD(nonce) + VARINT_FIELD(majorVersion) + if (majorVersion > BLOCK_MAJOR_VERSION_1) { + return false; + } + VARINT_FIELD(minorVersion) + VARINT_FIELD(timestamp); + FIELD(prevId); + FIELD(nonce); END_SERIALIZE() }; - struct block: public block_header - { - transaction miner_tx; - std::vector tx_hashes; + struct Block: public BlockHeader { + Transaction minerTx; + std::vector txHashes; BEGIN_SERIALIZE_OBJECT() - FIELDS(*static_cast(this)) - FIELD(miner_tx) - FIELD(tx_hashes) + FIELDS(*static_cast(this)); + FIELD(minerTx); + FIELD(txHashes); END_SERIALIZE() }; - - /************************************************************************/ - /* */ - /************************************************************************/ - struct account_public_address - { - crypto::public_key m_spend_public_key; - crypto::public_key m_view_public_key; + struct AccountPublicAddress { + crypto::public_key m_spendPublicKey; + crypto::public_key m_viewPublicKey; BEGIN_SERIALIZE_OBJECT() - FIELD(m_spend_public_key) - FIELD(m_view_public_key) + FIELD(m_spendPublicKey); + FIELD(m_viewPublicKey); END_SERIALIZE() - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_spend_public_key) - KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_view_public_key) - END_KV_SERIALIZE_MAP() }; - struct keypair - { + struct KeyPair { crypto::public_key pub; crypto::secret_key sec; - static inline keypair generate() - { - keypair k; + static KeyPair generate() { + KeyPair k; generate_keys(k.pub, k.sec); return k; } }; - //--------------------------------------------------------------- - } -BLOB_SERIALIZER(cryptonote::txout_to_key); -BLOB_SERIALIZER(cryptonote::txout_to_scripthash); +BLOB_SERIALIZER(cryptonote::TransactionOutputToKey); -VARIANT_TAG(binary_archive, cryptonote::txin_gen, 0xff); -VARIANT_TAG(binary_archive, cryptonote::txin_to_script, 0x0); -VARIANT_TAG(binary_archive, cryptonote::txin_to_scripthash, 0x1); -VARIANT_TAG(binary_archive, cryptonote::txin_to_key, 0x2); -VARIANT_TAG(binary_archive, cryptonote::txout_to_script, 0x0); -VARIANT_TAG(binary_archive, cryptonote::txout_to_scripthash, 0x1); -VARIANT_TAG(binary_archive, cryptonote::txout_to_key, 0x2); -VARIANT_TAG(binary_archive, cryptonote::transaction, 0xcc); -VARIANT_TAG(binary_archive, cryptonote::block, 0xbb); +VARIANT_TAG(binary_archive, cryptonote::TransactionInputGenerate, 0xff); +VARIANT_TAG(binary_archive, cryptonote::TransactionInputToScript, 0x0); +VARIANT_TAG(binary_archive, cryptonote::TransactionInputToScriptHash, 0x1); +VARIANT_TAG(binary_archive, cryptonote::TransactionInputToKey, 0x2); +VARIANT_TAG(binary_archive, cryptonote::TransactionInputMultisignature, 0x3); +VARIANT_TAG(binary_archive, cryptonote::TransactionOutputToScript, 0x0); +VARIANT_TAG(binary_archive, cryptonote::TransactionOutputToScriptHash, 0x1); +VARIANT_TAG(binary_archive, cryptonote::TransactionOutputToKey, 0x2); +VARIANT_TAG(binary_archive, cryptonote::TransactionOutputMultisignature, 0x3); +VARIANT_TAG(binary_archive, cryptonote::Transaction, 0xcc); +VARIANT_TAG(binary_archive, cryptonote::Block, 0xbb); -VARIANT_TAG(json_archive, cryptonote::txin_gen, "gen"); -VARIANT_TAG(json_archive, cryptonote::txin_to_script, "script"); -VARIANT_TAG(json_archive, cryptonote::txin_to_scripthash, "scripthash"); -VARIANT_TAG(json_archive, cryptonote::txin_to_key, "key"); -VARIANT_TAG(json_archive, cryptonote::txout_to_script, "script"); -VARIANT_TAG(json_archive, cryptonote::txout_to_scripthash, "scripthash"); -VARIANT_TAG(json_archive, cryptonote::txout_to_key, "key"); -VARIANT_TAG(json_archive, cryptonote::transaction, "tx"); -VARIANT_TAG(json_archive, cryptonote::block, "block"); +VARIANT_TAG(json_archive, cryptonote::TransactionInputGenerate, "generate"); +VARIANT_TAG(json_archive, cryptonote::TransactionInputToScript, "script"); +VARIANT_TAG(json_archive, cryptonote::TransactionInputToScriptHash, "scripthash"); +VARIANT_TAG(json_archive, cryptonote::TransactionInputToKey, "key"); +VARIANT_TAG(json_archive, cryptonote::TransactionInputMultisignature, "multisignature"); +VARIANT_TAG(json_archive, cryptonote::TransactionOutputToScript, "script"); +VARIANT_TAG(json_archive, cryptonote::TransactionOutputToScriptHash, "scripthash"); +VARIANT_TAG(json_archive, cryptonote::TransactionOutputToKey, "key"); +VARIANT_TAG(json_archive, cryptonote::TransactionOutputMultisignature, "multisignature"); +VARIANT_TAG(json_archive, cryptonote::Transaction, "Transaction"); +VARIANT_TAG(json_archive, cryptonote::Block, "Block"); -VARIANT_TAG(debug_archive, cryptonote::txin_gen, "gen"); -VARIANT_TAG(debug_archive, cryptonote::txin_to_script, "script"); -VARIANT_TAG(debug_archive, cryptonote::txin_to_scripthash, "scripthash"); -VARIANT_TAG(debug_archive, cryptonote::txin_to_key, "key"); -VARIANT_TAG(debug_archive, cryptonote::txout_to_script, "script"); -VARIANT_TAG(debug_archive, cryptonote::txout_to_scripthash, "scripthash"); -VARIANT_TAG(debug_archive, cryptonote::txout_to_key, "key"); -VARIANT_TAG(debug_archive, cryptonote::transaction, "tx"); -VARIANT_TAG(debug_archive, cryptonote::block, "block"); +VARIANT_TAG(debug_archive, cryptonote::TransactionInputGenerate, "generate"); +VARIANT_TAG(debug_archive, cryptonote::TransactionInputToScript, "script"); +VARIANT_TAG(debug_archive, cryptonote::TransactionInputToScriptHash, "scripthash"); +VARIANT_TAG(debug_archive, cryptonote::TransactionInputToKey, "key"); +VARIANT_TAG(debug_archive, cryptonote::TransactionInputMultisignature, "multisignature"); +VARIANT_TAG(debug_archive, cryptonote::TransactionOutputToScript, "script"); +VARIANT_TAG(debug_archive, cryptonote::TransactionOutputToScriptHash, "scripthash"); +VARIANT_TAG(debug_archive, cryptonote::TransactionOutputToKey, "key"); +VARIANT_TAG(debug_archive, cryptonote::TransactionOutputMultisignature, "multisignature"); +VARIANT_TAG(debug_archive, cryptonote::Transaction, "Transaction"); +VARIANT_TAG(debug_archive, cryptonote::Block, "Block"); diff --git a/src/cryptonote_core/cryptonote_basic_impl.cpp b/src/cryptonote_core/cryptonote_basic_impl.cpp index 0fef9a89..6f5fd66a 100644 --- a/src/cryptonote_core/cryptonote_basic_impl.cpp +++ b/src/cryptonote_core/cryptonote_basic_impl.cpp @@ -1,8 +1,7 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. - #include "include_base_utils.h" using namespace epee; @@ -23,158 +22,78 @@ namespace cryptonote { /* Cryptonote helper functions */ /************************************************************************/ //----------------------------------------------------------------------------------------------- - size_t get_max_block_size() - { - return CRYPTONOTE_MAX_BLOCK_SIZE; - } - //----------------------------------------------------------------------------------------------- - size_t get_max_tx_size() - { - return CRYPTONOTE_MAX_TX_SIZE; - } - //----------------------------------------------------------------------------------------------- - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward) { - uint64_t base_reward = (MONEY_SUPPLY - already_generated_coins) >> EMISSION_SPEED_FACTOR; + uint64_t getPenalizedAmount(uint64_t amount, size_t medianSize, size_t currentBlockSize) { + static_assert(sizeof(size_t) >= sizeof(uint32_t), "size_t is too small"); + assert(currentBlockSize <= 2 * medianSize); + assert(medianSize <= std::numeric_limits::max()); + assert(currentBlockSize <= std::numeric_limits::max()); - //make it soft - if (median_size < CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE) { - median_size = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE; + if (amount == 0) { + return 0; } - if (current_block_size <= median_size) { - reward = base_reward; - return true; + if (currentBlockSize <= medianSize) { + return amount; } - if(current_block_size > 2 * median_size) { - LOG_PRINT_L4("Block cumulative size is too big: " << current_block_size << ", expected less than " << 2 * median_size); + uint64_t productHi; + uint64_t productLo = mul128(amount, currentBlockSize * (UINT64_C(2) * medianSize - currentBlockSize), &productHi); + + uint64_t penalizedAmountHi; + uint64_t penalizedAmountLo; + div128_32(productHi, productLo, static_cast(medianSize), &penalizedAmountHi, &penalizedAmountLo); + div128_32(penalizedAmountHi, penalizedAmountLo, static_cast(medianSize), &penalizedAmountHi, &penalizedAmountLo); + + assert(0 == penalizedAmountHi); + assert(penalizedAmountLo < amount); + + return penalizedAmountLo; + } + //----------------------------------------------------------------------- + std::string getAccountAddressAsStr(uint64_t prefix, const AccountPublicAddress& adr) { + blobdata blob; + bool r = t_serializable_object_to_blob(adr, blob); + assert(r); + return tools::base58::encode_addr(prefix, blob); + } + //----------------------------------------------------------------------- + bool is_coinbase(const Transaction& tx) { + if(tx.vin.size() != 1) { return false; } - assert(median_size < std::numeric_limits::max()); - assert(current_block_size < std::numeric_limits::max()); - - uint64_t product_hi; - uint64_t product_lo = mul128(base_reward, current_block_size * (2 * median_size - current_block_size), &product_hi); - - uint64_t reward_hi; - uint64_t reward_lo; - div128_32(product_hi, product_lo, static_cast(median_size), &reward_hi, &reward_lo); - div128_32(reward_hi, reward_lo, static_cast(median_size), &reward_hi, &reward_lo); - assert(0 == reward_hi); - assert(reward_lo < base_reward); - - reward = reward_lo; - return true; - } - //------------------------------------------------------------------------------------ - uint8_t get_account_address_checksum(const public_address_outer_blob& bl) - { - const unsigned char* pbuf = reinterpret_cast(&bl); - uint8_t summ = 0; - for(size_t i = 0; i!= sizeof(public_address_outer_blob)-1; i++) - summ += pbuf[i]; - - return summ; - } - //----------------------------------------------------------------------- - std::string get_account_address_as_str(const account_public_address& adr) - { - return tools::base58::encode_addr(CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX, t_serializable_object_to_blob(adr)); - } - //----------------------------------------------------------------------- - bool is_coinbase(const transaction& tx) - { - if(tx.vin.size() != 1) - return false; - - if(tx.vin[0].type() != typeid(txin_gen)) - return false; - - return true; - } - //----------------------------------------------------------------------- - bool get_account_address_from_str(uint64_t& prefix, account_public_address& adr, const std::string& str) - { - if (2 * sizeof(public_address_outer_blob) != str.size()) - { - blobdata data; - if (!tools::base58::decode_addr(str, prefix, data)) - { - LOG_PRINT_L1("Invalid address format"); - return false; - } - - if (!::serialization::parse_binary(data, adr)) - { - LOG_PRINT_L1("Account public address keys can't be parsed"); - return false; - } - - if (!crypto::check_key(adr.m_spend_public_key) || !crypto::check_key(adr.m_view_public_key)) - { - LOG_PRINT_L1("Failed to validate address keys"); - return false; - } - } - else - { - // Old address format - prefix = CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; - - std::string buff; - if(!string_tools::parse_hexstr_to_binbuff(str, buff)) - return false; - - if(buff.size()!=sizeof(public_address_outer_blob)) - { - LOG_PRINT_L1("Wrong public address size: " << buff.size() << ", expected size: " << sizeof(public_address_outer_blob)); - return false; - } - - public_address_outer_blob blob = *reinterpret_cast(buff.data()); - - - if(blob.m_ver > CRYPTONOTE_PUBLIC_ADDRESS_TEXTBLOB_VER) - { - LOG_PRINT_L1("Unknown version of public address: " << blob.m_ver << ", expected " << CRYPTONOTE_PUBLIC_ADDRESS_TEXTBLOB_VER); - return false; - } - - if(blob.check_sum != get_account_address_checksum(blob)) - { - LOG_PRINT_L1("Wrong public address checksum"); - return false; - } - - //we success - adr = blob.m_address; - } - - return true; - } - //----------------------------------------------------------------------- - bool get_account_address_from_str(account_public_address& adr, const std::string& str) - { - uint64_t prefix; - if(!get_account_address_from_str(prefix, adr, str)) - return false; - - if(CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX != prefix) - { - LOG_PRINT_L1("Wrong address prefix: " << prefix << ", expected " << CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX); + if(tx.vin[0].type() != typeid(TransactionInputGenerate)) { return false; } return true; } //----------------------------------------------------------------------- + bool parseAccountAddressString(uint64_t& prefix, AccountPublicAddress& adr, const std::string& str) { + blobdata data; + if (!tools::base58::decode_addr(str, prefix, data)) { + LOG_PRINT_L1("Invalid address format"); + return false; + } - bool operator ==(const cryptonote::transaction& a, const cryptonote::transaction& b) { + if (!::serialization::parse_binary(data, adr)) { + LOG_PRINT_L1("Account public address keys can't be parsed"); + return false; + } + + if (!crypto::check_key(adr.m_spendPublicKey) || !crypto::check_key(adr.m_viewPublicKey)) { + LOG_PRINT_L1("Failed to validate address keys"); + return false; + } + + return true; + } + //----------------------------------------------------------------------- + bool operator ==(const cryptonote::Transaction& a, const cryptonote::Transaction& b) { return cryptonote::get_transaction_hash(a) == cryptonote::get_transaction_hash(b); } - - bool operator ==(const cryptonote::block& a, const cryptonote::block& b) { + //----------------------------------------------------------------------- + bool operator ==(const cryptonote::Block& a, const cryptonote::Block& b) { return cryptonote::get_block_hash(a) == cryptonote::get_block_hash(b); } } diff --git a/src/cryptonote_core/cryptonote_basic_impl.h b/src/cryptonote_core/cryptonote_basic_impl.h index 270eff65..e631b517 100644 --- a/src/cryptonote_core/cryptonote_basic_impl.h +++ b/src/cryptonote_core/cryptonote_basic_impl.h @@ -1,12 +1,15 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once -#include "cryptonote_basic.h" +//epee +#include "string_tools.h" + #include "crypto/crypto.h" #include "crypto/hash.h" +#include "cryptonote_core/cryptonote_basic.h" namespace cryptonote { @@ -22,31 +25,16 @@ namespace cryptonote { } }; - -#pragma pack(push, 1) - struct public_address_outer_blob - { - uint8_t m_ver; - account_public_address m_address; - uint8_t check_sum; - }; -#pragma pack (pop) - - /************************************************************************/ /* Cryptonote helper functions */ /************************************************************************/ - size_t get_max_block_size(); - size_t get_max_tx_size(); - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward); - uint8_t get_account_address_checksum(const public_address_outer_blob& bl); - std::string get_account_address_as_str(const account_public_address& adr); - bool get_account_address_from_str(uint64_t& prefix, account_public_address& adr, const std::string& str); - bool get_account_address_from_str(account_public_address& adr, const std::string& str); - bool is_coinbase(const transaction& tx); + uint64_t getPenalizedAmount(uint64_t amount, size_t medianSize, size_t currentBlockSize); + std::string getAccountAddressAsStr(uint64_t prefix, const AccountPublicAddress& adr); + bool parseAccountAddressString(uint64_t& prefix, AccountPublicAddress& adr, const std::string& str); + bool is_coinbase(const Transaction& tx); - bool operator ==(const cryptonote::transaction& a, const cryptonote::transaction& b); - bool operator ==(const cryptonote::block& a, const cryptonote::block& b); + bool operator ==(const cryptonote::Transaction& a, const cryptonote::Transaction& b); + bool operator ==(const cryptonote::Block& a, const cryptonote::Block& b); } template diff --git a/src/cryptonote_core/cryptonote_boost_serialization.h b/src/cryptonote_core/cryptonote_boost_serialization.h index a0a9cd91..8f1f1a5b 100644 --- a/src/cryptonote_core/cryptonote_boost_serialization.h +++ b/src/cryptonote_core/cryptonote_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -54,59 +54,55 @@ namespace boost a & reinterpret_cast(x); } - template - inline void serialize(Archive &a, cryptonote::txout_to_script &x, const boost::serialization::version_type ver) - { - a & x.keys; - a & x.script; + template void serialize(Archive& archive, cryptonote::TransactionInputToScript&, unsigned int version) { + assert(false); } + template void serialize(Archive& archive, cryptonote::TransactionInputToScriptHash&, unsigned int version) { + assert(false); + } + + template void serialize(Archive& archive, cryptonote::TransactionOutputToScript&, unsigned int version) { + assert(false); + } + + template void serialize(Archive& archive, cryptonote::TransactionOutputToScriptHash&, unsigned int version) { + assert(false); + } + + template void serialize(Archive& archive, cryptonote::TransactionInputMultisignature &output, unsigned int version) { + archive & output.amount; + archive & output.signatures; + archive & output.outputIndex; + } + + template void serialize(Archive& archive, cryptonote::TransactionOutputMultisignature &output, unsigned int version) { + archive & output.keys; + archive & output.requiredSignatures; + } template - inline void serialize(Archive &a, cryptonote::txout_to_key &x, const boost::serialization::version_type ver) + inline void serialize(Archive &a, cryptonote::TransactionOutputToKey &x, const boost::serialization::version_type ver) { a & x.key; } template - inline void serialize(Archive &a, cryptonote::txout_to_scripthash &x, const boost::serialization::version_type ver) - { - a & x.hash; - } - - template - inline void serialize(Archive &a, cryptonote::txin_gen &x, const boost::serialization::version_type ver) + inline void serialize(Archive &a, cryptonote::TransactionInputGenerate &x, const boost::serialization::version_type ver) { a & x.height; } template - inline void serialize(Archive &a, cryptonote::txin_to_script &x, const boost::serialization::version_type ver) - { - a & x.prev; - a & x.prevout; - a & x.sigset; - } - - template - inline void serialize(Archive &a, cryptonote::txin_to_scripthash &x, const boost::serialization::version_type ver) - { - a & x.prev; - a & x.prevout; - a & x.script; - a & x.sigset; - } - - template - inline void serialize(Archive &a, cryptonote::txin_to_key &x, const boost::serialization::version_type ver) + inline void serialize(Archive &a, cryptonote::TransactionInputToKey &x, const boost::serialization::version_type ver) { a & x.amount; - a & x.key_offsets; - a & x.k_image; + a & x.keyOffsets; + a & x.keyImage; } template - inline void serialize(Archive &a, cryptonote::tx_out &x, const boost::serialization::version_type ver) + inline void serialize(Archive &a, cryptonote::TransactionOutput &x, const boost::serialization::version_type ver) { a & x.amount; a & x.target; @@ -114,10 +110,10 @@ namespace boost template - inline void serialize(Archive &a, cryptonote::transaction &x, const boost::serialization::version_type ver) + inline void serialize(Archive &a, cryptonote::Transaction &x, const boost::serialization::version_type ver) { a & x.version; - a & x.unlock_time; + a & x.unlockTime; a & x.vin; a & x.vout; a & x.extra; @@ -126,16 +122,16 @@ namespace boost template - inline void serialize(Archive &a, cryptonote::block &b, const boost::serialization::version_type ver) + inline void serialize(Archive &a, cryptonote::Block &b, const boost::serialization::version_type ver) { - a & b.major_version; - a & b.minor_version; + a & b.majorVersion; + a & b.minorVersion; a & b.timestamp; - a & b.prev_id; + a & b.prevId; a & b.nonce; //------------------ - a & b.miner_tx; - a & b.tx_hashes; + a & b.minerTx; + a & b.txHashes; } } } diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp old mode 100644 new mode 100755 index 6065eafd..23ceb2a5 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1,21 +1,31 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "include_base_utils.h" -using namespace epee; - -#include -#include #include "cryptonote_core.h" + +#include +#include + +#include "storages/portable_storage_template_helper.h" +#include "include_base_utils.h" +#include "misc_log_ex.h" +#include "misc_language.h" +#include "warnings.h" + #include "common/command_line.h" #include "common/util.h" -#include "warnings.h" #include "crypto/crypto.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "cryptonote_core/cryptonote_stat_info.h" +#include "cryptonote_core/miner.h" +#include "cryptonote_core/CoreConfig.h" #include "cryptonote_config.h" -#include "cryptonote_format_utils.h" -#include "misc_language.h" +#include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "rpc/core_rpc_server_commands_defs.h" + + +using namespace epee; DISABLE_VS_WARNINGS(4355) @@ -23,15 +33,22 @@ namespace cryptonote { //----------------------------------------------------------------------------------------------- - core::core(i_cryptonote_protocol* pprotocol): - m_mempool(m_blockchain_storage), - m_blockchain_storage(m_mempool), - m_miner(this), - m_miner_address(boost::value_initialized()), + core::core(const Currency& currency, i_cryptonote_protocol* pprotocol): + m_currency(currency), + m_mempool(currency, m_blockchain_storage, m_timeProvider), + m_blockchain_storage(currency, m_mempool), + m_miner(new miner(currency, this)), m_starter_message_showed(false) { set_cryptonote_protocol(pprotocol); + m_blockchain_storage.addObserver(this); + m_mempool.addObserver(this); } + //----------------------------------------------------------------------------------------------- + core::~core() { + m_blockchain_storage.removeObserver(this); + } + //----------------------------------------------------------------------------------------------- void core::set_cryptonote_protocol(i_cryptonote_protocol* pprotocol) { if(pprotocol) @@ -66,21 +83,21 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) + bool core::get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) { return m_blockchain_storage.get_blocks(start_offset, count, blocks, txs); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) + bool core::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) { return m_blockchain_storage.get_blocks(start_offset, count, blocks); } //----------------------------------------------------------------------------------------------- - void core::get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) + void core::get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) { m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- - bool core::get_alternative_blocks(std::list& blocks) + bool core::get_alternative_blocks(std::list& blocks) { return m_blockchain_storage.get_alternative_blocks(blocks); } @@ -90,23 +107,21 @@ namespace cryptonote return m_blockchain_storage.get_alternative_blocks_count(); } //----------------------------------------------------------------------------------------------- - bool core::init(const boost::program_options::variables_map& vm, bool load_existing, bool testnet) - { - bool r = handle_command_line(vm); - - r = m_mempool.init(m_config_folder); + bool core::init(const CoreConfig& config, const MinerConfig& minerConfig, bool load_existing) { + m_config_folder = config.configFolder; + bool r = m_mempool.init(m_config_folder); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); - r = m_blockchain_storage.init(m_config_folder, load_existing, testnet); + r = m_blockchain_storage.init(m_config_folder, load_existing); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage"); - r = m_miner.init(vm); + r = m_miner->init(minerConfig); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage"); return load_state_data(); } //----------------------------------------------------------------------------------------------- - bool core::set_genesis_block(const block& b) + bool core::set_genesis_block(const Block& b) { return m_blockchain_storage.reset_and_set_genesis_block(b); } @@ -119,7 +134,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::deinit() { - m_miner.stop(); + m_miner->stop(); m_mempool.deinit(); m_blockchain_storage.deinit(); return true; @@ -131,7 +146,7 @@ namespace cryptonote //want to process all transactions sequentially CRITICAL_REGION_LOCAL(m_incoming_tx_lock); - if(tx_blob.size() > get_max_tx_size()) + if(tx_blob.size() > m_currency.maxTxSize()) { LOG_PRINT_L0("WRONG TRANSACTION BLOB, too big size " << tx_blob.size() << ", rejected"); tvc.m_verifivation_failed = true; @@ -140,7 +155,7 @@ namespace cryptonote crypto::hash tx_hash = null_hash; crypto::hash tx_prefixt_hash = null_hash; - transaction tx; + Transaction tx; if(!parse_tx_from_blob(tx, tx_hash, tx_prefixt_hash, tx_blob)) { @@ -165,19 +180,27 @@ namespace cryptonote } bool r = add_new_tx(tx, tx_hash, tx_prefixt_hash, tx_blob.size(), tvc, keeped_by_block); - if(tvc.m_verifivation_failed) - {LOG_PRINT_RED_L0("Transaction verification failed: " << tx_hash);} - else if(tvc.m_verifivation_impossible) - {LOG_PRINT_RED_L0("Transaction verification impossible: " << tx_hash);} + if(tvc.m_verifivation_failed) { + if (!tvc.m_tx_fee_too_small) { + LOG_PRINT_RED_L0("Transaction verification failed: " << tx_hash); + } else { + LOG_PRINT_L0("Transaction verification failed: " << tx_hash); + } + } else if(tvc.m_verifivation_impossible) { + LOG_PRINT_RED_L0("Transaction verification impossible: " << tx_hash); + } - if(tvc.m_added_to_pool) + if (tvc.m_added_to_pool) { LOG_PRINT_L1("tx added: " << tx_hash); + poolUpdated(); + } + return r; } //----------------------------------------------------------------------------------------------- bool core::get_stat_info(core_stat_info& st_inf) { - st_inf.mining_speed = m_miner.get_speed(); + st_inf.mining_speed = m_miner->get_speed(); st_inf.alternative_blocks = m_blockchain_storage.get_alternative_blocks_count(); st_inf.blockchain_height = m_blockchain_storage.get_current_blockchain_height(); st_inf.tx_pool_size = m_mempool.get_transactions_count(); @@ -186,7 +209,7 @@ namespace cryptonote } //----------------------------------------------------------------------------------------------- - bool core::check_tx_semantic(const transaction& tx, bool keeped_by_block) + bool core::check_tx_semantic(const Transaction& tx, bool keeped_by_block) { if(!tx.vin.size()) { @@ -222,36 +245,41 @@ namespace cryptonote return false; } - if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_comulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) + if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_comulative_blocksize_limit() - m_currency.minerTxBlobReservedSize()) { - LOG_PRINT_RED_L0("tx have to big size " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_comulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); + LOG_PRINT_RED_L0("transaction is too big " << get_object_blobsize(tx) << ", maximum allowed size is " << + (m_blockchain_storage.get_current_comulative_blocksize_limit() - m_currency.minerTxBlobReservedSize())); return false; } //check if tx use different key images if(!check_tx_inputs_keyimages_diff(tx)) { - LOG_PRINT_RED_L0("tx have to big size " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_comulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); + LOG_PRINT_RED_L0("tx has a few inputs with identical keyimages"); return false; } + if (!checkMultisignatureInputsDiff(tx)) { + LOG_PRINT_RED_L0("tx has a few multisignature inputs with identical output indexes"); + return false; + } return true; } //----------------------------------------------------------------------------------------------- - bool core::check_tx_inputs_keyimages_diff(const transaction& tx) + bool core::check_tx_inputs_keyimages_diff(const Transaction& tx) { std::unordered_set ki; - BOOST_FOREACH(const auto& in, tx.vin) - { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); - if(!ki.insert(tokey_in.k_image).second) - return false; + for (const auto& in : tx.vin) { + if (in.type() == typeid(TransactionInputToKey)) { + if (!ki.insert(boost::get(in).keyImage).second) + return false; + } } return true; } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(const transaction& tx, tx_verification_context& tvc, bool keeped_by_block) + bool core::add_new_tx(const Transaction& tx, tx_verification_context& tvc, bool keeped_by_block) { crypto::hash tx_hash = get_transaction_hash(tx); crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); @@ -270,7 +298,7 @@ namespace cryptonote // return m_blockchain_storage.get_outs(amount, pkeys); //} //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block) { + bool core::add_new_tx(const Transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block) { if (m_blockchain_storage.have_tx(tx_hash)) { LOG_PRINT_L2("tx " << tx_hash << " is already in blockchain"); return true; @@ -287,7 +315,7 @@ namespace cryptonote return m_mempool.add_tx(tx, tx_hash, blob_size, tvc, keeped_by_block); } //----------------------------------------------------------------------------------------------- - bool core::get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) + bool core::get_block_template(Block& b, const AccountPublicAddress& adr, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) { return m_blockchain_storage.create_block_template(b, adr, diffic, height, ex_nonce); } @@ -297,7 +325,7 @@ namespace cryptonote return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); } //----------------------------------------------------------------------------------------------- - bool core::find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) + bool core::find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) { return m_blockchain_storage.find_blockchain_supplement(qblock_ids, blocks, total_height, start_height, max_count); } @@ -327,109 +355,90 @@ namespace cryptonote return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, indexs); } //----------------------------------------------------------------------------------------------- - void core::pause_mine() - { - m_miner.pause(); + void core::pause_mining() { + m_miner->pause(); } //----------------------------------------------------------------------------------------------- - void core::resume_mine() - { - m_miner.resume(); - } - //----------------------------------------------------------------------------------------------- - bool core::handle_block_found(block& b) - { - block_verification_context bvc = boost::value_initialized(); - m_miner.pause(); - m_blockchain_storage.add_new_block(b, bvc); - //anyway - update miner template + void core::update_block_template_and_resume_mining() { update_miner_block_template(); - m_miner.resume(); + m_miner->resume(); + } + //----------------------------------------------------------------------------------------------- + bool core::handle_block_found(Block& b) { + block_verification_context bvc = boost::value_initialized(); + handle_incoming_block(b, bvc, true, true); - - CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "mined block failed verification"); - if(bvc.m_added_to_main_chain) - { - cryptonote_connection_context exclude_context = boost::value_initialized(); - NOTIFY_NEW_BLOCK::request arg = AUTO_VAL_INIT(arg); - arg.hop = 0; - arg.current_blockchain_height = m_blockchain_storage.get_current_blockchain_height(); - std::list missed_txs; - std::list txs; - m_blockchain_storage.get_transactions(b.tx_hashes, txs, missed_txs); - if(missed_txs.size() && m_blockchain_storage.get_block_id_by_height(get_block_height(b)) != get_block_hash(b)) - { - LOG_PRINT_L0("Block found but, seems that reorganize just happened after that, do not relay this block"); - return true; - } - CHECK_AND_ASSERT_MES(txs.size() == b.tx_hashes.size() && !missed_txs.size(), false, "cant find some transactions in found block:" << get_block_hash(b) << " txs.size()=" << txs.size() - << ", b.tx_hashes.size()=" << b.tx_hashes.size() << ", missed_txs.size()" << missed_txs.size()); - - block_to_blob(b, arg.b.block); - //pack transactions - BOOST_FOREACH(auto& tx, txs) - arg.b.txs.push_back(t_serializable_object_to_blob(tx)); - - m_pprotocol->relay_block(arg, exclude_context); + if (bvc.m_verifivation_failed) { + LOG_ERROR("mined block failed verification"); } + return bvc.m_added_to_main_chain; } //----------------------------------------------------------------------------------------------- void core::on_synchronized() { - m_miner.on_synchronized(); + m_miner->on_synchronized(); } - //bool core::get_backward_blocks_sizes(uint64_t from_height, std::vector& sizes, size_t count) - //{ - // return m_blockchain_storage.get_backward_blocks_sizes(from_height, sizes, count); - //} //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate) - { - bvc = boost::value_initialized(); - if(block_blob.size() > get_max_block_size()) - { + bool core::getPoolSymmetricDifference(const std::vector& known_pool_tx_ids, const crypto::hash& known_block_id, bool& isBcActual, std::vector& new_txs, std::vector& deleted_tx_ids) { + isBcActual = m_blockchain_storage.getPoolSymmetricDifference(known_pool_tx_ids, known_block_id, new_txs, deleted_tx_ids); + return true; + } + //----------------------------------------------------------------------------------------------- + bool core::handle_incoming_block_blob(const blobdata& block_blob, block_verification_context& bvc, bool control_miner, bool relay_block) { + if (block_blob.size() > m_currency.maxBlockBlobSize()) { LOG_PRINT_L0("WRONG BLOCK BLOB, too big size " << block_blob.size() << ", rejected"); bvc.m_verifivation_failed = true; return false; } - block b = AUTO_VAL_INIT(b); - if(!parse_and_validate_block_from_blob(block_blob, b)) - { + Block b = AUTO_VAL_INIT(b); + if (!parse_and_validate_block_from_blob(block_blob, b)) { LOG_PRINT_L0("Failed to parse and validate new block"); bvc.m_verifivation_failed = true; return false; } + return handle_incoming_block(b, bvc, control_miner, relay_block); + } + //----------------------------------------------------------------------------------------------- + bool core::handle_incoming_block(const Block& b, block_verification_context& bvc, bool control_miner, bool relay_block) { + if (control_miner) { + pause_mining(); + } + m_blockchain_storage.add_new_block(b, bvc); - if (bvc.m_added_to_main_chain && update_miner_blocktemplate) { - update_miner_block_template(); + + if (control_miner) { + update_block_template_and_resume_mining(); + } + + if (relay_block && bvc.m_added_to_main_chain) { + std::list missed_txs; + std::list txs; + m_blockchain_storage.get_transactions(b.txHashes, txs, missed_txs); + if (!missed_txs.empty() && m_blockchain_storage.get_block_id_by_height(get_block_height(b)) != get_block_hash(b)) { + LOG_PRINT_L0("Block added, but it seems that reorganize just happened after that, do not relay this block"); + } else { + CHECK_AND_ASSERT_MES(txs.size() == b.txHashes.size() && missed_txs.empty(), false, "can't find some transactions in found block:" << + get_block_hash(b) << " txs.size()=" << txs.size() << ", b.txHashes.size()=" << b.txHashes.size() << ", missed_txs.size()" << missed_txs.size()); + + NOTIFY_NEW_BLOCK::request arg = AUTO_VAL_INIT(arg); + arg.hop = 0; + arg.current_blockchain_height = m_blockchain_storage.get_current_blockchain_height(); + bool r = block_to_blob(b, arg.b.block); + CHECK_AND_ASSERT_MES(r, false, "failed to serialize block"); + for (auto& tx : txs) { + arg.b.txs.push_back(t_serializable_object_to_blob(tx)); + } + + cryptonote_connection_context exclude_context = boost::value_initialized(); + m_pprotocol->relay_block(arg, exclude_context); + } } return true; } - - void core::notify_new_block(const block& b) { - cryptonote_connection_context exclude_context = boost::value_initialized(); - NOTIFY_NEW_BLOCK::request arg = AUTO_VAL_INIT(arg); - arg.hop = 0; - arg.current_blockchain_height = m_blockchain_storage.get_current_blockchain_height(); - std::list missed_txs; - std::list txs; - m_blockchain_storage.get_transactions(b.tx_hashes, txs, missed_txs); - if (missed_txs.size() > 0 && m_blockchain_storage.get_block_id_by_height(get_block_height(b)) != get_block_hash(b)) { - LOG_PRINT_L0("Block found but reorganize happened after that, block will not be relayed"); - } else { - if (txs.size() != b.tx_hashes.size() || missed_txs.size()) { - LOG_ERROR("cant find some transactions in found block:" << get_block_hash(b) << " txs.size()=" << txs.size() << ", b.tx_hashes.size()=" << b.tx_hashes.size() << ", missed_txs.size()" << missed_txs.size()); - return; - } - block_to_blob(b, arg.b.block); - BOOST_FOREACH(auto& tx, txs) arg.b.txs.push_back(t_serializable_object_to_blob(tx)); - m_pprotocol->relay_block(arg, exclude_context); - } - } //----------------------------------------------------------------------------------------------- crypto::hash core::get_tail_id() { @@ -446,17 +455,17 @@ namespace cryptonote return m_blockchain_storage.have_block(id); } //----------------------------------------------------------------------------------------------- - bool core::parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash, const blobdata& blob) + bool core::parse_tx_from_blob(Transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash, const blobdata& blob) { return parse_and_validate_tx_from_blob(blob, tx, tx_hash, tx_prefix_hash); } //----------------------------------------------------------------------------------------------- - bool core::check_tx_syntax(const transaction& tx) + bool core::check_tx_syntax(const Transaction& tx) { return true; } //----------------------------------------------------------------------------------------------- - void core::get_pool_transactions(std::list& txs) + void core::get_pool_transactions(std::list& txs) { m_mempool.get_transactions(txs); } @@ -471,12 +480,16 @@ namespace cryptonote return m_blockchain_storage.handle_get_objects(arg, rsp); } //----------------------------------------------------------------------------------------------- + bool core::getBlockByHash(const crypto::hash &h, Block &blk) { + return core::get_block_by_hash(h, blk); + } + crypto::hash core::get_block_id_by_height(uint64_t height) { return m_blockchain_storage.get_block_id_by_height(height); } //----------------------------------------------------------------------------------------------- - bool core::get_block_by_hash(const crypto::hash &h, block &blk) { + bool core::get_block_by_hash(const crypto::hash &h, Block &blk) { return m_blockchain_storage.get_block_by_hash(h, blk); } //----------------------------------------------------------------------------------------------- @@ -491,7 +504,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::update_miner_block_template() { - m_miner.on_block_chain_update(); + m_miner->on_block_chain_update(); return true; } //----------------------------------------------------------------------------------------------- @@ -511,9 +524,94 @@ namespace cryptonote m_starter_message_showed = true; } - m_store_blockchain_interval.do_call(boost::bind(&blockchain_storage::store_blockchain, &m_blockchain_storage)); - m_miner.on_idle(); + m_miner->on_idle(); + m_mempool.on_idle(); return true; } //----------------------------------------------------------------------------------------------- + bool core::addObserver(ICoreObserver* observer) { + return m_observerManager.add(observer); + } + + bool core::removeObserver(ICoreObserver* observer) { + return m_observerManager.remove(observer); + } + + void core::blockchainUpdated() { + m_observerManager.notify(&ICoreObserver::blockchainUpdated); + } + + void core::txDeletedFromPool() { + poolUpdated(); + } + + void core::poolUpdated() { + m_observerManager.notify(&ICoreObserver::poolUpdated); + } + + bool core::queryBlocks(const std::list& knownBlockIds, uint64_t timestamp, + uint64_t& resStartHeight, uint64_t& resCurrentHeight, uint64_t& resFullOffset, std::list& entries) { + + LockedBlockchainStorage lbs(m_blockchain_storage); + + uint64_t currentHeight = lbs->get_current_blockchain_height(); + uint64_t startOffset = 0; + + if (!lbs->find_blockchain_supplement(knownBlockIds, startOffset)) { + return false; + } + + uint64_t startFullOffset = 0; + + if (!lbs->getLowerBound(timestamp, startOffset, startFullOffset)) + startFullOffset = startOffset; + + resFullOffset = startFullOffset; + + if (startOffset != startFullOffset) { + std::list blockIds; + if (!lbs->getBlockIds(startOffset, std::min(uint64_t(BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT), startFullOffset - startOffset), blockIds)) { + return false; + } + + for (const auto& id : blockIds) { + entries.push_back(BlockFullInfo()); + entries.back().block_id = id; + } + } + + auto blocksLeft = std::min(BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT - entries.size(), size_t(BLOCKS_SYNCHRONIZING_DEFAULT_COUNT)); + + if (blocksLeft) { + std::list blocks; + lbs->get_blocks(startFullOffset, blocksLeft, blocks); + + for (auto& b : blocks) { + BlockFullInfo item; + + item.block_id = get_block_hash(b); + + if (b.timestamp >= timestamp) { + // query transactions + std::list txs; + std::list missedTxs; + lbs->get_transactions(b.txHashes, txs, missedTxs); + + // fill data + block_complete_entry& completeEntry = item; + completeEntry.block = block_to_blob(b); + for (auto& tx : txs) { + completeEntry.txs.push_back(tx_to_blob(tx)); + } + } + + entries.push_back(std::move(item)); + } + } + + resCurrentHeight = currentHeight; + resStartHeight = startOffset; + + return true; + } } diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 921f83f2..c7b97ac3 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,78 +9,88 @@ #include "p2p/net_node_common.h" #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" -#include "storages/portable_storage_template_helper.h" +#include "Currency.h" #include "tx_pool.h" #include "blockchain_storage.h" -#include "miner.h" +#include "cryptonote_core/i_miner_handler.h" +#include "cryptonote_core/MinerConfig.h" #include "connection_context.h" -#include "cryptonote_core/cryptonote_stat_info.h" #include "warnings.h" #include "crypto/hash.h" +#include "ICore.h" +#include "ICoreObserver.h" +#include "common/ObserverManager.h" PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) -namespace cryptonote -{ - /************************************************************************/ - /* */ - /************************************************************************/ - class core: public i_miner_handler - { +namespace cryptonote { + struct core_stat_info; + class miner; + class CoreConfig; + + class core : public ICore, public i_miner_handler, public IBlockchainStorageObserver, public ITxPoolObserver { public: - core(i_cryptonote_protocol* pprotocol); - bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote_connection_context& context); + core(const Currency& currency, i_cryptonote_protocol* pprotocol); + ~core(); + bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS_request& arg, NOTIFY_RESPONSE_GET_OBJECTS_request& rsp, cryptonote_connection_context& context); bool on_idle(); - bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block); - bool handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate = true); - i_cryptonote_protocol* get_protocol(){return m_pprotocol;} + virtual bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block); + bool handle_incoming_block_blob(const blobdata& block_blob, block_verification_context& bvc, bool control_miner, bool relay_block); + const Currency& currency() const { return m_currency; } + virtual i_cryptonote_protocol* get_protocol(){return m_pprotocol;} //-------------------- i_miner_handler ----------------------- - virtual bool handle_block_found( block& b); - virtual bool get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce); + virtual bool handle_block_found(Block& b); + virtual bool get_block_template(Block& b, const AccountPublicAddress& adr, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce); + bool addObserver(ICoreObserver* observer); + bool removeObserver(ICoreObserver* observer); - miner& get_miner(){return m_miner;} + miner& get_miner() { return *m_miner; } static void init_options(boost::program_options::options_description& desc); - bool init(const boost::program_options::variables_map& vm, bool load_existing, bool testnet); - bool set_genesis_block(const block& b); + bool init(const CoreConfig& config, const MinerConfig& minerConfig, bool load_existing); + bool set_genesis_block(const Block& b); bool deinit(); uint64_t get_current_blockchain_height(); - bool get_blockchain_top(uint64_t& heeight, crypto::hash& top_id); - bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs); - bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks); + virtual bool get_blockchain_top(uint64_t& heeight, crypto::hash& top_id); + bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs); + bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks); template bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) { return m_blockchain_storage.get_blocks(block_ids, blocks, missed_bs); } + virtual bool queryBlocks(const std::list& block_ids, uint64_t timestamp, + uint64_t& start_height, uint64_t& current_height, uint64_t& full_offset, std::list& entries); crypto::hash get_block_id_by_height(uint64_t height); - void get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs); - bool get_block_by_hash(const crypto::hash &h, block &blk); + void get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs); + bool get_block_by_hash(const crypto::hash &h, Block &blk); //void get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid); - bool get_alternative_blocks(std::list& blocks); + virtual bool getBlockByHash(const crypto::hash &h, Block &blk) override; + + bool get_alternative_blocks(std::list& blocks); size_t get_alternative_blocks_count(); void set_cryptonote_protocol(i_cryptonote_protocol* pprotocol); void set_checkpoints(checkpoints&& chk_pts); - void get_pool_transactions(std::list& txs); + void get_pool_transactions(std::list& txs); size_t get_pool_transactions_count(); size_t get_blockchain_total_transactions(); //bool get_outs(uint64_t amount, std::list& pkeys); bool have_block(const crypto::hash& id); bool get_short_chain_history(std::list& ids); - bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp); - bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count); + virtual bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY_request& resp); + virtual bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count); bool get_stat_info(core_stat_info& st_inf); //bool get_backward_blocks_sizes(uint64_t from_height, std::vector& sizes, size_t count); - bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs); + virtual bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs); crypto::hash get_tail_id(); - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); - void pause_mine(); - void resume_mine(); + virtual bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_response& res); + void pause_mining(); + void update_block_template_and_resume_mining(); blockchain_storage& get_blockchain_storage(){return m_blockchain_storage;} //debug functions void print_blockchain(uint64_t start_index, uint64_t end_index); @@ -88,41 +98,44 @@ namespace cryptonote std::string print_pool(bool short_format); void print_blockchain_outs(const std::string& file); void on_synchronized(); - void notify_new_block(const block& b); + virtual bool getPoolSymmetricDifference(const std::vector& known_pool_tx_ids, const crypto::hash& known_block_id, bool& isBcActual, std::vector& new_txs, std::vector& deleted_tx_ids) override; private: - bool add_new_tx(const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block); - bool add_new_tx(const transaction& tx, tx_verification_context& tvc, bool keeped_by_block); + bool add_new_tx(const Transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block); + bool add_new_tx(const Transaction& tx, tx_verification_context& tvc, bool keeped_by_block); bool load_state_data(); - bool parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash, const blobdata& blob); + bool parse_tx_from_blob(Transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash, const blobdata& blob); + bool handle_incoming_block(const Block& b, block_verification_context& bvc, bool control_miner, bool relay_block); - bool check_tx_syntax(const transaction& tx); + bool check_tx_syntax(const Transaction& tx); //check correct values, amounts and all lightweight checks not related with database - bool check_tx_semantic(const transaction& tx, bool keeped_by_block); + bool check_tx_semantic(const Transaction& tx, bool keeped_by_block); //check if tx already in memory pool or in main blockchain bool is_key_image_spent(const crypto::key_image& key_im); - bool check_tx_ring_signature(const txin_to_key& tx, const crypto::hash& tx_prefix_hash, const std::vector& sig); + bool check_tx_ring_signature(const TransactionInputToKey& tx, const crypto::hash& tx_prefix_hash, const std::vector& sig); bool is_tx_spendtime_unlocked(uint64_t unlock_time); bool update_miner_block_template(); bool handle_command_line(const boost::program_options::variables_map& vm); bool on_update_blocktemplate_interval(); - bool check_tx_inputs_keyimages_diff(const transaction& tx); - + bool check_tx_inputs_keyimages_diff(const Transaction& tx); + virtual void blockchainUpdated() override; + virtual void txDeletedFromPool() override; + void poolUpdated(); + const Currency& m_currency; + CryptoNote::RealTimeProvider m_timeProvider; tx_memory_pool m_mempool; blockchain_storage m_blockchain_storage; i_cryptonote_protocol* m_pprotocol; epee::critical_section m_incoming_tx_lock; - //m_miner and m_miner_addres are probably temporary here - miner m_miner; - account_public_address m_miner_address; + std::unique_ptr m_miner; std::string m_config_folder; cryptonote_protocol_stub m_protocol_stub; - epee::math_helper::once_a_time_seconds<60*60*12, false> m_store_blockchain_interval; friend class tx_validate_inputs; std::atomic m_starter_message_showed; + tools::ObserverManager m_observerManager; }; } diff --git a/src/cryptonote_core/cryptonote_format_utils.cpp b/src/cryptonote_core/cryptonote_format_utils.cpp index e50d7b23..74d5159e 100644 --- a/src/cryptonote_core/cryptonote_format_utils.cpp +++ b/src/cryptonote_core/cryptonote_format_utils.cpp @@ -1,36 +1,41 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "include_base_utils.h" -using namespace epee; +#include + +// epee +#include "include_base_utils.h" +#include "misc_language.h" -#include "cryptonote_format_utils.h" -#include -#include "cryptonote_config.h" -#include "miner.h" #include "crypto/crypto.h" #include "crypto/hash.h" +#include "cryptonote_core/account.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "serialization/binary_utils.h" +#include "cryptonote_config.h" + +using namespace epee; namespace cryptonote { //--------------------------------------------------------------- - void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h) + void get_transaction_prefix_hash(const TransactionPrefix& tx, crypto::hash& h) { std::ostringstream s; binary_archive a(s); - ::serialization::serialize(a, const_cast(tx)); + ::serialization::serialize(a, const_cast(tx)); crypto::cn_fast_hash(s.str().data(), s.str().size(), h); } //--------------------------------------------------------------- - crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx) + crypto::hash get_transaction_prefix_hash(const TransactionPrefix& tx) { crypto::hash h = null_hash; get_transaction_prefix_hash(tx, h); return h; } //--------------------------------------------------------------- - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx) + bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, Transaction& tx) { std::stringstream ss; ss << tx_blob; @@ -40,7 +45,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash) + bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, Transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash) { std::stringstream ss; ss << tx_blob; @@ -54,83 +59,14 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce, size_t max_outs) { - tx.vin.clear(); - tx.vout.clear(); - tx.extra.clear(); - - keypair txkey = keypair::generate(); - add_tx_pub_key_to_extra(tx, txkey.pub); - if(!extra_nonce.empty()) - if(!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) - return false; - - txin_gen in; - in.height = height; - - uint64_t block_reward; - if(!get_block_reward(median_size, current_block_size, already_generated_coins, block_reward)) - { - LOG_PRINT_L0("Block is too big"); - return false; - } -#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - LOG_PRINT_L1("Creating block template: reward " << block_reward << - ", fee " << fee) -#endif - block_reward += fee; - - std::vector out_amounts; - decompose_amount_into_digits(block_reward, DEFAULT_FEE, - [&out_amounts](uint64_t a_chunk) { out_amounts.push_back(a_chunk); }, - [&out_amounts](uint64_t a_dust) { out_amounts.push_back(a_dust); }); - - CHECK_AND_ASSERT_MES(1 <= max_outs, false, "max_out must be non-zero"); - while (max_outs < out_amounts.size()) - { - out_amounts[out_amounts.size() - 2] += out_amounts.back(); - out_amounts.resize(out_amounts.size() - 1); - } - - uint64_t summary_amounts = 0; - for (size_t no = 0; no < out_amounts.size(); no++) - { - crypto::key_derivation derivation = AUTO_VAL_INIT(derivation);; - crypto::public_key out_eph_public_key = AUTO_VAL_INIT(out_eph_public_key); - bool r = crypto::generate_key_derivation(miner_address.m_view_public_key, txkey.sec, derivation); - CHECK_AND_ASSERT_MES(r, false, "while creating outs: failed to generate_key_derivation(" << miner_address.m_view_public_key << ", " << txkey.sec << ")"); - - r = crypto::derive_public_key(derivation, no, miner_address.m_spend_public_key, out_eph_public_key); - CHECK_AND_ASSERT_MES(r, false, "while creating outs: failed to derive_public_key(" << derivation << ", " << no << ", "<< miner_address.m_spend_public_key << ")"); - - txout_to_key tk; - tk.key = out_eph_public_key; - - tx_out out; - summary_amounts += out.amount = out_amounts[no]; - out.target = tk; - tx.vout.push_back(out); - } - - CHECK_AND_ASSERT_MES(summary_amounts == block_reward, false, "Failed to construct miner tx, summary_amounts = " << summary_amounts << " not equal block_reward = " << block_reward); - - tx.version = CURRENT_TRANSACTION_VERSION; - //lock - tx.unlock_time = height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW; - tx.vin.push_back(in); - //LOG_PRINT("MINER_TX generated ok, block_reward=" << print_money(block_reward) << "(" << print_money(block_reward - fee) << "+" << print_money(fee) - // << "), current_block_size=" << current_block_size << ", already_generated_coins=" << already_generated_coins << ", tx_id=" << get_transaction_hash(tx), LOG_LEVEL_2); - return true; - } - //--------------------------------------------------------------- - bool generate_key_image_helper(const account_keys& ack, const crypto::public_key& tx_public_key, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki) + bool generate_key_image_helper(const account_keys& ack, const crypto::public_key& tx_public_key, size_t real_output_index, KeyPair& in_ephemeral, crypto::key_image& ki) { crypto::key_derivation recv_derivation = AUTO_VAL_INIT(recv_derivation); bool r = crypto::generate_key_derivation(tx_public_key, ack.m_view_secret_key, recv_derivation); CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.m_view_secret_key << ")"); - r = crypto::derive_public_key(recv_derivation, real_output_index, ack.m_account_address.m_spend_public_key, in_ephemeral.pub); - CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to derive_public_key(" << recv_derivation << ", " << real_output_index << ", " << ack.m_account_address.m_spend_public_key << ")"); + r = crypto::derive_public_key(recv_derivation, real_output_index, ack.m_account_address.m_spendPublicKey, in_ephemeral.pub); + CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to derive_public_key(" << recv_derivation << ", " << real_output_index << ", " << ack.m_account_address.m_spendPublicKey << ")"); crypto::derive_secret_key(recv_derivation, real_output_index, ack.m_spend_secret_key, in_ephemeral.sec); @@ -148,59 +84,29 @@ namespace cryptonote return total; } //--------------------------------------------------------------- - bool parse_amount(uint64_t& amount, const std::string& str_amount_) - { - std::string str_amount = str_amount_; - boost::algorithm::trim(str_amount); - - size_t point_index = str_amount.find_first_of('.'); - size_t fraction_size; - if (std::string::npos != point_index) - { - fraction_size = str_amount.size() - point_index - 1; - while (CRYPTONOTE_DISPLAY_DECIMAL_POINT < fraction_size && '0' == str_amount.back()) - { - str_amount.erase(str_amount.size() - 1, 1); - --fraction_size; - } - if (CRYPTONOTE_DISPLAY_DECIMAL_POINT < fraction_size) - return false; - str_amount.erase(point_index, 1); - } - else - { - fraction_size = 0; - } - - if (str_amount.empty()) - return false; - - if (fraction_size < CRYPTONOTE_DISPLAY_DECIMAL_POINT) - { - str_amount.append(CRYPTONOTE_DISPLAY_DECIMAL_POINT - fraction_size, '0'); - } - - return string_tools::get_xtype_from_string(amount, str_amount); - } - //--------------------------------------------------------------- - bool get_tx_fee(const transaction& tx, uint64_t & fee) + bool get_tx_fee(const Transaction& tx, uint64_t & fee) { uint64_t amount_in = 0; uint64_t amount_out = 0; - BOOST_FOREACH(auto& in, tx.vin) - { - CHECK_AND_ASSERT_MES(in.type() == typeid(txin_to_key), 0, "unexpected type id in transaction"); - amount_in += boost::get(in).amount; + + for (const auto& in : tx.vin) { + if (in.type() == typeid(TransactionInputToKey)) { + amount_in += boost::get(in).amount; + } else if (in.type() == typeid(TransactionInputMultisignature)) { + amount_in += boost::get(in).amount; + } } - BOOST_FOREACH(auto& o, tx.vout) + + for (const auto& o : tx.vout) { amount_out += o.amount; + } CHECK_AND_ASSERT_MES(amount_in >= amount_out, false, "transaction spend (" < ar(iss); bool eof = false; - while (!eof) - { + while (!eof) { tx_extra_field field; bool r = ::do_serialize(ar, field); - CHECK_AND_NO_ASSERT_MES(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); + if (!r) { + LOG_PRINT_L4("failed to deserialize extra field. extra = " << + string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); + return false; + } tx_extra_fields.push_back(field); std::ios_base::iostate state = iss.rdstate(); eof = (EOF == iss.peek()); iss.clear(state); } - CHECK_AND_NO_ASSERT_MES(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); + + if (!::serialization::check_stream_state(ar)) { + LOG_PRINT_L4("failed to deserialize extra field. extra = " << + string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); + return false; + } return true; } @@ -248,12 +162,12 @@ namespace cryptonote return pub_key_field.pub_key; } //--------------------------------------------------------------- - crypto::public_key get_tx_pub_key_from_extra(const transaction& tx) + crypto::public_key get_tx_pub_key_from_extra(const Transaction& tx) { return get_tx_pub_key_from_extra(tx.extra); } //--------------------------------------------------------------- - bool add_tx_pub_key_to_extra(transaction& tx, const crypto::public_key& tx_pub_key) + bool add_tx_pub_key_to_extra(Transaction& tx, const crypto::public_key& tx_pub_key) { tx.extra.resize(tx.extra.size() + 1 + sizeof(crypto::public_key)); tx.extra[tx.extra.size() - 1 - sizeof(crypto::public_key)] = TX_EXTRA_TAG_PUBKEY; @@ -294,30 +208,80 @@ namespace cryptonote payment_id = *reinterpret_cast(extra_nonce.data() + 1); return true; } - //--------------------------------------------------------------- - bool construct_tx(const account_keys& sender_account_keys, const std::vector& sources, const std::vector& destinations, std::vector extra, transaction& tx, uint64_t unlock_time) + + bool parsePaymentId(const std::string& paymentIdString, crypto::hash& paymentId) { + cryptonote::blobdata binData; + if (!epee::string_tools::parse_hexstr_to_binbuff(paymentIdString, binData)) { + return false; + } + + if (sizeof(crypto::hash) != binData.size()) { + return false; + } + + paymentId = *reinterpret_cast(binData.data()); + return true; + } + + + bool createTxExtraWithPaymentId(const std::string& paymentIdString, std::vector& extra) { + crypto::hash paymentIdBin; + + if (!parsePaymentId(paymentIdString, paymentIdBin)) { + return false; + } + + std::string extraNonce; + cryptonote::set_payment_id_to_tx_extra_nonce(extraNonce, paymentIdBin); + + if (!cryptonote::add_extra_nonce_to_tx_extra(extra, extraNonce)) { + return false; + } + + return true; + } + + bool getPaymentIdFromTxExtra(const std::vector& extra, crypto::hash& paymentId) { + std::vector tx_extra_fields; + if(!parse_tx_extra(extra, tx_extra_fields)) { + return false; + } + + tx_extra_nonce extra_nonce; + if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) { + if (!get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, paymentId)) { + return false; + } + } else { + return false; + } + + return true; + } + + bool construct_tx(const account_keys& sender_account_keys, const std::vector& sources, const std::vector& destinations, std::vector extra, Transaction& tx, uint64_t unlock_time) { tx.vin.clear(); tx.vout.clear(); tx.signatures.clear(); tx.version = CURRENT_TRANSACTION_VERSION; - tx.unlock_time = unlock_time; + tx.unlockTime = unlock_time; tx.extra = extra; - keypair txkey = keypair::generate(); + KeyPair txkey = KeyPair::generate(); add_tx_pub_key_to_extra(tx, txkey.pub); struct input_generation_context_data { - keypair in_ephemeral; + KeyPair in_ephemeral; }; std::vector in_contexts; uint64_t summary_inputs_money = 0; //fill inputs - BOOST_FOREACH(const tx_source_entry& src_entr, sources) + for (const tx_source_entry& src_entr : sources) { if(src_entr.real_output >= src_entr.outputs.size()) { @@ -328,7 +292,7 @@ namespace cryptonote //key_derivation recv_derivation; in_contexts.push_back(input_generation_context_data()); - keypair& in_ephemeral = in_contexts.back().in_ephemeral; + KeyPair& in_ephemeral = in_contexts.back().in_ephemeral; crypto::key_image img; if(!generate_key_image_helper(sender_account_keys, src_entr.real_out_tx_key, src_entr.real_output_in_tx_index, in_ephemeral, img)) return false; @@ -343,15 +307,16 @@ namespace cryptonote } //put key image into tx input - txin_to_key input_to_key; + TransactionInputToKey input_to_key; input_to_key.amount = src_entr.amount; - input_to_key.k_image = img; + input_to_key.keyImage = img; //fill outputs array and use relative offsets - BOOST_FOREACH(const tx_source_entry::output_entry& out_entry, src_entr.outputs) - input_to_key.key_offsets.push_back(out_entry.first); + for (const tx_source_entry::output_entry& out_entry : src_entr.outputs) { + input_to_key.keyOffsets.push_back(out_entry.first); + } - input_to_key.key_offsets = absolute_output_offsets_to_relative(input_to_key.key_offsets); + input_to_key.keyOffsets = absolute_output_offsets_to_relative(input_to_key.keyOffsets); tx.vin.push_back(input_to_key); } @@ -362,20 +327,19 @@ namespace cryptonote uint64_t summary_outs_money = 0; //fill outputs size_t output_index = 0; - BOOST_FOREACH(const tx_destination_entry& dst_entr, shuffled_dsts) - { + for (const tx_destination_entry& dst_entr : shuffled_dsts) { CHECK_AND_ASSERT_MES(dst_entr.amount > 0, false, "Destination with wrong amount: " << dst_entr.amount); crypto::key_derivation derivation; crypto::public_key out_eph_public_key; - bool r = crypto::generate_key_derivation(dst_entr.addr.m_view_public_key, txkey.sec, derivation); - CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << dst_entr.addr.m_view_public_key << ", " << txkey.sec << ")"); + bool r = crypto::generate_key_derivation(dst_entr.addr.m_viewPublicKey, txkey.sec, derivation); + CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << dst_entr.addr.m_viewPublicKey << ", " << txkey.sec << ")"); - r = crypto::derive_public_key(derivation, output_index, dst_entr.addr.m_spend_public_key, out_eph_public_key); - CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to derive_public_key(" << derivation << ", " << output_index << ", "<< dst_entr.addr.m_spend_public_key << ")"); + r = crypto::derive_public_key(derivation, output_index, dst_entr.addr.m_spendPublicKey, out_eph_public_key); + CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to derive_public_key(" << derivation << ", " << output_index << ", "<< dst_entr.addr.m_spendPublicKey << ")"); - tx_out out; + TransactionOutput out; out.amount = dst_entr.amount; - txout_to_key tk; + TransactionOutputToKey tk; tk.key = out_eph_public_key; out.target = tk; tx.vout.push_back(out); @@ -397,12 +361,10 @@ namespace cryptonote std::stringstream ss_ring_s; size_t i = 0; - BOOST_FOREACH(const tx_source_entry& src_entr, sources) - { + for (const tx_source_entry& src_entr : sources) { ss_ring_s << "pub_keys:" << ENDL; std::vector keys_ptrs; - BOOST_FOREACH(const tx_source_entry::output_entry& o, src_entr.outputs) - { + for (const tx_source_entry::output_entry& o : src_entr.outputs) { keys_ptrs.push_back(&o.second); ss_ring_s << o.second << ENDL; } @@ -410,10 +372,12 @@ namespace cryptonote tx.signatures.push_back(std::vector()); std::vector& sigs = tx.signatures.back(); sigs.resize(src_entr.outputs.size()); - crypto::generate_ring_signature(tx_prefix_hash, boost::get(tx.vin[i]).k_image, keys_ptrs, in_contexts[i].in_ephemeral.sec, src_entr.real_output, sigs.data()); + crypto::generate_ring_signature(tx_prefix_hash, boost::get(tx.vin[i]).keyImage, keys_ptrs, + in_contexts[i].in_ephemeral.sec, src_entr.real_output, sigs.data()); ss_ring_s << "signatures:" << ENDL; std::for_each(sigs.begin(), sigs.end(), [&](const crypto::signature& s){ss_ring_s << s << ENDL;}); - ss_ring_s << "prefix_hash:" << tx_prefix_hash << ENDL << "in_ephemeral_key: " << in_contexts[i].in_ephemeral.sec << ENDL << "real_output: " << src_entr.real_output; + ss_ring_s << "prefix_hash:" << tx_prefix_hash << ENDL << "in_ephemeral_key: " << in_contexts[i].in_ephemeral.sec << + ENDL << "real_output: " << src_entr.real_output; i++; } @@ -422,75 +386,118 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool get_inputs_money_amount(const transaction& tx, uint64_t& money) + bool get_inputs_money_amount(const Transaction& tx, uint64_t& money) { money = 0; - BOOST_FOREACH(const auto& in, tx.vin) - { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); - money += tokey_in.amount; + + for (const auto& in : tx.vin) { + uint64_t amount = 0; + + if (in.type() == typeid(TransactionInputToKey)) { + amount = boost::get(in).amount; + } else if (in.type() == typeid(TransactionInputMultisignature)) { + amount = boost::get(in).amount; + } + + money += amount; } return true; } //--------------------------------------------------------------- - uint64_t get_block_height(const block& b) + uint64_t get_block_height(const Block& b) { - CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, 0, "wrong miner tx in block: " << get_block_hash(b) << ", b.miner_tx.vin.size() != 1"); - CHECKED_GET_SPECIFIC_VARIANT(b.miner_tx.vin[0], const txin_gen, coinbase_in, 0); + CHECK_AND_ASSERT_MES(b.minerTx.vin.size() == 1, 0, "wrong miner tx in block: " << get_block_hash(b) << ", b.minerTx.vin.size() != 1"); + CHECKED_GET_SPECIFIC_VARIANT(b.minerTx.vin[0], const TransactionInputGenerate, coinbase_in, 0); return coinbase_in.height; } //--------------------------------------------------------------- - bool check_inputs_types_supported(const transaction& tx) - { - BOOST_FOREACH(const auto& in, tx.vin) - { - CHECK_AND_ASSERT_MES(in.type() == typeid(txin_to_key), false, "wrong variant type: " - << in.type().name() << ", expected " << typeid(txin_to_key).name() - << ", in transaction id=" << get_transaction_hash(tx)); - - } - return true; - } - //----------------------------------------------------------------------------------------------- - bool check_outs_valid(const transaction& tx) - { - BOOST_FOREACH(const tx_out& out, tx.vout) - { - CHECK_AND_ASSERT_MES(out.target.type() == typeid(txout_to_key), false, "wrong variant type: " - << out.target.type().name() << ", expected " << typeid(txout_to_key).name() - << ", in transaction id=" << get_transaction_hash(tx)); - - CHECK_AND_NO_ASSERT_MES(0 < out.amount, false, "zero amount ouput in transaction id=" << get_transaction_hash(tx)); - - if(!check_key(boost::get(out.target).key)) + bool check_inputs_types_supported(const Transaction& tx) { + for (const auto& in : tx.vin) { + if (in.type() != typeid(TransactionInputToKey) && in.type() != typeid(TransactionInputMultisignature)) { + LOG_PRINT_L1("Transaction << " << get_transaction_hash(tx) << " contains inputs with invalid type."); return false; + } } + return true; } //----------------------------------------------------------------------------------------------- - bool check_money_overflow(const transaction& tx) + bool check_outs_valid(const Transaction& tx) { + for (const TransactionOutput& out : tx.vout) { + //assert(out.target.type() == typeid(TransactionOutputToKey) || out.target.type() == typeid(TransactionOutputMultisignature)); + if (out.target.type() == typeid(TransactionOutputToKey)) { + CHECK_AND_NO_ASSERT_MES(0 < out.amount, false, "zero amount ouput in transaction id=" << get_transaction_hash(tx)); + + if (!check_key(boost::get(out.target).key)) { + return false; + } + } else if (out.target.type() == typeid(TransactionOutputMultisignature)) { + const TransactionOutputMultisignature& multisignatureOutput = ::boost::get(out.target); + if (multisignatureOutput.requiredSignatures > multisignatureOutput.keys.size()) { + LOG_PRINT_L1("Transaction << " << get_transaction_hash(tx) << " contains multisignature output with invalid required signature count."); + return false; + } + + for (const crypto::public_key& key : multisignatureOutput.keys) { + if (!check_key(key)) { + LOG_PRINT_L1("Transaction << " << get_transaction_hash(tx) << " contains multisignature output with invalid public keys."); + return false; + } + } + } else { + LOG_PRINT_L1("Transaction << " << get_transaction_hash(tx) << " contains outputs with invalid type."); + return false; + } + } + + return true; + } + + //----------------------------------------------------------------------------------------------- + bool checkMultisignatureInputsDiff(const Transaction& tx) { + std::set> inputsUsage; + for (const auto& inv : tx.vin) { + if (inv.type() == typeid(TransactionInputMultisignature)) { + const TransactionInputMultisignature& in = ::boost::get(inv); + if (!inputsUsage.insert(std::make_pair(in.amount, static_cast(in.outputIndex))).second) { + return false; + } + } + } + return true; + } + + //----------------------------------------------------------------------------------------------- + bool check_money_overflow(const Transaction& tx) { return check_inputs_overflow(tx) && check_outs_overflow(tx); } //--------------------------------------------------------------- - bool check_inputs_overflow(const transaction& tx) + bool check_inputs_overflow(const Transaction& tx) { uint64_t money = 0; - BOOST_FOREACH(const auto& in, tx.vin) - { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); - if(money > tokey_in.amount + money) + + for (const auto& in : tx.vin) { + uint64_t amount = 0; + + if (in.type() == typeid(TransactionInputToKey)) { + amount = boost::get(in).amount; + } else if (in.type() == typeid(TransactionInputMultisignature)) { + amount = boost::get(in).amount; + } + + if (money > amount + money) return false; - money += tokey_in.amount; + + money += amount; } return true; } //--------------------------------------------------------------- - bool check_outs_overflow(const transaction& tx) + bool check_outs_overflow(const Transaction& tx) { uint64_t money = 0; - BOOST_FOREACH(const auto& o, tx.vout) - { + for (const auto& o : tx.vout) { if(money > o.amount + money) return false; money += o.amount; @@ -498,11 +505,12 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - uint64_t get_outs_money_amount(const transaction& tx) + uint64_t get_outs_money_amount(const Transaction& tx) { uint64_t outputs_amount = 0; - BOOST_FOREACH(const auto& o, tx.vout) + for (const auto& o : tx.vout) { outputs_amount += o.amount; + } return outputs_amount; } //--------------------------------------------------------------- @@ -514,17 +522,25 @@ namespace cryptonote res.insert(8, "...."); return res; } + //--------------------------------------------------------------- - bool is_out_to_acc(const account_keys& acc, const txout_to_key& out_key, const crypto::public_key& tx_pub_key, size_t output_index) + bool is_out_to_acc(const account_keys& acc, const TransactionOutputToKey& out_key, const crypto::key_derivation& derivation, size_t keyIndex) + { + crypto::public_key pk; + derive_public_key(derivation, keyIndex, acc.m_account_address.m_spendPublicKey, pk); + return pk == out_key.key; + } + + //--------------------------------------------------------------- + bool is_out_to_acc(const account_keys& acc, const TransactionOutputToKey& out_key, const crypto::public_key& tx_pub_key, size_t keyIndex) { crypto::key_derivation derivation; generate_key_derivation(tx_pub_key, acc.m_view_secret_key, derivation); - crypto::public_key pk; - derive_public_key(derivation, output_index, acc.m_account_address.m_spend_public_key, pk); - return pk == out_key.key; + return is_out_to_acc(acc, out_key, derivation, keyIndex); } + //--------------------------------------------------------------- - bool lookup_acc_outs(const account_keys& acc, const transaction& tx, std::vector& outs, uint64_t& money_transfered) + bool lookup_acc_outs(const account_keys& acc, const Transaction& tx, std::vector& outs, uint64_t& money_transfered) { crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); if(null_pkey == tx_pub_key) @@ -532,19 +548,29 @@ namespace cryptonote return lookup_acc_outs(acc, tx, tx_pub_key, outs, money_transfered); } //--------------------------------------------------------------- - bool lookup_acc_outs(const account_keys& acc, const transaction& tx, const crypto::public_key& tx_pub_key, std::vector& outs, uint64_t& money_transfered) + bool lookup_acc_outs(const account_keys& acc, const Transaction& tx, const crypto::public_key& tx_pub_key, std::vector& outs, uint64_t& money_transfered) { money_transfered = 0; - size_t i = 0; - BOOST_FOREACH(const tx_out& o, tx.vout) - { - CHECK_AND_ASSERT_MES(o.target.type() == typeid(txout_to_key), false, "wrong type id in transaction out" ); - if(is_out_to_acc(acc, boost::get(o.target), tx_pub_key, i)) - { - outs.push_back(i); - money_transfered += o.amount; + size_t keyIndex = 0; + size_t outputIndex = 0; + + crypto::key_derivation derivation; + generate_key_derivation(tx_pub_key, acc.m_view_secret_key, derivation); + + for (const TransactionOutput& o : tx.vout) { + assert(o.target.type() == typeid(TransactionOutputToKey) || o.target.type() == typeid(TransactionOutputMultisignature)); + if (o.target.type() == typeid(TransactionOutputToKey)) { + if (is_out_to_acc(acc, boost::get(o.target), derivation, keyIndex)) { + outs.push_back(outputIndex); + money_transfered += o.amount; + } + + ++keyIndex; + } else if (o.target.type() == typeid(TransactionOutputMultisignature)) { + keyIndex += boost::get(o.target).keys.size(); } - i++; + + ++outputIndex; } return true; } @@ -554,17 +580,6 @@ namespace cryptonote cn_fast_hash(blob.data(), blob.size(), res); } //--------------------------------------------------------------- - std::string print_money(uint64_t amount) - { - std::string s = std::to_string(amount); - if(s.size() < CRYPTONOTE_DISPLAY_DECIMAL_POINT+1) - { - s.insert(0, CRYPTONOTE_DISPLAY_DECIMAL_POINT+1 - s.size(), '0'); - } - s.insert(s.size() - CRYPTONOTE_DISPLAY_DECIMAL_POINT, "."); - return s; - } - //--------------------------------------------------------------- crypto::hash get_blob_hash(const blobdata& blob) { crypto::hash h = null_hash; @@ -572,7 +587,7 @@ namespace cryptonote return h; } //--------------------------------------------------------------- - crypto::hash get_transaction_hash(const transaction& t) + crypto::hash get_transaction_hash(const Transaction& t) { crypto::hash h = null_hash; size_t blob_size = 0; @@ -580,95 +595,58 @@ namespace cryptonote return h; } //--------------------------------------------------------------- - bool get_transaction_hash(const transaction& t, crypto::hash& res) + bool get_transaction_hash(const Transaction& t, crypto::hash& res) { size_t blob_size = 0; return get_object_hash(t, res, blob_size); } //--------------------------------------------------------------- - bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size) + bool get_transaction_hash(const Transaction& t, crypto::hash& res, size_t& blob_size) { return get_object_hash(t, res, blob_size); } //--------------------------------------------------------------- - bool get_block_hashing_blob(const block& b, blobdata& blob) - { - if(!t_serializable_object_to_blob(static_cast(b), blob)) + bool get_block_hashing_blob(const Block& b, blobdata& blob) { + if (!t_serializable_object_to_blob(static_cast(b), blob)) { return false; + } crypto::hash tree_root_hash = get_tx_tree_hash(b); blob.append(reinterpret_cast(&tree_root_hash), sizeof(tree_root_hash)); - blob.append(tools::get_varint_data(b.tx_hashes.size() + 1)); + blob.append(tools::get_varint_data(b.txHashes.size() + 1)); return true; } //--------------------------------------------------------------- - bool get_block_hash(const block& b, crypto::hash& res) - { + bool get_block_hash(const Block& b, crypto::hash& res) { blobdata blob; - if (!get_block_hashing_blob(b, blob)) + if (!get_block_hashing_blob(b, blob)) { return false; + } + return get_object_hash(blob, res); } //--------------------------------------------------------------- - crypto::hash get_block_hash(const block& b) - { + crypto::hash get_block_hash(const Block& b) { crypto::hash p = null_hash; get_block_hash(b, p); return p; } //--------------------------------------------------------------- - void generate_genesis_tx(transaction& tx) { - account_public_address ac = boost::value_initialized(); - std::vector sz; - construct_miner_tx(0, 0, 0, 0, 0, ac, tx); // zero fee in genesis - blobdata txb = tx_to_blob(tx); - } - - std::string get_genesis_tx_hex() { - transaction tx; - - generate_genesis_tx(tx); - blobdata txb = tx_to_blob(tx); - std::string hex_tx_represent = string_tools::buff_to_hex_nodelimer(txb); - - return hex_tx_represent; - } - - bool generateGenesisBlock(block& bl) - { - //genesis block - bl = boost::value_initialized(); - - //hard code coinbase tx in genesis block, because "tru" generating tx use random, but genesis should be always the same - std::string genesis_coinbase_tx_hex = GENESIS_COINBASE_TX_HEX; - - blobdata tx_bl; - string_tools::parse_hexstr_to_binbuff(genesis_coinbase_tx_hex, tx_bl); - bool r = parse_and_validate_tx_from_blob(tx_bl, bl.miner_tx); - CHECK_AND_ASSERT_MES(r, false, "failed to parse coinbase tx from hard coded blob"); - bl.major_version = CURRENT_BLOCK_MAJOR_VERSION; - bl.minor_version = CURRENT_BLOCK_MINOR_VERSION; - bl.timestamp = 0; - bl.nonce = 70; - //miner::find_nonce_for_given_block(bl, 1, 0); - return true; - } - - bool generateTestnetGenesisBlock(cryptonote::block& b) { - if (!generateGenesisBlock(b)) { + bool get_aux_block_header_hash(const Block& b, crypto::hash& res) { + blobdata blob; + if (!get_block_hashing_blob(b, blob)) { return false; } - b.nonce += 1; - return true; + return get_object_hash(blob, res); } - //--------------------------------------------------------------- - bool get_block_longhash(crypto::cn_context &context, const block& b, crypto::hash& res, uint64_t height) - { + bool get_block_longhash(crypto::cn_context &context, const Block& b, crypto::hash& res) { blobdata bd; - if(!get_block_hashing_blob(b, bd)) + if (!get_block_hashing_blob(b, bd)) { return false; + } + crypto::cn_slow_hash(context, bd.data(), bd.size(), res); return true; } @@ -693,14 +671,7 @@ namespace cryptonote return res; } //--------------------------------------------------------------- - crypto::hash get_block_longhash(crypto::cn_context &context, const block& b, uint64_t height) - { - crypto::hash p = null_hash; - get_block_longhash(context, b, p, height); - return p; - } - //--------------------------------------------------------------- - bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b) + bool parse_and_validate_block_from_blob(const blobdata& b_blob, Block& b) { std::stringstream ss; ss << b_blob; @@ -710,22 +681,22 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - blobdata block_to_blob(const block& b) + blobdata block_to_blob(const Block& b) { return t_serializable_object_to_blob(b); } //--------------------------------------------------------------- - bool block_to_blob(const block& b, blobdata& b_blob) + bool block_to_blob(const Block& b, blobdata& b_blob) { return t_serializable_object_to_blob(b, b_blob); } //--------------------------------------------------------------- - blobdata tx_to_blob(const transaction& tx) + blobdata tx_to_blob(const Transaction& tx) { return t_serializable_object_to_blob(tx); } //--------------------------------------------------------------- - bool tx_to_blob(const transaction& tx, blobdata& b_blob) + bool tx_to_blob(const Transaction& tx, blobdata& b_blob) { return t_serializable_object_to_blob(tx, b_blob); } @@ -742,15 +713,16 @@ namespace cryptonote return h; } //--------------------------------------------------------------- - crypto::hash get_tx_tree_hash(const block& b) + crypto::hash get_tx_tree_hash(const Block& b) { std::vector txs_ids; crypto::hash h = null_hash; size_t bl_sz = 0; - get_transaction_hash(b.miner_tx, h, bl_sz); + get_transaction_hash(b.minerTx, h, bl_sz); txs_ids.push_back(h); - BOOST_FOREACH(auto& th, b.tx_hashes) + for (auto& th : b.txHashes) { txs_ids.push_back(th); + } return get_tx_tree_hash(txs_ids); } //--------------------------------------------------------------- diff --git a/src/cryptonote_core/cryptonote_format_utils.h b/src/cryptonote_core/cryptonote_format_utils.h index c70d920d..e961219a 100644 --- a/src/cryptonote_core/cryptonote_format_utils.h +++ b/src/cryptonote_core/cryptonote_format_utils.h @@ -1,24 +1,29 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once -#include "cryptonote_protocol/cryptonote_protocol_defs.h" -#include "cryptonote_core/cryptonote_basic_impl.h" -#include "account.h" + +#include + +#include + #include "include_base_utils.h" + #include "crypto/crypto.h" #include "crypto/hash.h" +#include "cryptonote_core/cryptonote_basic_impl.h" +#include "cryptonote_core/difficulty.h" +#include "cryptonote_protocol/blobdatatype.h" namespace cryptonote { //--------------------------------------------------------------- - void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h); - crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx); - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash); - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx); - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 1); + void get_transaction_prefix_hash(const TransactionPrefix& tx, crypto::hash& h); + crypto::hash get_transaction_prefix_hash(const TransactionPrefix& tx); + bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, Transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash); + bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, Transaction& tx); struct tx_source_entry { @@ -34,14 +39,14 @@ namespace cryptonote struct tx_destination_entry { uint64_t amount; //money - account_public_address addr; //destination address + AccountPublicAddress addr; //destination address - tx_destination_entry() : amount(0), addr(AUTO_VAL_INIT(addr)) { } - tx_destination_entry(uint64_t a, const account_public_address &ad) : amount(a), addr(ad) { } + tx_destination_entry() : amount(0), addr(boost::value_initialized()) { } + tx_destination_entry(uint64_t a, const AccountPublicAddress &ad) : amount(a), addr(ad) { } }; //--------------------------------------------------------------- - bool construct_tx(const account_keys& sender_account_keys, const std::vector& sources, const std::vector& destinations, std::vector extra, transaction& tx, uint64_t unlock_time); + bool construct_tx(const account_keys& sender_account_keys, const std::vector& sources, const std::vector& destinations, std::vector extra, Transaction& tx, uint64_t unlock_time); template bool find_tx_extra_field_by_type(const std::vector& tx_extra_fields, T& field) @@ -56,46 +61,47 @@ namespace cryptonote bool parse_tx_extra(const std::vector& tx_extra, std::vector& tx_extra_fields); crypto::public_key get_tx_pub_key_from_extra(const std::vector& tx_extra); - crypto::public_key get_tx_pub_key_from_extra(const transaction& tx); - bool add_tx_pub_key_to_extra(transaction& tx, const crypto::public_key& tx_pub_key); + crypto::public_key get_tx_pub_key_from_extra(const Transaction& tx); + bool add_tx_pub_key_to_extra(Transaction& tx, const crypto::public_key& tx_pub_key); bool add_extra_nonce_to_tx_extra(std::vector& tx_extra, const blobdata& extra_nonce); void set_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash& payment_id); bool get_payment_id_from_tx_extra_nonce(const blobdata& extra_nonce, crypto::hash& payment_id); - bool is_out_to_acc(const account_keys& acc, const txout_to_key& out_key, const crypto::public_key& tx_pub_key, size_t output_index); - bool lookup_acc_outs(const account_keys& acc, const transaction& tx, const crypto::public_key& tx_pub_key, std::vector& outs, uint64_t& money_transfered); - bool lookup_acc_outs(const account_keys& acc, const transaction& tx, std::vector& outs, uint64_t& money_transfered); - bool get_tx_fee(const transaction& tx, uint64_t & fee); - uint64_t get_tx_fee(const transaction& tx); - bool generate_key_image_helper(const account_keys& ack, const crypto::public_key& tx_public_key, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki); + bool is_out_to_acc(const account_keys& acc, const TransactionOutputToKey& out_key, const crypto::public_key& tx_pub_key, size_t keyIndex); + bool is_out_to_acc(const account_keys& acc, const TransactionOutputToKey& out_key, const crypto::key_derivation& derivation, size_t keyIndex); + bool lookup_acc_outs(const account_keys& acc, const Transaction& tx, const crypto::public_key& tx_pub_key, std::vector& outs, uint64_t& money_transfered); + bool lookup_acc_outs(const account_keys& acc, const Transaction& tx, std::vector& outs, uint64_t& money_transfered); + bool get_tx_fee(const Transaction& tx, uint64_t & fee); + uint64_t get_tx_fee(const Transaction& tx); + bool generate_key_image_helper(const account_keys& ack, const crypto::public_key& tx_public_key, size_t real_output_index, KeyPair& in_ephemeral, crypto::key_image& ki); void get_blob_hash(const blobdata& blob, crypto::hash& res); crypto::hash get_blob_hash(const blobdata& blob); std::string short_hash_str(const crypto::hash& h); + bool createTxExtraWithPaymentId(const std::string& paymentIdString, std::vector& extra); + //returns false if payment id is not found or parse error + bool getPaymentIdFromTxExtra(const std::vector& extra, crypto::hash& paymentId); + bool parsePaymentId(const std::string& paymentIdString, crypto::hash& paymentId); - crypto::hash get_transaction_hash(const transaction& t); - bool get_transaction_hash(const transaction& t, crypto::hash& res); - bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size); - bool get_block_hashing_blob(const block& b, blobdata& blob); - bool get_block_hash(const block& b, crypto::hash& res); - crypto::hash get_block_hash(const block& b); - bool get_block_longhash(crypto::cn_context &context, const block& b, crypto::hash& res, uint64_t height); - crypto::hash get_block_longhash(crypto::cn_context &context, const block& b, uint64_t height); - bool generateGenesisBlock(block& bl); - bool generateTestnetGenesisBlock(block& bl); - std::string get_genesis_tx_hex(); - bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b); - bool get_inputs_money_amount(const transaction& tx, uint64_t& money); - uint64_t get_outs_money_amount(const transaction& tx); - bool check_inputs_types_supported(const transaction& tx); - bool check_outs_valid(const transaction& tx); - bool parse_amount(uint64_t& amount, const std::string& str_amount); + crypto::hash get_transaction_hash(const Transaction& t); + bool get_transaction_hash(const Transaction& t, crypto::hash& res); + bool get_transaction_hash(const Transaction& t, crypto::hash& res, size_t& blob_size); + bool get_block_hashing_blob(const Block& b, blobdata& blob); + bool get_aux_block_header_hash(const Block& b, crypto::hash& res); + bool get_block_hash(const Block& b, crypto::hash& res); + crypto::hash get_block_hash(const Block& b); + bool get_block_longhash(crypto::cn_context &context, const Block& b, crypto::hash& res); + bool parse_and_validate_block_from_blob(const blobdata& b_blob, Block& b); + bool get_inputs_money_amount(const Transaction& tx, uint64_t& money); + uint64_t get_outs_money_amount(const Transaction& tx); + bool check_inputs_types_supported(const Transaction& tx); + bool check_outs_valid(const Transaction& tx); + bool checkMultisignatureInputsDiff(const Transaction& tx); - bool check_money_overflow(const transaction& tx); - bool check_outs_overflow(const transaction& tx); - bool check_inputs_overflow(const transaction& tx); - uint64_t get_block_height(const block& b); + bool check_money_overflow(const Transaction& tx); + bool check_outs_overflow(const Transaction& tx); + bool check_inputs_overflow(const Transaction& tx); + uint64_t get_block_height(const Block& b); std::vector relative_output_offsets_to_absolute(const std::vector& off); std::vector absolute_output_offsets_to_relative(const std::vector& off); - std::string print_money(uint64_t amount); //--------------------------------------------------------------- template bool t_serializable_object_to_blob(const t_object& to, blobdata& b_blob) @@ -123,10 +129,22 @@ namespace cryptonote } //--------------------------------------------------------------- template + bool get_object_blobsize(const t_object& o, size_t& size) { + blobdata blob; + if (!t_serializable_object_to_blob(o, blob)) { + size = (std::numeric_limits::max)(); + return false; + } + size = blob.size(); + return true; + } + //--------------------------------------------------------------- + template size_t get_object_blobsize(const t_object& o) { - blobdata b = t_serializable_object_to_blob(o); - return b.size(); + size_t size; + get_object_blobsize(o, size); + return size; } //--------------------------------------------------------------- template @@ -139,11 +157,10 @@ namespace cryptonote } //--------------------------------------------------------------- template - std::string obj_to_json_str(T& obj) - { + std::string obj_to_json_str(const T& obj) { std::stringstream ss; json_archive ar(ss, true); - bool r = ::serialization::serialize(ar, obj); + bool r = ::serialization::serialize(ar, *const_cast(&obj)); CHECK_AND_ASSERT_MES(r, "", "obj_to_json_str failed: serialization::serialize returned false"); return ss.str(); } @@ -190,13 +207,13 @@ namespace cryptonote } } //--------------------------------------------------------------- - blobdata block_to_blob(const block& b); - bool block_to_blob(const block& b, blobdata& b_blob); - blobdata tx_to_blob(const transaction& b); - bool tx_to_blob(const transaction& b, blobdata& b_blob); + blobdata block_to_blob(const Block& b); + bool block_to_blob(const Block& b, blobdata& b_blob); + blobdata tx_to_blob(const Transaction& b); + bool tx_to_blob(const Transaction& b, blobdata& b_blob); void get_tx_tree_hash(const std::vector& tx_hashes, crypto::hash& h); crypto::hash get_tx_tree_hash(const std::vector& tx_hashes); - crypto::hash get_tx_tree_hash(const block& b); + crypto::hash get_tx_tree_hash(const Block& b); #define CHECKED_GET_SPECIFIC_VARIANT(variant_var, specific_type, variable_name, fail_return_val) \ CHECK_AND_ASSERT_MES(variant_var.type() == typeid(specific_type), fail_return_val, "wrong variant type: " << variant_var.type().name() << ", expected " << typeid(specific_type).name()); \ diff --git a/src/cryptonote_core/cryptonote_serialization.cpp b/src/cryptonote_core/cryptonote_serialization.cpp new file mode 100644 index 00000000..1c08cd64 --- /dev/null +++ b/src/cryptonote_core/cryptonote_serialization.cpp @@ -0,0 +1,369 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "cryptonote_serialization.h" + +#include "serialization/ISerializer.h" +#include "serialization/SerializationOverloads.h" +#include "serialization/BinaryInputStreamSerializer.h" +#include "serialization/BinaryOutputStreamSerializer.h" +#include "crypto/crypto.h" +#include "cryptonote_config.h" + +#include +#include + +#include +#include + +namespace { + +struct BinaryVariantTagGetter: boost::static_visitor { + uint8_t operator()(const cryptonote::TransactionInputGenerate) { return 0xff; } + uint8_t operator()(const cryptonote::TransactionInputToScript) { return 0x0; } + uint8_t operator()(const cryptonote::TransactionInputToScriptHash) { return 0x1; } + uint8_t operator()(const cryptonote::TransactionInputToKey) { return 0x2; } + uint8_t operator()(const cryptonote::TransactionInputMultisignature) { return 0x3; } + uint8_t operator()(const cryptonote::TransactionOutputToScript) { return 0x0; } + uint8_t operator()(const cryptonote::TransactionOutputToScriptHash) { return 0x1; } + uint8_t operator()(const cryptonote::TransactionOutputToKey) { return 0x2; } + uint8_t operator()(const cryptonote::TransactionOutputMultisignature) { return 0x3; } + uint8_t operator()(const cryptonote::Transaction) { return 0xcc; } + uint8_t operator()(const cryptonote::Block) { return 0xbb; } +}; + +struct VariantSerializer : boost::static_visitor<> { + VariantSerializer(cryptonote::ISerializer& serializer, const std::string& name) : s(serializer), name(name) {} + + void operator() (cryptonote::TransactionInputGenerate& param) { s(param, name); } + void operator() (cryptonote::TransactionInputToScript& param) { s(param, name); } + void operator() (cryptonote::TransactionInputToScriptHash& param) { s(param, name); } + void operator() (cryptonote::TransactionInputToKey& param) { s(param, name); } + void operator() (cryptonote::TransactionInputMultisignature& param) { s(param, name); } + void operator() (cryptonote::TransactionOutputToScript& param) { s(param, name); } + void operator() (cryptonote::TransactionOutputToScriptHash& param) { s(param, name); } + void operator() (cryptonote::TransactionOutputToKey& param) { s(param, name); } + void operator() (cryptonote::TransactionOutputMultisignature& param) { s(param, name); } + + cryptonote::ISerializer& s; + const std::string& name; +}; + +void getVariantValue(cryptonote::ISerializer& serializer, uint8_t tag, cryptonote::TransactionInput& in) { + switch(tag) { + case 0xff: { + cryptonote::TransactionInputGenerate v; + serializer(v, "data"); + in = v; + break; + } + case 0x0: { + cryptonote::TransactionInputToScript v; + serializer(v, "data"); + in = v; + break; + } + case 0x1: { + cryptonote::TransactionInputToScriptHash v; + serializer(v, "data"); + in = v; + break; + } + case 0x2: { + cryptonote::TransactionInputToKey v; + serializer(v, "data"); + in = v; + break; + } + case 0x3: { + cryptonote::TransactionInputMultisignature v; + serializer(v, "data"); + in = v; + break; + } + default: + throw std::runtime_error("Unknown variant tag"); + } +} + +void getVariantValue(cryptonote::ISerializer& serializer, uint8_t tag, cryptonote::TransactionOutputTarget& out) { + switch(tag) { + case 0x0: { + cryptonote::TransactionOutputToScript v; + serializer(v, "data"); + out = v; + break; + } + case 0x1: { + cryptonote::TransactionOutputToScriptHash v; + serializer(v, "data"); + out = v; + break; + } + case 0x2: { + cryptonote::TransactionOutputToKey v; + serializer(v, "data"); + out = v; + break; + } + case 0x3: { + cryptonote::TransactionOutputMultisignature v; + serializer(v, "data"); + out = v; + break; + } + default: + throw std::runtime_error("Unknown variant tag"); + } +} + +template +void serializePod(T& v, const std::string& name, cryptonote::ISerializer& serializer) { + serializer.binary(&v, sizeof(v), name); +} + +void serializeVarintVector(std::vector& vector, cryptonote::ISerializer& serializer, const std::string& name) { + std::size_t size = vector.size(); + serializer.beginArray(size, name); + vector.resize(size); + + for (size_t i = 0; i < size; ++i) { + serializer(vector[i], ""); + } + + serializer.endArray(); +} + +} + +namespace crypto { + +void serialize(public_key& pubKey, const std::string& name, cryptonote::ISerializer& serializer) { + serializePod(pubKey, name, serializer); +} + +void serialize(secret_key& secKey, const std::string& name, cryptonote::ISerializer& serializer) { + serializePod(secKey, name, serializer); +} + +void serialize(hash& h, const std::string& name, cryptonote::ISerializer& serializer) { + serializePod(h, name, serializer); +} + +void serialize(key_image& keyImage, const std::string& name, cryptonote::ISerializer& serializer) { + serializePod(keyImage, name, serializer); +} + +void serialize(chacha8_iv& chacha, const std::string& name, cryptonote::ISerializer& serializer) { + serializePod(chacha, name, serializer); +} + +} + +namespace cryptonote { + +void serialize(TransactionPrefix& txP, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + uint64_t version = static_cast(txP.version); + serializer(version, "version"); + txP.version = static_cast(version); + serializer(txP.unlockTime, "unlock_time"); + serializer(txP.vin, "vin"); + serializer(txP.vout, "vout"); + serializeAsBinary(txP.extra, "extra", serializer); + serializer.endObject(); +} + +void serialize(Transaction& tx, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + + uint64_t version = static_cast(tx.version); + serializer(version, "version"); + tx.version = static_cast(version); + //TODO: make version. check version here + serializer(tx.unlockTime, "unlock_time"); + serializer(tx.vin, "vin"); + serializer(tx.vout, "vout"); + serializeAsBinary(tx.extra, "extra", serializer); + + std::size_t sigSize = tx.vin.size(); + //TODO: make arrays without sizes +// serializer.beginArray(sigSize, "signatures"); + tx.signatures.resize(sigSize); + + bool signaturesNotExpected = tx.signatures.empty(); + if (!signaturesNotExpected && tx.vin.size() != tx.signatures.size()) { + throw std::runtime_error("Serialization error: unexpected signatures size"); + } + + for (size_t i = 0; i < tx.vin.size(); ++i) { + size_t signatureSize = Transaction::getSignatureSize(tx.vin[i]); + if (signaturesNotExpected) { + if (signatureSize == 0) { + continue; + } else { + throw std::runtime_error("Serialization error: signatures are not expected"); + } + } + + if (serializer.type() == ISerializer::OUTPUT) { + if (signatureSize != tx.signatures[i].size()) { + throw std::runtime_error("Serialization error: unexpected signatures size"); + } + } else { + tx.signatures[i].resize(signatureSize); + } + + for (crypto::signature& sig: tx.signatures[i]) { + serializePod(sig, "", serializer); + } + } +// serializer.endArray(); + + serializer.endObject(); +} + +void serialize(TransactionInput& in, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + + if (serializer.type() == ISerializer::OUTPUT) { + BinaryVariantTagGetter tagGetter; + uint8_t tag = boost::apply_visitor(tagGetter, in); + serializer.binary(&tag, sizeof(tag), "type"); + + VariantSerializer visitor(serializer, "value"); + boost::apply_visitor(visitor, in); + } else { + uint8_t tag; + serializer.binary(&tag, sizeof(tag), "type"); + + getVariantValue(serializer, tag, in); + } + + serializer.endObject(); +} + +void serialize(TransactionInputGenerate& gen, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + uint64_t height = static_cast(gen.height); + serializer(height, "height"); + gen.height = static_cast(height); + serializer.endObject(); +} + +void serialize(TransactionInputToScript& script, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + serializer.endObject(); +} + +void serialize(TransactionInputToScriptHash& scripthash, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + serializer.endObject(); +} + +void serialize(TransactionInputToKey& key, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + serializer(key.amount, "amount"); + serializeVarintVector(key.keyOffsets, serializer, "key_offsets"); + serializer(key.keyImage, "k_image"); + serializer.endObject(); +} + +void serialize(TransactionInputMultisignature& multisignature, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + serializer(multisignature.amount, "amount"); + serializer(multisignature.signatures, "signatures"); + serializer(multisignature.outputIndex, "outputIndex"); + serializer.endObject(); +} + +void serialize(TransactionOutput& output, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + serializer(output.amount, "amount"); + serializer(output.target, "target"); + serializer.endObject(); +} + +void serialize(TransactionOutputTarget& output, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + + if (serializer.type() == ISerializer::OUTPUT) { + BinaryVariantTagGetter tagGetter; + uint8_t tag = boost::apply_visitor(tagGetter, output); + serializer.binary(&tag, sizeof(tag), "type"); + + VariantSerializer visitor(serializer, "data"); + boost::apply_visitor(visitor, output); + } else { + uint8_t tag; + serializer.binary(&tag, sizeof(tag), "type"); + + getVariantValue(serializer, tag, output); + } + + serializer.endObject(); +} + +void serialize(TransactionOutputToScript& script, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + serializer.endObject(); +} + +void serialize(TransactionOutputToScriptHash& scripthash, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + serializer.endObject(); +} + +void serialize(TransactionOutputToKey& key, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + serializer(key.key, "key"); + serializer.endObject(); +} + +void serialize(TransactionOutputMultisignature& multisignature, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + serializer(multisignature.keys, "keys"); + serializer(multisignature.requiredSignatures, "required_signatures"); + serializer.endObject(); +} + +void serializeBlockHeader(BlockHeader& header, ISerializer& serializer) { + serializer(header.majorVersion, "major_version"); + if (header.majorVersion > BLOCK_MAJOR_VERSION_1) { + throw std::runtime_error("Wrong major version"); + } + + serializer(header.minorVersion, "minor_version"); + serializer(header.timestamp, "timestamp"); + serializer(header.prevId, "prev_id"); + serializer.binary(&header.nonce, sizeof(header.nonce), "nonce"); +} + +void serialize(BlockHeader& header, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + serializeBlockHeader(header, serializer); + serializer.endObject(); +} + +void serialize(Block& block, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + + serializeBlockHeader(block, serializer); + + serializer(block.minerTx, "miner_tx"); + serializer(block.txHashes, "tx_hashes"); + + serializer.endObject(); +} + +void serialize(AccountPublicAddress& address, const std::string& name, ISerializer& serializer) { + serializer.beginObject(name); + + serializer(address.m_spendPublicKey, "spend_public_key"); + serializer(address.m_viewPublicKey, "view_public_key"); + + serializer.endObject(); +} + +} //namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_serialization.h b/src/cryptonote_core/cryptonote_serialization.h new file mode 100644 index 00000000..fc9736dd --- /dev/null +++ b/src/cryptonote_core/cryptonote_serialization.h @@ -0,0 +1,47 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include "cryptonote_basic.h" + +namespace cryptonote { +class ISerializer; +} + +namespace crypto { + +void serialize(public_key& pubKey, const std::string& name, cryptonote::ISerializer& enumerator); +void serialize(secret_key& secKey, const std::string& name, cryptonote::ISerializer& enumerator); +void serialize(hash& h, const std::string& name, cryptonote::ISerializer& enumerator); +void serialize(chacha8_iv& chacha, const std::string& name, cryptonote::ISerializer& enumerator); +void serialize(key_image& keyImage, const std::string& name, cryptonote::ISerializer& enumerator); + +} //namespace crypto + +namespace cryptonote { +void serialize(TransactionPrefix& txP, const std::string& name, ISerializer& serializer); +void serialize(Transaction& tx, const std::string& name, ISerializer& serializer); +void serialize(TransactionInput& in, const std::string& name, ISerializer& serializer); +void serialize(TransactionOutput& in, const std::string& name, ISerializer& serializer); + +void serialize(TransactionInputGenerate& gen, const std::string& name, ISerializer& serializer); +void serialize(TransactionInputToScript& script, const std::string& name, ISerializer& serializer); +void serialize(TransactionInputToScriptHash& scripthash, const std::string& name, ISerializer& serializer); +void serialize(TransactionInputToKey& key, const std::string& name, ISerializer& serializer); +void serialize(TransactionInputMultisignature& multisignature, const std::string& name, ISerializer& serializer); + +void serialize(TransactionOutput& output, const std::string& name, ISerializer& serializer); + +void serialize(TransactionOutputTarget& output, const std::string& name, ISerializer& serializer); + +void serialize(TransactionOutputToScript& script, const std::string& name, ISerializer& serializer); +void serialize(TransactionOutputToScriptHash& scripthash, const std::string& name, ISerializer& serializer); +void serialize(TransactionOutputToKey& key, const std::string& name, ISerializer& serializer); +void serialize(TransactionOutputMultisignature& multisignature, const std::string& name, ISerializer& serializer); +void serialize(BlockHeader& header, const std::string& name, ISerializer& serializer); +void serialize(Block& block, const std::string& name, ISerializer& serializer); +void serialize(AccountPublicAddress& address, const std::string& name, ISerializer& serializer); + +} //namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_stat_info.h b/src/cryptonote_core/cryptonote_stat_info.h index 0cdee0f7..0e25cfaa 100644 --- a/src/cryptonote_core/cryptonote_stat_info.h +++ b/src/cryptonote_core/cryptonote_stat_info.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/cryptonote_core/difficulty.cpp b/src/cryptonote_core/difficulty.cpp index 2774c505..a9752101 100644 --- a/src/cryptonote_core/difficulty.cpp +++ b/src/cryptonote_core/difficulty.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -62,50 +62,4 @@ namespace cryptonote { carry = cadc(high, top, carry); return !carry; } - - difficulty_type next_difficulty(vector timestamps, vector cumulative_difficulties, size_t target_seconds) { - //cutoff DIFFICULTY_LAG - if(timestamps.size() > DIFFICULTY_WINDOW) - { - timestamps.resize(DIFFICULTY_WINDOW); - cumulative_difficulties.resize(DIFFICULTY_WINDOW); - } - - - size_t length = timestamps.size(); - assert(length == cumulative_difficulties.size()); - if (length <= 1) { - return 1; - } - static_assert(DIFFICULTY_WINDOW >= 2, "Window is too small"); - assert(length <= DIFFICULTY_WINDOW); - sort(timestamps.begin(), timestamps.end()); - size_t cut_begin, cut_end; - static_assert(2 * DIFFICULTY_CUT <= DIFFICULTY_WINDOW - 2, "Cut length is too large"); - if (length <= DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) { - cut_begin = 0; - cut_end = length; - } else { - cut_begin = (length - (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) + 1) / 2; - cut_end = cut_begin + (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT); - } - assert(/*cut_begin >= 0 &&*/ cut_begin + 2 <= cut_end && cut_end <= length); - uint64_t time_span = timestamps[cut_end - 1] - timestamps[cut_begin]; - if (time_span == 0) { - time_span = 1; - } - difficulty_type total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin]; - assert(total_work > 0); - uint64_t low, high; - mul(total_work, target_seconds, low, high); - if (high != 0 || low + time_span - 1 < low) { - return 0; - } - return (low + time_span - 1) / time_span; - } - - difficulty_type next_difficulty(vector timestamps, vector cumulative_difficulties) - { - return next_difficulty(std::move(timestamps), std::move(cumulative_difficulties), DIFFICULTY_TARGET); - } } diff --git a/src/cryptonote_core/difficulty.h b/src/cryptonote_core/difficulty.h index d38580e3..76906958 100644 --- a/src/cryptonote_core/difficulty.h +++ b/src/cryptonote_core/difficulty.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,4 @@ namespace cryptonote typedef std::uint64_t difficulty_type; bool check_hash(const crypto::hash &hash, difficulty_type difficulty); - difficulty_type next_difficulty(std::vector timestamps, std::vector cumulative_difficulties); - difficulty_type next_difficulty(std::vector timestamps, std::vector cumulative_difficulties, size_t target_seconds); } diff --git a/src/cryptonote_core/i_miner_handler.h b/src/cryptonote_core/i_miner_handler.h new file mode 100644 index 00000000..4c19780e --- /dev/null +++ b/src/cryptonote_core/i_miner_handler.h @@ -0,0 +1,18 @@ +// Copyright (c) 2011-2015 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/difficulty.h" + +namespace cryptonote { + struct i_miner_handler { + virtual bool handle_block_found(Block& b) = 0; + virtual bool get_block_template(Block& b, const AccountPublicAddress& adr, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) = 0; + + protected: + ~i_miner_handler(){}; + }; +} diff --git a/src/cryptonote_core/miner.cpp b/src/cryptonote_core/miner.cpp index f9ade9f3..fe9123a8 100644 --- a/src/cryptonote_core/miner.cpp +++ b/src/cryptonote_core/miner.cpp @@ -1,14 +1,16 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. - #include #include -#include + +#include +#include #include #include -#include +#include + #include "misc_language.h" #include "include_base_utils.h" #include "cryptonote_basic_impl.h" @@ -23,27 +25,20 @@ using namespace epee; #include "miner.h" - - +#include +#include namespace cryptonote { - namespace - { - const command_line::arg_descriptor arg_extra_messages = {"extra-messages-file", "Specify file for extra messages to include into coinbase transactions", "", true}; - const command_line::arg_descriptor arg_start_mining = {"start-mining", "Specify wallet address to mining for", "", true}; - const command_line::arg_descriptor arg_mining_threads = {"mining-threads", "Specify mining threads count", 0, true}; - } - - - miner::miner(i_miner_handler* phandler):m_stop(1), - m_template(boost::value_initialized()), + miner::miner(const Currency& currency, i_miner_handler* phandler): + m_currency(currency), + m_stop(1), + m_template(boost::value_initialized()), m_template_no(0), m_diffic(0), m_thread_index(0), m_phandler(phandler), - m_height(0), m_pausers_count(0), m_threads_total(0), m_starter_nonce(0), @@ -53,38 +48,34 @@ namespace cryptonote m_do_mining(false), m_current_hash_rate(0) { - } //----------------------------------------------------------------------------------------------------- - miner::~miner() - { + miner::~miner() { stop(); } //----------------------------------------------------------------------------------------------------- - bool miner::set_block_template(const block& bl, const difficulty_type& di, uint64_t height) - { + bool miner::set_block_template(const Block& bl, const difficulty_type& di) { CRITICAL_REGION_LOCAL(m_template_lock); m_template = bl; + m_diffic = di; - m_height = height; ++m_template_no; m_starter_nonce = crypto::rand(); return true; } //----------------------------------------------------------------------------------------------------- - bool miner::on_block_chain_update() - { - if(!is_mining()) + bool miner::on_block_chain_update() { + if (!is_mining()) { return true; + } return request_block_template(); } //----------------------------------------------------------------------------------------------------- - bool miner::request_block_template() - { - block bl = AUTO_VAL_INIT(bl); + bool miner::request_block_template() { + Block bl = AUTO_VAL_INIT(bl); difficulty_type di = AUTO_VAL_INIT(di); - uint64_t height = AUTO_VAL_INIT(height); + uint64_t height; cryptonote::blobdata extra_nonce; if(m_extra_messages.size() && m_config.current_extra_message_index < m_extra_messages.size()) { @@ -96,7 +87,7 @@ namespace cryptonote LOG_ERROR("Failed to get_block_template(), stopping mining"); return false; } - set_block_template(bl, di, height); + set_block_template(bl, di); return true; } //----------------------------------------------------------------------------------------------------- @@ -139,26 +130,16 @@ namespace cryptonote m_last_hr_merge_time = misc_utils::get_tick_count(); m_hashes = 0; } - //----------------------------------------------------------------------------------------------------- - void miner::init_options(boost::program_options::options_description& desc) - { - command_line::add_arg(desc, arg_extra_messages); - command_line::add_arg(desc, arg_start_mining); - command_line::add_arg(desc, arg_mining_threads); - } - //----------------------------------------------------------------------------------------------------- - bool miner::init(const boost::program_options::variables_map& vm) - { - if(command_line::has_arg(vm, arg_extra_messages)) - { + + bool miner::init(const MinerConfig& config) { + if (!config.extraMessages.empty()) { std::string buff; - bool r = file_io_utils::load_file_to_string(command_line::get_arg(vm, arg_extra_messages), buff); - CHECK_AND_ASSERT_MES(r, false, "Failed to load file with extra messages: " << command_line::get_arg(vm, arg_extra_messages)); + bool r = file_io_utils::load_file_to_string(config.extraMessages, buff); + CHECK_AND_ASSERT_MES(r, false, "Failed to load file with extra messages: " << config.extraMessages); std::vector extra_vec; boost::split(extra_vec, buff, boost::is_any_of("\n"), boost::token_compress_on ); m_extra_messages.resize(extra_vec.size()); - for(size_t i = 0; i != extra_vec.size(); i++) - { + for(size_t i = 0; i != extra_vec.size(); i++) { string_tools::trim(extra_vec[i]); if(!extra_vec[i].size()) continue; @@ -166,24 +147,21 @@ namespace cryptonote if(buff != "0") m_extra_messages[i] = buff; } - m_config_folder_path = boost::filesystem::path(command_line::get_arg(vm, arg_extra_messages)).parent_path().string(); + m_config_folder_path = boost::filesystem::path(config.extraMessages).parent_path().string(); m_config = AUTO_VAL_INIT(m_config); - epee::serialization::load_t_from_json_file(m_config, m_config_folder_path + "/" + MINER_CONFIG_FILE_NAME); + epee::serialization::load_t_from_json_file(m_config, m_config_folder_path + "/" + cryptonote::parameters::MINER_CONFIG_FILE_NAME); LOG_PRINT_L0("Loaded " << m_extra_messages.size() << " extra messages, current index " << m_config.current_extra_message_index); } - if(command_line::has_arg(vm, arg_start_mining)) - { - if(!cryptonote::get_account_address_from_str(m_mine_address, command_line::get_arg(vm, arg_start_mining))) - { - LOG_ERROR("Target account address " << command_line::get_arg(vm, arg_start_mining) << " has wrong format, starting daemon canceled"); + if(!config.startMining.empty()) { + if (!m_currency.parseAccountAddressString(config.startMining, m_mine_address)) { + LOG_ERROR("Target account address " << config.startMining << " has wrong format, starting daemon canceled"); return false; } m_threads_total = 1; m_do_mining = true; - if(command_line::has_arg(vm, arg_mining_threads)) - { - m_threads_total = command_line::get_arg(vm, arg_mining_threads); + if(config.miningThreads > 0) { + m_threads_total = config.miningThreads; } } @@ -195,7 +173,7 @@ namespace cryptonote return !m_stop; } //----------------------------------------------------------------------------------------------------- - bool miner::start(const account_public_address& adr, size_t threads_count, const boost::thread::attributes& attrs) + bool miner::start(const AccountPublicAddress& adr, size_t threads_count, const boost::thread::attributes& attrs) { m_mine_address = adr; m_threads_total = static_cast(threads_count); @@ -254,18 +232,61 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------------- - bool miner::find_nonce_for_given_block(crypto::cn_context &context, block& bl, const difficulty_type& diffic, uint64_t height) - { - for(; bl.nonce != std::numeric_limits::max(); bl.nonce++) - { - crypto::hash h; - get_block_longhash(context, bl, h, height); + bool miner::find_nonce_for_given_block(crypto::cn_context &context, Block& bl, const difficulty_type& diffic) { - if(check_hash(h, diffic)) - { - return true; + unsigned nthreads = std::thread::hardware_concurrency(); + + if (nthreads > 0 && diffic > 5) { + std::vector> threads(nthreads); + std::atomic foundNonce; + std::atomic found(false); + uint32_t startNonce = crypto::rand(); + + for (unsigned i = 0; i < nthreads; ++i) { + threads[i] = std::async(std::launch::async, [&, i]() { + crypto::cn_context localctx; + crypto::hash h; + + Block lb(bl); // copy to local block + + for (uint32_t nonce = startNonce + i; !found; nonce += nthreads) { + lb.nonce = nonce; + + if (!get_block_longhash(localctx, lb, h)) { + return; + } + + if (check_hash(h, diffic)) { + foundNonce = nonce; + found = true; + return; + } + } + }); + } + + for (auto& t : threads) { + t.wait(); + } + + if (found) { + bl.nonce = foundNonce.load(); + } + + return found; + } else { + for (; bl.nonce != std::numeric_limits::max(); bl.nonce++) { + crypto::hash h; + if (!get_block_longhash(context, bl, h)) { + return false; + } + + if (check_hash(h, diffic)) { + return true; + } } } + return false; } //----------------------------------------------------------------------------------------------------- @@ -307,11 +328,10 @@ namespace cryptonote LOG_PRINT_L0("Miner thread was started ["<< th_local_index << "]"); log_space::log_singletone::set_thread_log_prefix(std::string("[miner ") + std::to_string(th_local_index) + "]"); uint32_t nonce = m_starter_nonce + th_local_index; - uint64_t height = 0; difficulty_type local_diff = 0; uint32_t local_template_ver = 0; crypto::cn_context context; - block b; + Block b; while(!m_stop) { if(m_pausers_count)//anti split workaround @@ -326,7 +346,6 @@ namespace cryptonote CRITICAL_REGION_BEGIN(m_template_lock); b = m_template; local_diff = m_diffic; - height = m_height; CRITICAL_REGION_END(); local_template_ver = m_template_no; nonce = m_starter_nonce + th_local_index; @@ -341,9 +360,12 @@ namespace cryptonote b.nonce = nonce; crypto::hash h; - get_block_longhash(context, b, h, height); + if (!m_stop && !get_block_longhash(context, b, h)) { + LOG_ERROR("Failed to get block long hash"); + m_stop = true; + } - if(!m_stop && check_hash(h, local_diff)) + if (!m_stop && check_hash(h, local_diff)) { //we lucky! ++m_config.current_extra_message_index; @@ -354,7 +376,7 @@ namespace cryptonote }else { //success update, lets update config - epee::serialization::store_t_to_json_file(m_config, m_config_folder_path + "/" + MINER_CONFIG_FILE_NAME); + epee::serialization::store_t_to_json_file(m_config, m_config_folder_path + "/" + cryptonote::parameters::MINER_CONFIG_FILE_NAME); } } @@ -366,4 +388,3 @@ namespace cryptonote } //----------------------------------------------------------------------------------------------------- } - diff --git a/src/cryptonote_core/miner.h b/src/cryptonote_core/miner.h index a5365482..0bebd2d4 100644 --- a/src/cryptonote_core/miner.h +++ b/src/cryptonote_core/miner.h @@ -1,40 +1,33 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once -#include #include -#include "cryptonote_basic.h" -#include "difficulty.h" + +#include + +// epee +#include "serialization/keyvalue_serialization.h" #include "math_helper.h" +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/Currency.h" +#include "cryptonote_core/difficulty.h" +#include "cryptonote_core/i_miner_handler.h" +#include "cryptonote_core/MinerConfig.h" -namespace cryptonote -{ - - struct i_miner_handler - { - virtual bool handle_block_found(block& b) = 0; - virtual bool get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) = 0; - protected: - ~i_miner_handler(){}; - }; - - /************************************************************************/ - /* */ - /************************************************************************/ - class miner - { +namespace cryptonote { + class miner { public: - miner(i_miner_handler* phandler); + miner(const Currency& currency, i_miner_handler* phandler); ~miner(); - bool init(const boost::program_options::variables_map& vm); - static void init_options(boost::program_options::options_description& desc); - bool set_block_template(const block& bl, const difficulty_type& diffic, uint64_t height); + + bool init(const MinerConfig& config); + bool set_block_template(const Block& bl, const difficulty_type& diffic); bool on_block_chain_update(); - bool start(const account_public_address& adr, size_t threads_count, const boost::thread::attributes& attrs); + bool start(const AccountPublicAddress& adr, size_t threads_count, const boost::thread::attributes& attrs); uint64_t get_speed(); void send_stop_signal(); bool stop(); @@ -42,7 +35,7 @@ namespace cryptonote bool on_idle(); void on_synchronized(); //synchronous analog (for fast calls) - static bool find_nonce_for_given_block(crypto::cn_context &context, block& bl, const difficulty_type& diffic, uint64_t height); + static bool find_nonce_for_given_block(crypto::cn_context &context, Block& bl, const difficulty_type& diffic); void pause(); void resume(); void do_print_hashrate(bool do_hr); @@ -62,13 +55,13 @@ namespace cryptonote }; + const Currency& m_currency; volatile uint32_t m_stop; epee::critical_section m_template_lock; - block m_template; + Block m_template; std::atomic m_template_no; std::atomic m_starter_nonce; difficulty_type m_diffic; - uint64_t m_height; volatile uint32_t m_thread_index; volatile uint32_t m_threads_total; std::atomic m_pausers_count; @@ -77,7 +70,7 @@ namespace cryptonote std::list m_threads; epee::critical_section m_threads_lock; i_miner_handler* m_phandler; - account_public_address m_mine_address; + AccountPublicAddress m_mine_address; epee::math_helper::once_a_time_seconds<5> m_update_block_template_interval; epee::math_helper::once_a_time_seconds<2> m_update_merge_hr_interval; std::vector m_extra_messages; @@ -90,9 +83,5 @@ namespace cryptonote std::list m_last_hash_rates; bool m_do_print_hashrate; bool m_do_mining; - }; } - - - diff --git a/src/cryptonote_core/tx_extra.h b/src/cryptonote_core/tx_extra.h index bf964823..129454fe 100644 --- a/src/cryptonote_core/tx_extra.h +++ b/src/cryptonote_core/tx_extra.h @@ -1,9 +1,15 @@ -// Copyright (c) 2011-2014 The Cryptonote developers +// Copyright (c) 2011-2015 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once +#include "crypto/crypto.h" +#include "crypto/hash.h" +#include "serialization/binary_archive.h" +#include "serialization/crypto.h" +#include "serialization/serialization.h" +#include "serialization/variant.h" #define TX_EXTRA_PADDING_MAX_COUNT 255 #define TX_EXTRA_NONCE_MAX_COUNT 255 @@ -11,7 +17,6 @@ #define TX_EXTRA_TAG_PADDING 0x00 #define TX_EXTRA_TAG_PUBKEY 0x01 #define TX_EXTRA_NONCE 0x02 -#define TX_EXTRA_MERGE_MINING_TAG 0x03 #define TX_EXTRA_NONCE_PAYMENT_ID 0x00 @@ -83,62 +88,13 @@ namespace cryptonote END_SERIALIZE() }; - struct tx_extra_merge_mining_tag - { - struct serialize_helper - { - tx_extra_merge_mining_tag& mm_tag; - - serialize_helper(tx_extra_merge_mining_tag& mm_tag_) : mm_tag(mm_tag_) - { - } - - BEGIN_SERIALIZE() - VARINT_FIELD_N("depth", mm_tag.depth) - FIELD_N("merkle_root", mm_tag.merkle_root) - END_SERIALIZE() - }; - - size_t depth; - crypto::hash merkle_root; - - // load - template