diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index d3045a22..daa5db2a 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -38,6 +38,7 @@ #include "serialization/binary_utils.h" // dump_binary(), parse_binary() #include "serialization/json_utils.h" // dump_json() #include "include_base_utils.h" +#include "blockchain_db/db_types.h" #include // for db flag arguments @@ -77,12 +78,48 @@ using namespace cryptonote; using namespace epee; -int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags) +std::string join_set_strings(const std::unordered_set& db_types_all, const char* delim) +{ + std::string result; + std::ostringstream s; + std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator(s, delim)); + result = s.str(); + if (result.length() > 0) + result.erase(result.end()-strlen(delim), result.end()); + return result; +} + +// db_type: lmdb, berkeley +// db_mode: safe, fast, fastest +int get_db_flags_from_mode(const std::string& db_type, const std::string& db_mode) +{ + uint64_t BDB_FAST_MODE = 0; + uint64_t BDB_FASTEST_MODE = 0; + uint64_t BDB_SAFE_MODE = 0; + +#if defined(BERKELEY_DB) + BDB_FAST_MODE = DB_TXN_WRITE_NOSYNC; + BDB_FASTEST_MODE = DB_TXN_NOSYNC; + BDB_SAFE_MODE = DB_TXN_SYNC; +#endif + + int db_flags = 0; + bool islmdb = db_type == "lmdb"; + if (db_mode == "safe") + db_flags = islmdb ? MDB_NORDAHEAD : BDB_SAFE_MODE; + else if (db_mode == "fast") + db_flags = islmdb ? MDB_NOMETASYNC | MDB_NOSYNC | MDB_NORDAHEAD : BDB_FAST_MODE; + else if (db_mode == "fastest") + db_flags = islmdb ? MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | MDB_NOMETASYNC | MDB_NOSYNC : BDB_FASTEST_MODE; + return db_flags; +} + +int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& db_flags) { std::vector db_args; boost::split(db_args, db_arg_str, boost::is_any_of("#")); - db_engine = db_args.front(); - boost::algorithm::trim(db_engine); + db_type = db_args.front(); + boost::algorithm::trim(db_type); if (db_args.size() == 1) { @@ -94,28 +131,73 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, in return 1; } +#if !defined(BERKELEY_DB) + if (db_type == "berkeley") + { + LOG_ERROR("BerkeleyDB support disabled."); + return false; + } +#endif + std::string db_arg_str2 = db_args[1]; boost::split(db_args, db_arg_str2, boost::is_any_of(",")); - for (auto& it : db_args) + + // optionally use a composite mode instead of individual flags + const std::unordered_set db_modes {"safe", "fast", "fastest"}; + std::string db_mode; + if (db_args.size() == 1) { - boost::algorithm::trim(it); - if (it.empty()) - continue; - LOG_PRINT_L1("LMDB flag: " << it); - if (it == "nosync") - mdb_flags |= MDB_NOSYNC; - else if (it == "nometasync") - mdb_flags |= MDB_NOMETASYNC; - else if (it == "writemap") - mdb_flags |= MDB_WRITEMAP; - else if (it == "mapasync") - mdb_flags |= MDB_MAPASYNC; - else if (it == "nordahead") - mdb_flags |= MDB_NORDAHEAD; - else + if (db_modes.count(db_args[0]) > 0) { - std::cerr << "unrecognized database flag: " << it << ENDL; - return 1; + db_mode = db_args[0]; + } + } + if (! db_mode.empty()) + { + db_flags = get_db_flags_from_mode(db_type, db_mode); + } + else + { + for (auto& it : db_args) + { + boost::algorithm::trim(it); + if (it.empty()) + continue; + if (db_type == "lmdb") + { + LOG_PRINT_L1("LMDB flag: " << it); + if (it == "nosync") + db_flags |= MDB_NOSYNC; + else if (it == "nometasync") + db_flags |= MDB_NOMETASYNC; + else if (it == "writemap") + db_flags |= MDB_WRITEMAP; + else if (it == "mapasync") + db_flags |= MDB_MAPASYNC; + else if (it == "nordahead") + db_flags |= MDB_NORDAHEAD; + else + { + std::cerr << "unrecognized database flag: " << it << ENDL; + return 1; + } + } +#if defined(BERKELEY_DB) + else if (db_type == "berkeley") + { + if (it == "txn_write_nosync") + db_flags = DB_TXN_WRITE_NOSYNC; + else if (it == "txn_nosync") + db_flags = DB_TXN_NOSYNC; + else if (it == "txn_sync") + db_flags = DB_TXN_SYNC; + else + { + std::cerr << "unrecognized database flag: " << it << ENDL; + return 1; + } + } +#endif } } return 0; @@ -131,7 +213,7 @@ int pop_blocks(FakeCore& simple_core, int num_blocks) if (simple_core.support_batch) use_batch = true; else - LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); + LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring"); } if (use_batch) @@ -176,11 +258,11 @@ template int import_from_file(FakeCore& simple_core, const std::string& import_file_path, uint64_t block_stop=0) { #if !defined(BLOCKCHAIN_DB) - static_assert(std::is_same::value || std::is_same::value, + static_assert(std::is_same::value || std::is_same::value, "FakeCore constraint error"); #endif #if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) - if (std::is_same::value) + if (std::is_same::value) { // Reset stats, in case we're using newly created db, accumulating stats // from addition of genesis block. @@ -250,7 +332,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path, if (simple_core.support_batch) use_batch = true; else - LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); + LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring"); } if (use_batch) @@ -530,11 +612,19 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path, int main(int argc, char* argv[]) { #if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY) - std::string default_db_engine = "memory"; + std::string default_db_type = "memory"; + std::string default_db_engine_compiled = "memory"; #else - std::string default_db_engine = "lmdb"; + std::string default_db_type = "lmdb"; + std::string default_db_engine_compiled = "blockchain_db"; #endif + std::unordered_set db_types_all = cryptonote::blockchain_db_types; + db_types_all.insert("memory"); + + std::string available_dbs = join_set_strings(db_types_all, ", "); + available_dbs = "available: " + available_dbs; + uint32_t log_level = LOG_LEVEL_0; uint64_t num_blocks = 0; uint64_t block_stop = 0; @@ -566,8 +656,7 @@ int main(int argc, char* argv[]) , false }; const command_line::arg_descriptor arg_database = { - "database", "available: memory, lmdb" - , default_db_engine + "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor arg_verify = {"verify", "Verify blocks and transactions during import", true}; @@ -648,8 +737,6 @@ int main(int argc, char* argv[]) } } - std::vector db_engines {"memory", "lmdb"}; - opt_testnet = command_line::get_arg(vm, arg_testnet_on); auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; m_config_folder = command_line::get_arg(vm, data_dir_arg); @@ -677,23 +764,34 @@ int main(int argc, char* argv[]) } - std::string db_engine; - int mdb_flags = 0; + std::string db_type; + std::string db_engine_compiled; + int db_flags = 0; int res = 0; - res = parse_db_arguments(db_arg_str, db_engine, mdb_flags); + res = parse_db_arguments(db_arg_str, db_type, db_flags); if (res) { std::cerr << "Error parsing database argument(s)" << ENDL; return 1; } - if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end()) + if (db_types_all.count(db_type) == 0) { - std::cerr << "Invalid database engine: " << db_engine << std::endl; + std::cerr << "Invalid database type: " << db_type << std::endl; return 1; } - LOG_PRINT_L0("database: " << db_engine); + if ((db_type == "lmdb") || (db_type == "berkeley")) + { + db_engine_compiled = "blockchain_db"; + } + else + { + db_engine_compiled = "memory"; + } + + LOG_PRINT_L0("database: " << db_type); + LOG_PRINT_L0("database flags: " << db_flags); LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha); if (opt_batch) { @@ -722,31 +820,31 @@ int main(int argc, char* argv[]) // for multi_db_runtime: #if !defined(BLOCKCHAIN_DB) - if (db_engine == "lmdb") + if (db_type == "lmdb" || db_type == "berkeley") { - fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags); + fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags); import_from_file(simple_core, import_file_path, block_stop); } - else if (db_engine == "memory") + else if (db_type == "memory") { fake_core_memory simple_core(m_config_folder, opt_testnet); import_from_file(simple_core, import_file_path, block_stop); } else { - std::cerr << "database engine unrecognized" << ENDL; + std::cerr << "database type unrecognized" << ENDL; return 1; } // for multi_db_compile: #else - if (db_engine != default_db_engine) + if (db_engine_compiled != default_db_engine_compiled) { - std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl; + std::cerr << "Invalid database type for compiled version: " << db_type << std::endl; return 1; } #if BLOCKCHAIN_DB == DB_LMDB - fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags); + fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags); #else fake_core_memory simple_core(m_config_folder, opt_testnet); #endif diff --git a/src/blockchain_utilities/fake_core.h b/src/blockchain_utilities/fake_core.h index 29f34026..20a3361b 100644 --- a/src/blockchain_utilities/fake_core.h +++ b/src/blockchain_utilities/fake_core.h @@ -33,6 +33,7 @@ #include "cryptonote_core/blockchain_storage.h" // in-memory DB #include "blockchain_db/blockchain_db.h" #include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/berkeleydb/db_bdb.h" using namespace cryptonote; @@ -47,7 +48,7 @@ namespace #if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB -struct fake_core_lmdb +struct fake_core_db { Blockchain m_storage; HardFork* m_hardfork = nullptr; @@ -57,15 +58,26 @@ struct fake_core_lmdb // for multi_db_runtime: #if !defined(BLOCKCHAIN_DB) - fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool) + fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(&m_storage), m_storage(m_pool) // for multi_db_compile: #else - fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool) + fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(m_storage), m_storage(m_pool) #endif { m_pool.init(path.string()); - BlockchainDB* db = new BlockchainLMDB(); + BlockchainDB* db = nullptr; + if (db_type == "lmdb") + db = new BlockchainLMDB(); +#if defined(BERKELEY_DB) + else if (db_type == "berkeley") + db = new BlockchainBDB(); +#endif + else + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } boost::filesystem::path folder(path); @@ -76,7 +88,7 @@ struct fake_core_lmdb const std::string filename = folder.string(); try { - db->open(filename, mdb_flags); + db->open(filename, db_flags); } catch (const std::exception& e) { @@ -96,7 +108,7 @@ struct fake_core_lmdb support_batch = true; support_add_block = true; } - ~fake_core_lmdb() + ~fake_core_db() { m_storage.get_db().check_hard_fork_info(); m_storage.deinit();