From f7e99047e47bd290ff763645f19d19c64a3bd4cd Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 26 Oct 2015 15:52:07 +0000 Subject: [PATCH 1/2] db_lmdb: add versioning, to detect incompatible format changes --- src/blockchain_db/lmdb/db_lmdb.cpp | 91 +++++++++++++++++++++++++++--- src/blockchain_db/lmdb/db_lmdb.h | 2 + 2 files changed, 85 insertions(+), 8 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 1e7078c6..fc4f59cc 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -39,6 +39,10 @@ using epee::string_tools::pod_to_hex; +// Increase when the DB changes in a non backward compatible way, and there +// is no automatic conversion, so that a full resync is needed. +#define VERSION 0 + namespace { @@ -122,6 +126,19 @@ private: std::unique_ptr data; }; +template<> +struct MDB_val_copy: public MDB_val +{ + MDB_val_copy(const char *s) : + data(strdup(s)) + { + mv_size = strlen(s) + 1; // include the NUL, makes it easier for compares + mv_data = data.get(); + } +private: + std::unique_ptr data; +}; + auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) { const uint64_t va = *(const uint64_t*)a->mv_data; @@ -154,6 +171,13 @@ int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } +int compare_string(const MDB_val *a, const MDB_val *b) +{ + const char *va = (const char*) a->mv_data; + const char *vb = (const char*) b->mv_data; + return strcmp(va, vb); +} + const char* const LMDB_BLOCKS = "blocks"; const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; @@ -178,6 +202,8 @@ const char* const LMDB_SPENT_KEYS = "spent_keys"; const char* const LMDB_HF_STARTING_HEIGHTS = "hf_starting_heights"; const char* const LMDB_HF_VERSIONS = "hf_versions"; +const char* const LMDB_PROPERTIES = "properties"; + inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string) { if (mdb_dbi_open(txn, name, flags, &dbi)) @@ -1037,6 +1063,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_HF_STARTING_HEIGHTS, MDB_CREATE, m_hf_starting_heights, "Failed to open db handle for m_hf_starting_heights"); lmdb_db_open(txn, LMDB_HF_VERSIONS, MDB_CREATE, m_hf_versions, "Failed to open db handle for m_hf_versions"); + lmdb_db_open(txn, LMDB_PROPERTIES, MDB_CREATE, m_properties, "Failed to open db handle for m_properties"); + mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); mdb_set_compare(txn, m_spent_keys, compare_hash32); @@ -1046,6 +1074,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) mdb_set_compare(txn, m_tx_heights, compare_hash32); mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); mdb_set_compare(txn, m_hf_versions, compare_uint64); + mdb_set_compare(txn, m_properties, compare_string); // get and keep current height MDB_stat db_stats; @@ -1059,6 +1088,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) throw0(DB_ERROR("Failed to query m_output_indices")); m_num_outputs = db_stats.ms_entries; + bool compatible = true; + // ND: This "new" version of the lmdb database is incompatible with // the previous version. Ensure that the output_keys database is // sizeof(output_data_t) in length. Otherwise, inform user and @@ -1077,16 +1108,60 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // LOG_PRINT_L0("Output keys size: " << v.mv_size); if(v.mv_size != sizeof(output_data_t)) - { - txn.abort(); - mdb_env_close(m_env); - m_open = false; - LOG_PRINT_RED_L0("Existing lmdb database is incompatible with this version."); - LOG_PRINT_RED_L0("Please delete the existing database and resync."); - return; - } + compatible = false; } + MDB_val_copy k("version"); + MDB_val v; + auto get_result = mdb_get(txn, m_properties, &k, &v); + if(get_result == MDB_SUCCESS) + { + if (*(const uint32_t*)v.mv_data > VERSION) + { + LOG_PRINT_RED_L0("Existing lmdb database was made by a later version. We don't know how it will change yet."); + compatible = false; + } + else if (VERSION > 0 && *(const uint32_t*)v.mv_data < VERSION) + { + compatible = false; + } + } + else + { + // if not found, but we're on version 0, it's fine. If the DB's empty, it's fine too. + if (VERSION > 0 && m_height > 0) + compatible = false; + } + + if (!compatible) + { + txn.abort(); + mdb_env_close(m_env); + m_open = false; + LOG_PRINT_RED_L0("Existing lmdb database is incompatible with this version."); + LOG_PRINT_RED_L0("Please delete the existing database and resync."); + return; + } + + if (!(mdb_flags & MDB_RDONLY)) + { + // only write version on an empty DB + if (m_height == 0) + { + MDB_val_copy k("version"); + MDB_val_copy v(VERSION); + auto put_result = mdb_put(txn, m_properties, &k, &v, 0); + if (put_result != MDB_SUCCESS) + { + txn.abort(); + mdb_env_close(m_env); + m_open = false; + LOG_PRINT_RED_L0("Failed to write version to database."); + return; + } + } + } + // commit the transaction txn.commit(); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 6125ef2e..85c2dffc 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -306,6 +306,8 @@ private: MDB_dbi m_hf_starting_heights; MDB_dbi m_hf_versions; + MDB_dbi m_properties; + uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; From ecf0b8340651403a62ae22ef3f0dd4574152b4b1 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 26 Oct 2015 17:57:44 +0000 Subject: [PATCH 2/2] db_bdb: add versioning, to detect incompatible format changes --- src/blockchain_db/berkeleydb/db_bdb.cpp | 81 +++++++++++++++++++++++++ src/blockchain_db/berkeleydb/db_bdb.h | 2 + 2 files changed, 83 insertions(+) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 6560ce5c..c387d036 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -38,6 +38,10 @@ using epee::string_tools::pod_to_hex; #define DB_DEFAULT_TX (m_write_txn != nullptr ? *m_write_txn : (DbTxn*) nullptr) +// Increase when the DB changes in a non backward compatible way, and there +// is no automatic conversion, so that a full resync is needed. +#define VERSION 0 + namespace { @@ -121,6 +125,8 @@ const char* const BDB_SPENT_KEYS = "spent_keys"; const char* const BDB_HF_STARTING_HEIGHTS = "hf_starting_heights"; const char* const BDB_HF_VERSIONS = "hf_versions"; +const char* const BDB_PROPERTIES = "properties"; + const unsigned int MB = 1024 * 1024; // ND: FIXME: db keeps running out of locks when doing full syncs. Possible bug??? Set it to 5K for now. const unsigned int DB_MAX_LOCKS = 5000; @@ -180,6 +186,22 @@ private: std::unique_ptr m_data; }; +template<> +struct Dbt_copy: public Dbt +{ + Dbt_copy(const char *s) : + m_data(strdup(s)) + { + size_t len = strlen(s) + 1; // include the NUL, makes it easier for compare + set_data(m_data.get()); + set_size(len); + set_ulen(len); + set_flags(DB_DBT_USERMEM); + } +private: + std::unique_ptr m_data; +}; + struct Dbt_safe : public Dbt { Dbt_safe() @@ -793,6 +815,8 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags) m_hf_starting_heights = new Db(m_env, 0); m_hf_versions = new Db(m_env, 0); + m_properties = new Db(m_env, 0); + // Tell DB about Dbs that need duplicate support // Note: no need to tell about sorting, // as the default is insertion order, which we want @@ -845,6 +869,8 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags) m_hf_starting_heights->open(txn, BDB_HF_STARTING_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0); m_hf_versions->open(txn, BDB_HF_VERSIONS, NULL, DB_RECNO, DB_CREATE, 0); + m_properties->open(txn, BDB_PROPERTIES, NULL, DB_HASH, DB_CREATE, 0); + txn.commit(); DB_BTREE_STAT* stats; @@ -865,6 +891,56 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags) m_num_outputs = stats->bt_nkeys; delete stats; + // checks for compatibility + bool compatible = true; + + Dbt_copy key("version"); + Dbt_copy result; + auto get_result = m_properties->get(DB_DEFAULT_TX, &key, &result, 0); + if (get_result == 0) + { + if (result > VERSION) + { + LOG_PRINT_RED_L0("Existing BerkeleyDB database was made by a later version. We don't know how it will change yet."); + compatible = false; + } + else if (VERSION > 0 && result < VERSION) + { + compatible = false; + } + } + else + { + // if not found, but we're on version 0, it's fine. If the DB's empty, it's fine too. + if (VERSION > 0 && m_height > 0) + compatible = false; + } + + if (!compatible) + { + m_open = false; + LOG_PRINT_RED_L0("Existing BerkeleyDB database is incompatible with this version."); + LOG_PRINT_RED_L0("Please delete the existing database and resync."); + return; + } + + if (1 /* this can't be set readonly atm */) + { + // only write version on an empty DB + if (m_height == 0) + { + Dbt_copy k("version"); + Dbt_copy v(VERSION); + auto put_result = m_properties->put(DB_DEFAULT_TX, &k, &v, 0); + if (put_result != 0) + { + m_open = false; + LOG_PRINT_RED_L0("Failed to write version to database."); + return; + } + } + } + // run checkpoint thread m_run_checkpoint = true; m_checkpoint_thread.reset(new boost::thread(&BlockchainBDB::checkpoint_worker, this)); @@ -920,6 +996,8 @@ void BlockchainBDB::sync() m_hf_starting_heights->sync(0); m_hf_versions->sync(0); + + m_properties->sync(0); } catch (const std::exception& e) { @@ -1000,6 +1078,9 @@ std::vector BlockchainBDB::get_filenames() const m_hf_versions->get_dbname(pfname, pdbname); filenames.push_back(fname); + m_properties->get_dbname(pfname, pdbname); + filenames.push_back(fname); + std::vector full_paths; for (auto& filename : filenames) diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index b51d745d..a3383b2b 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -445,6 +445,8 @@ private: Db* m_hf_starting_heights; Db* m_hf_versions; + Db* m_properties; + uint64_t m_height; uint64_t m_num_outputs; std::string m_folder;