// Copyright (c) 2011-2016 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once #include #include #include #include #include #include #include #include #include #include template class SwappedMap { private: struct Descriptor { uint64_t offset; uint64_t index; }; public: typedef typename std::pair value_type; class const_iterator { public: //typedef ptrdiff_t difference_type; //typedef std::bidirectional_iterator_tag iterator_category; //typedef std::pair* pointer; //typedef std::pair& reference; //typedef std::pair value_type; const_iterator(SwappedMap* swappedMap, typename std::unordered_map::const_iterator descriptorsIterator) : m_swappedMap(swappedMap), m_descriptorsIterator(descriptorsIterator) { } const_iterator& operator++() { ++m_descriptorsIterator; return *this; } bool operator !=(const_iterator other) const { return m_descriptorsIterator != other.m_descriptorsIterator; } bool operator ==(const_iterator other) const { return m_descriptorsIterator == other.m_descriptorsIterator; } const std::pair& operator*() const { return *m_swappedMap->load(m_descriptorsIterator->first, m_descriptorsIterator->second.offset); } const std::pair* operator->() const { return m_swappedMap->load(m_descriptorsIterator->first, m_descriptorsIterator->second.offset); } typename std::unordered_map::const_iterator innerIterator() const { return m_descriptorsIterator; } private: SwappedMap* m_swappedMap; typename std::unordered_map::const_iterator m_descriptorsIterator; }; typedef const_iterator iterator; SwappedMap(); //SwappedMap(const SwappedMap&) = delete; ~SwappedMap(); //SwappedMap& operator=(const SwappedMap&) = delete; bool open(const std::string& itemFileName, const std::string& indexFileName, size_t poolSize); void close(); uint64_t size() const; const_iterator begin(); const_iterator end(); size_t count(const Key& key) const; const_iterator find(const Key& key); void clear(); void erase(const_iterator iterator); std::pair insert(const std::pair& value); private: std::fstream m_itemsFile; std::fstream m_indexesFile; size_t m_poolSize; std::unordered_map m_descriptors; uint64_t m_itemsFileSize; std::unordered_map m_items; std::list m_cache; std::unordered_map::iterator> m_cacheIterators; uint64_t m_cacheHits; uint64_t m_cacheMisses; uint64_t descriptorsCounter; std::pair* prepare(const Key& key); const std::pair* load(const Key& key, uint64_t offset); }; template SwappedMap::SwappedMap() { } template SwappedMap::~SwappedMap() { close(); } template bool SwappedMap::open(const std::string& itemFileName, const std::string& indexFileName, size_t poolSize) { if (poolSize == 0) { return false; } descriptorsCounter = 0; m_itemsFile.open(itemFileName, std::ios::in | std::ios::out | std::ios::binary); m_indexesFile.open(indexFileName, std::ios::in | std::ios::out | std::ios::binary); if (m_itemsFile && m_indexesFile) { uint64_t count; m_indexesFile.read(reinterpret_cast(&count), sizeof count); if (!m_indexesFile) { return false; } std::unordered_map descriptors; uint64_t itemsFileSize = 0; for (uint64_t i = 0; i < count; ++i) { bool valid; m_indexesFile.read(reinterpret_cast(&valid), sizeof valid); if (!m_indexesFile) { return false; } Key key; m_indexesFile.read(reinterpret_cast(&key), sizeof key); if (!m_indexesFile) { return false; } uint32_t itemSize; m_indexesFile.read(reinterpret_cast(&itemSize), sizeof itemSize); if (!m_indexesFile) { return false; } if (valid) { Descriptor descriptor = { itemsFileSize, i }; descriptors.insert(std::make_pair(key, descriptor)); } descriptorsCounter++; itemsFileSize += itemSize; } m_descriptors.swap(descriptors); m_itemsFileSize = itemsFileSize; } else { m_itemsFile.open(itemFileName, std::ios::out | std::ios::binary); m_itemsFile.close(); m_itemsFile.open(itemFileName, std::ios::in | std::ios::out | std::ios::binary); m_indexesFile.open(indexFileName, std::ios::out | std::ios::binary); uint64_t count = 0; m_indexesFile.write(reinterpret_cast(&count), sizeof count); if (!m_indexesFile) { return false; } m_indexesFile.close(); m_indexesFile.open(indexFileName, std::ios::in | std::ios::out | std::ios::binary); m_descriptors.clear(); m_itemsFileSize = 0; } m_poolSize = poolSize; m_items.clear(); m_cache.clear(); m_cacheIterators.clear(); m_cacheHits = 0; m_cacheMisses = 0; return true; } template void SwappedMap::close() { std::cout << "SwappedMap cache hits: " << m_cacheHits << ", misses: " << m_cacheMisses << " (" << std::fixed << std::setprecision(2) << static_cast(m_cacheMisses) / (m_cacheHits + m_cacheMisses) * 100 << "%)" << std::endl; } template uint64_t SwappedMap::size() const { return m_descriptors.size(); } template typename SwappedMap::const_iterator SwappedMap::begin() { return const_iterator(this, m_descriptors.cbegin()); } template typename SwappedMap::const_iterator SwappedMap::end() { return const_iterator(this, m_descriptors.cend()); } template size_t SwappedMap::count(const Key& key) const { return m_descriptors.count(key); } template typename SwappedMap::const_iterator SwappedMap::find(const Key& key) { return const_iterator(this, m_descriptors.find(key)); } template void SwappedMap::clear() { if (!m_indexesFile) { throw std::runtime_error("SwappedMap::clear"); } m_indexesFile.seekp(0); uint64_t count = 0; m_indexesFile.write(reinterpret_cast(&count), sizeof count); if (!m_indexesFile) { throw std::runtime_error("SwappedMap::clear"); } m_descriptors.clear(); m_itemsFileSize = 0; m_items.clear(); m_cache.clear(); m_cacheIterators.clear(); descriptorsCounter = 0; } template void SwappedMap::erase(const_iterator iterator) { if (!m_indexesFile) { throw std::runtime_error("SwappedMap::erase"); } typename std::unordered_map::const_iterator descriptorsIterator = iterator.innerIterator(); m_indexesFile.seekp(sizeof(uint64_t) + (sizeof(bool) + sizeof(Key) + sizeof(uint32_t)) * descriptorsIterator->second.index); bool valid = false; m_indexesFile.write(reinterpret_cast(&valid), sizeof valid); if (!m_indexesFile) { throw std::runtime_error("SwappedMap::erase"); } m_descriptors.erase(descriptorsIterator); auto cacheIteratorsIterator = m_cacheIterators.find(descriptorsIterator->first); if (cacheIteratorsIterator != m_cacheIterators.end()) { m_items.erase(descriptorsIterator->first); m_cache.erase(cacheIteratorsIterator->second); m_cacheIterators.erase(cacheIteratorsIterator); } } template std::pair::const_iterator, bool> SwappedMap::insert(const std::pair& value) { uint64_t itemsFileSize; { if (!m_itemsFile) { throw std::runtime_error("SwappedMap::insert"); } m_itemsFile.seekp(m_itemsFileSize); try { boost::archive::binary_oarchive archive(m_itemsFile); archive & value.second; } catch (std::exception&) { throw std::runtime_error("SwappedMap::insert"); } itemsFileSize = m_itemsFile.tellp(); } { if (!m_indexesFile) { throw std::runtime_error("SwappedMap::insert"); } m_indexesFile.seekp(sizeof(uint64_t) + (sizeof(bool) + sizeof(Key) + sizeof(uint32_t)) * descriptorsCounter); bool valid = true; m_indexesFile.write(reinterpret_cast(&valid), sizeof valid); if (!m_indexesFile) { throw std::runtime_error("SwappedMap::insert"); } m_indexesFile.write(reinterpret_cast(&value.first), sizeof value.first); if (!m_indexesFile) { throw std::runtime_error("SwappedMap::insert"); } uint32_t itemSize = static_cast(itemsFileSize - m_itemsFileSize); m_indexesFile.write(reinterpret_cast(&itemSize), sizeof itemSize); if (!m_indexesFile) { throw std::runtime_error("SwappedMap::insert"); } m_indexesFile.seekp(0); uint64_t count = descriptorsCounter + 1; m_indexesFile.write(reinterpret_cast(&count), sizeof count); if (!m_indexesFile) { throw std::runtime_error("SwappedMap::insert"); } } Descriptor descriptor = { m_itemsFileSize, descriptorsCounter }; auto descriptorsInsert = m_descriptors.insert(std::make_pair(value.first, descriptor)); m_itemsFileSize = itemsFileSize; descriptorsCounter++; T* newItem = &prepare(value.first)->second; *newItem = value.second; return std::make_pair(const_iterator(this, descriptorsInsert.first), true); } template std::pair* SwappedMap::prepare(const Key& key) { if (m_items.size() == m_poolSize) { typename std::list::iterator cacheIter = m_cache.begin(); m_items.erase(*cacheIter); m_cacheIterators.erase(*cacheIter); m_cache.erase(cacheIter); } std::pair::iterator, bool> itemInsert = m_items.insert(std::make_pair(key, T())); typename std::list::iterator cacheIter = m_cache.insert(m_cache.end(), key); m_cacheIterators.insert(std::make_pair(key, cacheIter)); return &*itemInsert.first; } template const std::pair* SwappedMap::load(const Key& key, uint64_t offset) { auto itemIterator = m_items.find(key); if (itemIterator != m_items.end()) { auto cacheIteratorsIterator = m_cacheIterators.find(key); if (cacheIteratorsIterator->second != --m_cache.end()) { m_cache.splice(m_cache.end(), m_cache, cacheIteratorsIterator->second); } ++m_cacheHits; return &*itemIterator; } typename std::unordered_map::iterator descriptorsIterator = m_descriptors.find(key); if (descriptorsIterator == m_descriptors.end()) { throw std::runtime_error("SwappedMap::load"); } if (!m_itemsFile) { throw std::runtime_error("SwappedMap::load"); } m_itemsFile.seekg(descriptorsIterator->second.offset); T tempItem; try { boost::archive::binary_iarchive archive(m_itemsFile); archive & tempItem; } catch (std::exception&) { throw std::runtime_error("SwappedMap::load"); } std::pair* item = prepare(key); std::swap(tempItem, item->second); ++m_cacheMisses; return item; }