From 99265240b8f1d6274a66c89f3f55541ae787bad6 Mon Sep 17 00:00:00 2001 From: YuhangQ Date: Sun, 7 Nov 2021 12:57:26 +0800 Subject: [PATCH] add cache --- invodb/btree/node.h | 88 ++++++++++++++++++++++++++---------- invodb/file/page_manager.cpp | 78 +++++++++++++++++--------------- invodb/file/page_manager.h | 11 +++-- invodb/main.cpp | 29 +++++++----- invodb/models/cache.h | 21 +++++++-- invodb/models/collection.cpp | 4 +- 6 files changed, 149 insertions(+), 82 deletions(-) diff --git a/invodb/btree/node.h b/invodb/btree/node.h index 354d576..5145183 100644 --- a/invodb/btree/node.h +++ b/invodb/btree/node.h @@ -2,6 +2,9 @@ // Created by i on 2021/10/24. // + +#define UNLIMITED_MEMORY true + #ifndef INVODB_NODE_H #define INVODB_NODE_H @@ -11,38 +14,30 @@ #include #include #include "file/page_manager.h" -#include "models/cache.h" +#include "invodb/models/cache.h" template class BTreeNode { public: - BTreeNode(const int& address); - static std::shared_ptr> getNode(const int &index); - static BTreeNode* release(const int &index); - int insert(KT const &key); - int findPos(KT const &key); - void update() { - if(leaf) return; - for(int i=0; i<=size; i++) { - auto node = getNode(linkSet[i]); - node->parent = address; - node->save(); - } - } - void release(); - void clear(); - int save(); + static const int m = M_SIZE; static const int maxCount = m - 1; static const int minLeafCount = m / 2; static const int minLinkCount = (m - 1) / 2; - bool enough() { - if(leaf) return size >= minLeafCount; - else return size >= minLinkCount; - } - bool full() { - return size == maxCount; - } + static std::shared_ptr> getNode(const int &index); + static BTreeNode* release(const int &index); + + BTreeNode(const int& address); + + int insert(KT const &key); + int findPos(KT const &key); + void update(); + void release(); + void clear(); + int save(); + bool enough(); + bool full(); + KT keySet[m + 1]; int linkSet[m + 1]; int parent; @@ -51,8 +46,17 @@ public: bool leaf; int size; int address; +private: + //static std::map>> cache; + static LRUCache>> cache; }; +//template +//std::map>> BTreeNode::cache; + +template +LRUCache>> BTreeNode::cache(100000); + template BTreeNode::BTreeNode(const int& address): address(address) { clear(); @@ -94,11 +98,26 @@ std::shared_ptr> BTreeNode::ge if(index == 0) { throw "invalid address!"; } +// if(!cache.exist(index)) { +// auto p = std::make_shared>(index); +// cache.put(index, p); +// return p; +// } else { +// auto p = cache.get(index); +// cache.put(index, p); +// return p; +// } + return std::make_shared>(index); + +// if(cache.count(index) == 0) { +// cache[index] = std::make_shared>(index); +// } } template BTreeNode *BTreeNode::release(const int &index) { + //cache.remove(index); PageManager::Instance().release(index); return nullptr; } @@ -192,4 +211,25 @@ int BTreeNode::save() { return p; } +template +void BTreeNode::update() { + if(leaf) return; + for(int i=0; i<=size; i++) { + auto node = getNode(linkSet[i]); + node->parent = address; + node->save(); + } +} + +template +bool BTreeNode::enough() { + if(leaf) return size >= minLeafCount; + else return size >= minLinkCount; +} + +template +bool BTreeNode::full() { + return size == maxCount; +} + #endif //INVODB_NODE_H diff --git a/invodb/file/page_manager.cpp b/invodb/file/page_manager.cpp index 1438625..013ea2e 100644 --- a/invodb/file/page_manager.cpp +++ b/invodb/file/page_manager.cpp @@ -6,35 +6,37 @@ #include "btree/list.h" - int PageManager::loadDatabase(const char *filename) { Instance().stream.open(filename); Instance().stream.seekp(0, std::ios::end); int index = Instance().stream.tellp() / 1024; if(index == 0) { - StoragePage(0).save(); + StoragePage page(0); + page.setIntStartFrom(4, 2); + page.save(); StoragePage(1).save(); - StoragePage(2).save(); } return 0; } -std::shared_ptr PageManager::getPage(const int &index) { +StoragePage PageManager::getPage(const int &index) { - if(cache.exist(index)) { - return cache.get(index); - } +// if(cache.exist(index)) { +// return *cache.get(index); +// } - auto page = std::make_shared(index); + StoragePage page(index); // 调整指针位置 stream.clear(); stream.seekg((long long)index * 1024); - stream.read(*page, 1024); + stream.read(page, 1024); return page; } void PageManager::setPage(const int &index, const StoragePage &page) { - cache.put(index, std::make_shared(page)); + +// auto p = cache.put(index, std::make_shared(page)); +// if(p == nullptr) return; stream.clear(); stream.seekg((long long)index * 1024); @@ -45,35 +47,39 @@ int PageManager::allocate() { // try to allocate from free block list auto page = getPage(0); - int index = page->getIntStartFrom(0); + int index = page.getIntStartFrom(0); if(index != 0) { auto allocatePage = getPage(index); - page->setIntStartFrom(0, allocatePage->next()); + page.setIntStartFrom(0, allocatePage.next()); // reset - allocatePage->clear(); - allocatePage->save(); - page->save(); - //printf("allocate: %d\n", index); + allocatePage.clear(); + allocatePage.save(); + page.save(); + //printf("1allocate: %d\n", index); return index; } + index = page.getIntStartFrom(4); + //printf("2allocate: %d\n", index); - // allocate block at the end - stream.seekp(0, std::ios::end); - index = stream.tellp() / 1024; + page.setIntStartFrom(4, index + 1); + page.save(); + // allocate block at the end +// stream.seekp(0, std::ios::end); +// index = stream.tellp() / 1024; setPage(index, StoragePage(index)); return index; } void PageManager::release(const int &index, const bool &next) { auto page = getPage(0); - int head = page->getIntStartFrom(0); + int head = page.getIntStartFrom(0); auto releasePage = getPage(index); - releasePage->setNext(head); - page->setIntStartFrom(0, releasePage->getAddress()); - page->save(); - releasePage->save(); + releasePage.setNext(head); + page.setIntStartFrom(0, releasePage.getAddress()); + page.save(); + releasePage.save(); //printf("release: %d\n", index); } @@ -84,11 +90,11 @@ nlohmann::json PageManager::readJSONFromFile(const int &index) { auto page = getPage(index); while(true) { for(int i=0; i<1016; i++) { - if((*page)[i] == '\0') break; - content.push_back((*page)[i]); + if(page[i] == '\0') break; + content.push_back(page[i]); } - if(page->next() == 0) break; - page = getPage(page->next()); + if(page.next() == 0) break; + page = getPage(page.next()); } return nlohmann::json::parse(content); @@ -99,21 +105,21 @@ int PageManager::saveJSONToFile(const nlohmann::json& json) { int size = content.size(); auto page = getPage(allocate()); - int res = page->getAddress(); + int res = page.getAddress(); int p = 0; while(p < size) { int len = std::min(size - p, 1016); - page->setStartFrom(0, &content.c_str()[p], len); - page->save(); + page.setStartFrom(0, &content.c_str()[p], len); + page.save(); p += len; if(p < size) { int newPage = allocate(); - int lastPage = page->getAddress(); - page->setNext(newPage); - page->save(); + int lastPage = page.getAddress(); + page.setNext(newPage); + page.save(); page = getPage(newPage); - page->setLast(lastPage); - page->save(); + page.setLast(lastPage); + page.save(); } } return res; diff --git a/invodb/file/page_manager.h b/invodb/file/page_manager.h index 824ca49..336c3e1 100644 --- a/invodb/file/page_manager.h +++ b/invodb/file/page_manager.h @@ -11,7 +11,7 @@ #include "storage_page.h" #include "json/json.hpp" -#include "models/cache.h" +#include "invodb/models/cache.h" template class List; @@ -23,8 +23,11 @@ public: return instance; } static int loadDatabase(const char *filename); - std::shared_ptr getPage(const int &index); + + StoragePage getPage(const int &index); void setPage(const int &index, const StoragePage &page); + template + static std::shared_ptr getNode(const int &index); int allocate(); void release(const int &index, const bool &next = true); int saveJSONToFile(const nlohmann::json& json); @@ -32,9 +35,9 @@ public: private: std::map map; std::fstream stream; - LRUCache cache; + LRUCache> cache; // 私有化实现单例 - PageManager():cache(LRUCache(100000)) {} + PageManager():cache(LRUCache>(100000)) {} ~PageManager() {} PageManager(const PageManager&); PageManager& operator=(const PageManager&); diff --git a/invodb/main.cpp b/invodb/main.cpp index efb8fe0..0754425 100644 --- a/invodb/main.cpp +++ b/invodb/main.cpp @@ -30,16 +30,23 @@ int main() { } -// freopen("qq.txt", "r", stdin); -// char qq[100], phone[100]; -// for(int i=0; i<400000; i++) { -// if(i % 1000 == 0) printf("[%d/%d] Inserting!\n", i, 400000); -// scanf("%s%s", qq, phone); -// nlohmann::json json; -// json["qq"] = qq; -// json["phone"] = phone; -// col->insert(json); -// } + freopen("qq.txt", "r", stdin); + const int n = 1000000; + char qq[100], phone[100]; + + clock_t start = clock(); + for(int i=0; iinsert(json); + + if(i % (n/1000) == 0) { + printf("[%d/%d] time=%fs!\n", i, n, (double)(clock() - start) / CLOCKS_PER_SEC); + start = clock(); + } + } //col->test(); @@ -57,7 +64,7 @@ int main() { //} // )"); - testAndBenchmark(50000); + testAndBenchmark(20000); return 0; } diff --git a/invodb/models/cache.h b/invodb/models/cache.h index afcf954..4f86db8 100644 --- a/invodb/models/cache.h +++ b/invodb/models/cache.h @@ -7,6 +7,7 @@ #include #include +#include "invodb/file/storage_page.h" template class LRUCache { @@ -17,11 +18,16 @@ public: return hash.find(key) != hash.end(); } - std::shared_ptr get(KT const &key) { + void remove(KT const &key) { + ls.erase(hash[key]); + hash.erase(key); + } + + VT get(KT const &key) { if (hash.find(key) == hash.end()) throw "cache error"; else { - std::shared_ptr value = hash[key]->second; + VT value = hash[key]->second; ls.erase(hash[key]); ls.push_front(std::make_pair(key, value)); hash[key] = ls.begin(); @@ -29,22 +35,27 @@ public: } } - void put(KT const &key, std::shared_ptr const &value) { + VT put(KT const &key, VT const &value) { + VT res; if (hash.find(key) != hash.end()) { ls.erase(hash[key]); } else if (ls.size() >= capacity) { + res = ls.back().second; + printf("fuck\n"); + exit(0); hash.erase(ls.back().first); ls.pop_back(); } ls.push_front(std::make_pair(key, value)); hash[key] = ls.begin(); + return res; } private: int capacity; - std::list>> ls; - std::unordered_map>>::iterator> hash; + std::list> ls; + std::unordered_map>::iterator> hash; }; diff --git a/invodb/models/collection.cpp b/invodb/models/collection.cpp index f87d455..590dc84 100644 --- a/invodb/models/collection.cpp +++ b/invodb/models/collection.cpp @@ -4,7 +4,7 @@ #include "collection.h" -BTree Collection::colList(2); +BTree Collection::colList(1); std::map Collection::map; void Collection::loadCollections() { @@ -231,7 +231,7 @@ void Collection::removeIndex(const std::string indexName, bool indexValue, const void Collection::test() { index->print(); - auto qq = new BTree(8); + auto qq = new BTree(7); while(true) { std::string q; std::cin >> q;