add cache

invodb/btree/node.h

@@ -2,6 +2,9 @@
 // Created by i on 2021/10/24.
 //
 
+
+#define UNLIMITED_MEMORY true
+
 #ifndef INVODB_NODE_H
 #define INVODB_NODE_H
 
@@ -11,38 +14,30 @@
 #include <map>
 #include <type_traits>
 #include "file/page_manager.h"
-#include "models/cache.h"
+#include "invodb/models/cache.h"
 
 template<int M_SIZE, typename KT, int K_SIZE>
 class BTreeNode {
 public:
-    BTreeNode(const int& address);
-    static std::shared_ptr<BTreeNode<M_SIZE, KT, K_SIZE>> getNode(const int &index);
-    static BTreeNode<M_SIZE, KT, K_SIZE>* release(const int &index);
-    int insert(KT const &key);
-    int findPos(KT const &key);
-    void update() {
-        if(leaf) return;
-        for(int i=0; i<=size; i++) {
-            auto node = getNode(linkSet[i]);
-            node->parent = address;
-            node->save();
-        }
-    }
-    void release();
-    void clear();
-    int save();
+
     static const int m = M_SIZE;
     static const int maxCount = m - 1;
     static const int minLeafCount = m / 2;
     static const int minLinkCount = (m - 1) / 2;
-    bool enough() {
-        if(leaf) return size >= minLeafCount;
-        else return size >= minLinkCount;
-    }
-    bool full() {
-        return size == maxCount;
-    }
+    static std::shared_ptr<BTreeNode<M_SIZE, KT, K_SIZE>> getNode(const int &index);
+    static BTreeNode<M_SIZE, KT, K_SIZE>* release(const int &index);
+
+    BTreeNode(const int& address);
+
+    int insert(KT const &key);
+    int findPos(KT const &key);
+    void update();
+    void release();
+    void clear();
+    int save();
+    bool enough();
+    bool full();
+
     KT keySet[m + 1];
     int linkSet[m + 1];
     int parent;
@@ -51,8 +46,17 @@ public:
     bool leaf;
     int size;
     int address;
+private:
+    //static std::map<int, std::shared_ptr<BTreeNode<M_SIZE, KT, K_SIZE>>> cache;
+    static LRUCache<int, std::shared_ptr<BTreeNode<M_SIZE, KT, K_SIZE>>> cache;
 };
 
+//template<int M_SIZE, typename KT, int K_SIZE>
+//std::map<int, std::shared_ptr<BTreeNode<M_SIZE, KT, K_SIZE>>> BTreeNode<M_SIZE, KT, K_SIZE>::cache;
+
+template<int M_SIZE, typename KT, int K_SIZE>
+LRUCache<int, std::shared_ptr<BTreeNode<M_SIZE, KT, K_SIZE>>> BTreeNode<M_SIZE, KT, K_SIZE>::cache(100000);
+
 template<int M_SIZE, typename KT, int K_SIZE>
 BTreeNode<M_SIZE, KT, K_SIZE>::BTreeNode(const int& address): address(address) {
     clear();
@@ -94,11 +98,26 @@ std::shared_ptr<BTreeNode<M_SIZE, KT, K_SIZE>> BTreeNode<M_SIZE, KT, K_SIZE>::ge
     if(index == 0) {
         throw "invalid address!";
     }
+//    if(!cache.exist(index)) {
+//        auto p = std::make_shared<BTreeNode<M_SIZE, KT, K_SIZE>>(index);
+//        cache.put(index, p);
+//        return p;
+//    } else {
+//        auto p = cache.get(index);
+//        cache.put(index, p);
+//        return p;
+//    }
+
     return std::make_shared<BTreeNode<M_SIZE, KT, K_SIZE>>(index);
+
+//    if(cache.count(index) == 0) {
+//        cache[index] = std::make_shared<BTreeNode<M_SIZE, KT, K_SIZE>>(index);
+//    }
 }
 
 template<int M_SIZE, typename KT, int K_SIZE>
 BTreeNode<M_SIZE, KT, K_SIZE> *BTreeNode<M_SIZE, KT, K_SIZE>::release(const int &index) {
+    //cache.remove(index);
     PageManager::Instance().release(index);
     return nullptr;
 }
@@ -192,4 +211,25 @@ int BTreeNode<M_SIZE, KT, K_SIZE>::save() {
     return p;
 }
 
+template<int M_SIZE, typename KT, int K_SIZE>
+void BTreeNode<M_SIZE, KT, K_SIZE>::update() {
+    if(leaf) return;
+    for(int i=0; i<=size; i++) {
+        auto node = getNode(linkSet[i]);
+        node->parent = address;
+        node->save();
+    }
+}
+
+template<int M_SIZE, typename KT, int K_SIZE>
+bool BTreeNode<M_SIZE, KT, K_SIZE>::enough() {
+    if(leaf) return size >= minLeafCount;
+    else return size >= minLinkCount;
+}
+
+template<int M_SIZE, typename KT, int K_SIZE>
+bool BTreeNode<M_SIZE, KT, K_SIZE>::full() {
+    return size == maxCount;
+}
+
 #endif //INVODB_NODE_H

invodb/file/page_manager.cpp

@@ -6,35 +6,37 @@
 #include "btree/list.h"
 
 
-
 int PageManager::loadDatabase(const char *filename) {
     Instance().stream.open(filename);
     Instance().stream.seekp(0, std::ios::end);
     int index = Instance().stream.tellp() / 1024;
     if(index == 0) {
-        StoragePage(0).save();
+        StoragePage page(0);
+        page.setIntStartFrom(4, 2);
+        page.save();
         StoragePage(1).save();
-        StoragePage(2).save();
     }
     return 0;
 }
 
-std::shared_ptr<StoragePage> PageManager::getPage(const int &index) {
+StoragePage PageManager::getPage(const int &index) {
 
-    if(cache.exist(index)) {
-        return cache.get(index);
-    }
+//    if(cache.exist(index)) {
+//        return *cache.get(index);
+//    }
 
-    auto page = std::make_shared<StoragePage>(index);
+    StoragePage page(index);
     // 调整指针位置
     stream.clear();
     stream.seekg((long long)index * 1024);
-    stream.read(*page, 1024);
+    stream.read(page, 1024);
     return page;
 }
 
 void PageManager::setPage(const int &index, const StoragePage &page) {
-    cache.put(index, std::make_shared<StoragePage>(page));
+
+//    auto p = cache.put(index, std::make_shared<StoragePage>(page));
+//    if(p == nullptr) return;
 
     stream.clear();
     stream.seekg((long long)index * 1024);
@@ -45,35 +47,39 @@ int PageManager::allocate() {
 
 //     try to allocate from free block list
     auto page = getPage(0);
-    int index =  page->getIntStartFrom(0);
+    int index =  page.getIntStartFrom(0);
     if(index != 0) {
         auto allocatePage = getPage(index);
 
-        page->setIntStartFrom(0, allocatePage->next());
+        page.setIntStartFrom(0, allocatePage.next());
         // reset
-        allocatePage->clear();
-        allocatePage->save();
-        page->save();
-        //printf("allocate: %d\n", index);
+        allocatePage.clear();
+        allocatePage.save();
+        page.save();
+        //printf("1allocate: %d\n", index);
         return index;
     }
 
+    index = page.getIntStartFrom(4);
+    //printf("2allocate: %d\n", index);
 
-    // allocate block at the end
-    stream.seekp(0, std::ios::end);
-    index = stream.tellp() / 1024;
+    page.setIntStartFrom(4, index + 1);
+    page.save();
+    //      allocate block at the end
+//    stream.seekp(0, std::ios::end);
+//    index = stream.tellp() / 1024;
     setPage(index, StoragePage(index));
     return index;
 }
 
 void PageManager::release(const int &index, const bool &next) {
     auto page = getPage(0);
-    int head = page->getIntStartFrom(0);
+    int head = page.getIntStartFrom(0);
     auto releasePage = getPage(index);
-    releasePage->setNext(head);
-    page->setIntStartFrom(0, releasePage->getAddress());
-    page->save();
-    releasePage->save();
+    releasePage.setNext(head);
+    page.setIntStartFrom(0, releasePage.getAddress());
+    page.save();
+    releasePage.save();
     //printf("release: %d\n", index);
 
 }
@@ -84,11 +90,11 @@ nlohmann::json PageManager::readJSONFromFile(const int &index) {
     auto page = getPage(index);
     while(true) {
         for(int i=0; i<1016; i++) {
-            if((*page)[i] == '\0') break;
-            content.push_back((*page)[i]);
+            if(page[i] == '\0') break;
+            content.push_back(page[i]);
         }
-        if(page->next() == 0) break;
-        page = getPage(page->next());
+        if(page.next() == 0) break;
+        page = getPage(page.next());
     }
 
     return nlohmann::json::parse(content);
@@ -99,21 +105,21 @@ int PageManager::saveJSONToFile(const nlohmann::json& json) {
     int size = content.size();
 
     auto page = getPage(allocate());
-    int res = page->getAddress();
+    int res = page.getAddress();
     int p = 0;
     while(p < size) {
         int len = std::min(size - p, 1016);
-        page->setStartFrom(0, &content.c_str()[p], len);
-        page->save();
+        page.setStartFrom(0, &content.c_str()[p], len);
+        page.save();
         p += len;
         if(p < size) {
             int newPage = allocate();
-            int lastPage = page->getAddress();
-            page->setNext(newPage);
-            page->save();
+            int lastPage = page.getAddress();
+            page.setNext(newPage);
+            page.save();
             page = getPage(newPage);
-            page->setLast(lastPage);
-            page->save();
+            page.setLast(lastPage);
+            page.save();
         }
     }
     return res;

invodb/file/page_manager.h

@@ -11,7 +11,7 @@
 
 #include "storage_page.h"
 #include "json/json.hpp"
-#include "models/cache.h"
+#include "invodb/models/cache.h"
 
 template<typename T, int T_SIZE>
 class List;
@@ -23,8 +23,11 @@ public:
         return instance;
     }
     static int loadDatabase(const char *filename);
-    std::shared_ptr<StoragePage> getPage(const int &index);
+
+    StoragePage getPage(const int &index);
     void setPage(const int &index, const StoragePage &page);
+    template<typename T>
+    static std::shared_ptr<T> getNode(const int &index);
     int allocate();
     void release(const int &index, const bool &next = true);
     int saveJSONToFile(const nlohmann::json& json);
@@ -32,9 +35,9 @@ public:
 private:
     std::map<int, StoragePage> map;
     std::fstream stream;
-    LRUCache<int, StoragePage> cache;
+    LRUCache<int, std::shared_ptr<StoragePage>> cache;
     // 私有化实现单例
-    PageManager():cache(LRUCache<int, StoragePage>(100000)) {}
+    PageManager():cache(LRUCache<int, std::shared_ptr<StoragePage>>(100000)) {}
     ~PageManager() {}
     PageManager(const PageManager&);
     PageManager& operator=(const PageManager&);

invodb/main.cpp

@@ -30,16 +30,23 @@ int main() {
     }
 
 
-//    freopen("qq.txt", "r", stdin);
-//    char qq[100], phone[100];
-//    for(int i=0; i<400000; i++) {
-//        if(i % 1000 == 0) printf("[%d/%d] Inserting!\n", i, 400000);
-//        scanf("%s%s", qq, phone);
-//        nlohmann::json json;
-//        json["qq"] = qq;
-//        json["phone"] = phone;
-//        col->insert(json);
-//    }
+    freopen("qq.txt", "r", stdin);
+    const int n = 1000000;
+    char qq[100], phone[100];
+
+    clock_t start = clock();
+    for(int i=0; i<n; i++) {
+        scanf("%s%s", qq, phone);
+        nlohmann::json json;
+        json["qq"] = qq;
+        json["phone"] = phone;
+        col->insert(json);
+
+        if(i % (n/1000) == 0) {
+            printf("[%d/%d] time=%fs!\n", i, n, (double)(clock() - start) / CLOCKS_PER_SEC);
+            start = clock();
+        }
+    }
 
     //col->test();
 
@@ -57,7 +64,7 @@ int main() {
 //}
 //    )");
 
-    testAndBenchmark(50000);
+    testAndBenchmark(20000);
 
     return 0;
 }

invodb/models/cache.h

@@ -7,6 +7,7 @@
 
 #include <list>
 #include <map>
+#include "invodb/file/storage_page.h"
 
 template<typename KT, typename VT>
 class LRUCache {
@@ -17,11 +18,16 @@ public:
         return hash.find(key) != hash.end();
     }
 
-    std::shared_ptr<VT> get(KT const &key) {
+    void remove(KT const &key) {
+        ls.erase(hash[key]);
+        hash.erase(key);
+    }
+
+    VT get(KT const &key) {
         if (hash.find(key) == hash.end())
             throw "cache error";
         else {
-            std::shared_ptr<VT> value = hash[key]->second;
+            VT value = hash[key]->second;
             ls.erase(hash[key]);
             ls.push_front(std::make_pair(key, value));
             hash[key] = ls.begin();
@@ -29,22 +35,27 @@ public:
         }
     }
 
-    void put(KT const &key, std::shared_ptr<VT> const &value) {
+    VT put(KT const &key, VT const &value) {
+        VT res;
         if (hash.find(key) != hash.end()) {
             ls.erase(hash[key]);
         }
         else if (ls.size() >= capacity) {
+            res = ls.back().second;
+            printf("fuck\n");
+            exit(0);
             hash.erase(ls.back().first);
             ls.pop_back();
         }
         ls.push_front(std::make_pair(key, value));
         hash[key] = ls.begin();
+        return res;
     }
 
 private:
     int capacity;
-    std::list<std::pair<KT, std::shared_ptr<VT>>> ls;
-    std::unordered_map<KT, typename std::list<std::pair<KT, std::shared_ptr<VT>>>::iterator> hash;
+    std::list<std::pair<KT, VT>> ls;
+    std::unordered_map<KT, typename std::list<std::pair<KT, VT>>::iterator> hash;
 };
 
 

invodb/models/collection.cpp

@@ -4,7 +4,7 @@
 
 #include "collection.h"
 
-BTree<std::string, 32> Collection::colList(2);
+BTree<std::string, 32> Collection::colList(1);
 std::map<std::string, Collection*> Collection::map;
 
 void Collection::loadCollections() {
@@ -231,7 +231,7 @@ void Collection::removeIndex(const std::string indexName, bool indexValue, const
 
 void Collection::test() {
     index->print();
-    auto qq = new BTree<std::string, 128>(8);
+    auto qq = new BTree<std::string, 128>(7);
     while(true) {
         std::string q;
         std::cin >> q;