From 6cb59287d78fd388801e25522068d101324ae591 Mon Sep 17 00:00:00 2001 From: YuhangQ Date: Sat, 23 Oct 2021 16:28:57 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9B=E5=BB=BA=20collection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 6 ++- invodb/file/page_manager.cpp | 32 +++---------- invodb/file/page_manager.h | 18 +------- invodb/file/storage_page.cpp | 56 +++++++++++++++++++++++ invodb/file/storage_page.h | 36 +++++++++++++++ invodb/file/virtual_storage.cpp | 47 ------------------- invodb/file/virtual_storage.h | 38 ---------------- invodb/main.cpp | 9 +++- invodb/main.h | 3 +- invodb/models/collection.cpp | 81 +++++++++++++++++++++++++++++++++ invodb/models/collection.h | 21 ++++++++- invodb/models/json.cpp | 9 +++- invodb/models/json.h | 7 +++ invodb/utils/logger.h | 59 ++++++++++++++++++++++++ invodb/utils/uuid.h | 22 +++++++++ 15 files changed, 310 insertions(+), 134 deletions(-) create mode 100644 invodb/file/storage_page.cpp create mode 100644 invodb/file/storage_page.h delete mode 100644 invodb/file/virtual_storage.cpp delete mode 100644 invodb/file/virtual_storage.h create mode 100644 invodb/utils/logger.h create mode 100644 invodb/utils/uuid.h diff --git a/CMakeLists.txt b/CMakeLists.txt index c6d564b..a9ab76e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,8 +3,12 @@ project(InvoDB) set(CMAKE_CXX_STANDARD 14) + + +include_directories(/opt/homebrew/Cellar/boost/1.76.0/include) include_directories(.) +include_directories(./invodb) add_executable(InvoDB invodb/main.cpp - invodb/main.h invodb/file/page_manager.cpp invodb/file/page_manager.h invodb/file/virtual_storage.cpp invodb/file/virtual_storage.h invodb/models/json.cpp invodb/models/json.h invodb/invodb.cpp invodb/invodb.h invodb/models/collection.cpp invodb/models/collection.h) + invodb/main.h invodb/file/page_manager.cpp invodb/file/page_manager.h invodb/models/json.cpp invodb/models/json.h invodb/invodb.cpp invodb/invodb.h invodb/models/collection.cpp invodb/models/collection.h invodb/file/storage_page.cpp invodb/file/storage_page.h invodb/utils/logger.h invodb/utils/uuid.h) diff --git a/invodb/file/page_manager.cpp b/invodb/file/page_manager.cpp index 506335e..4f5face 100644 --- a/invodb/file/page_manager.cpp +++ b/invodb/file/page_manager.cpp @@ -10,7 +10,7 @@ int PageManager::loadDatabase(const char *filename) { } StoragePage PageManager::getPage(const int &index) { - StoragePage page; + StoragePage page(index); // 调整指针位置 stream.clear(); stream.seekg(index * 1024); @@ -24,29 +24,9 @@ void PageManager::setPage(const int &index, const StoragePage &page) { stream.write(page, 1024); } -void StoragePage::print() { - for(int i=0; i<16; i++) { - for(int j=0; j<256; j++) { - printf("%u ", page[i * 16 + j]); - } - printf("\n"); - } +int PageManager::allocate() { + stream.seekp(0, std::ios::end); + int index = stream.tellp() / 1024; + setPage(index, StoragePage(index)); + return index; } - -/** - * 获取下一个连接的页，返回零则到达链结尾 - * @return - */ -int StoragePage::next() { - return *((const int *)&page[1020]); -} - -int StoragePage::setNext(const int &nextPage) { - *((int *)&page[1020]) = nextPage; - return 0; -} - -int *StoragePage::intArray() { - return (int *)page; -} - diff --git a/invodb/file/page_manager.h b/invodb/file/page_manager.h index 055bcb0..c90576f 100644 --- a/invodb/file/page_manager.h +++ b/invodb/file/page_manager.h @@ -7,21 +7,8 @@ #include #include -#include -class StoragePage { -public: - void print(); - int next(); - int setNext(const int& nextPage); - int* intArray(); - StoragePage() { memset(page, 0, sizeof(page)); } - char& operator[] (int index) { return this->page[index]; } - operator const char *() const { return this->page; } - operator char *() { return this->page; } -private: - char page[1024]; -}; +#include "storage_page.h" class PageManager { public: @@ -33,10 +20,9 @@ public: StoragePage getPage(const int &index); void setPage(const int &index, const StoragePage &page); int allocate(); - + void free(const int &index); private: std::fstream stream; - // 私有化实现单例 PageManager() {} ~PageManager() {} diff --git a/invodb/file/storage_page.cpp b/invodb/file/storage_page.cpp new file mode 100644 index 0000000..3d3ef34 --- /dev/null +++ b/invodb/file/storage_page.cpp @@ -0,0 +1,56 @@ +// +// Created by YuhangQ on 2021/10/22. +// + +#include "storage_page.h" +#include "page_manager.h" + + +void StoragePage::print() { + printf("-------------------PageInfo--------------------\n"); + for(int i=0; i<8; i++) { + for(int j=0; j<32; j++) { + printf("%u ", intArray()[i*32+j]); + } + printf("\n"); + } +} + +int StoragePage::next() { + return getIntStartFrom(1020); +} + +void StoragePage::setNext(const int& nextPage) { + setIntStartFrom(1020, nextPage); +} + +int StoragePage::last() { + return getIntStartFrom(1016); +} + +void StoragePage::setLast(const int& lastPage) { + setIntStartFrom(1016, lastPage); +} + +int *StoragePage::intArray() { + return (int *)page; +} + +int StoragePage::getIntStartFrom(const int& index) { + return *((const int *)&page[index]); +} +void StoragePage::setIntStartFrom(const int& index, const int& value) { + *((int *)&page[index]) = value; +} + +void StoragePage::setStringStartFrom(const int &index, const char *str) { + for(int i=0; i +#include + + +class PageManager; + +class StoragePage { +public: + void print(); + int next(); + void setNext(const int& nextPage); + int last(); + void setLast(const int& lastPage); + void save(); + int getIntStartFrom(const int &index); + void setIntStartFrom(const int &index, const int &value); + void setStringStartFrom(const int &index, const char *str); + int *intArray(); + StoragePage(const int& id) { memset(page, 0, sizeof(page)); } + char& operator[] (int index) { return this->page[index]; } + operator const char *() const { return this->page; } + operator char *() { return this->page; } +private: + char page[1024]; + int address; +}; + + +#endif //INVODB_STORAGE_PAGE_H diff --git a/invodb/file/virtual_storage.cpp b/invodb/file/virtual_storage.cpp deleted file mode 100644 index cde0183..0000000 --- a/invodb/file/virtual_storage.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// -// Created by YuhangQ on 2021/9/30. -// - -#include "virtual_storage.h" - -int VirtualStorage::loadDatabase(const char *filename) { - int result = PageManager::loadDatabase(filename); - return result; -} - -StoragePage VirtualStorage::getPage(const int &index) { - PageManager& manager = PageManager::Instance(); - return manager.getPage(getPhysicalAddress(index)); -} - -void VirtualStorage::setPage(const int &index, const StoragePage &page) { - PageManager& manager = PageManager::Instance(); - manager.setPage(getPhysicalAddress(index), page); -} - -int VirtualStorage::getPhysicalAddress(const int &index) { - loadPageInfoTable(index >> 2); - return (table[index >> 2].addr << 2) || (index & 3); -} - -int VirtualStorage::getNextAddress(const int &index) { - loadPageInfoTable(index >> 2); - return table[index >> 2].next; -} - -void VirtualStorage::loadPageInfoTable(const int &index) { - PageManager& manager = PageManager::Instance(); - // 未命中 cache - if(table.count(index) == 0) { - int pageIndex = 1 + index / 512; - // 加载页表 - StoragePage page = manager.getPage(pageIndex); - int *arr = page.intArray(); - for(int i=0; i<512; i++) { - table[(pageIndex - 1) * 512 + i] = TableEl{ arr[2*i], arr[2*i+1] }; - } - } -} - - - diff --git a/invodb/file/virtual_storage.h b/invodb/file/virtual_storage.h deleted file mode 100644 index 6b380d2..0000000 --- a/invodb/file/virtual_storage.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Created by YuhangQ on 2021/9/30. -// - -#ifndef INVODB_VIRTUAL_STORAGE_H -#define INVODB_VIRTUAL_STORAGE_H - -#include -#include "page_manager.h" - -class VirtualStorage { -public: - static VirtualStorage& Instance() { - static VirtualStorage instance; - return instance; - } - static int loadDatabase(const char *filename); - StoragePage getPage(const int &index); - void setPage(const int &index, const StoragePage &page); - int allocate(); -private: - struct TableEl { - int addr; - int next; - }; - // 页表 Cache - std::map table; - int getPhysicalAddress(const int &index); - int getNextAddress(const int &index); - void loadPageInfoTable(const int &index); - // 私有化实现单例 - VirtualStorage() {} - ~VirtualStorage() {} - VirtualStorage(const PageManager&); - VirtualStorage& operator=(const PageManager&); -}; - -#endif //INVODB_VIRTUAL_STORAGE_H diff --git a/invodb/main.cpp b/invodb/main.cpp index 45f0635..5f03c6e 100644 --- a/invodb/main.cpp +++ b/invodb/main.cpp @@ -5,12 +5,17 @@ #include "main.h" int main() { - VirtualStorage::loadDatabase("test.invodb"); - VirtualStorage& storage = VirtualStorage::Instance(); + PageManager::loadDatabase("test.invodb"); + Collection::loadCollections(); + + PageManager& manager = PageManager::Instance(); + Collection &col = Collection::getCollection("hello"); + JSON json("{\"hello\": 1}"); + col.insert(json); return 0; } \ No newline at end of file diff --git a/invodb/main.h b/invodb/main.h index 50ec8eb..9ad4342 100644 --- a/invodb/main.h +++ b/invodb/main.h @@ -6,7 +6,6 @@ #define INVODB_MAIN_H #include -#include "file/virtual_storage.h" -#include "invodb/models/json.h" +#include "models/collection.h" #endif //INVODB_MAIN_H diff --git a/invodb/models/collection.cpp b/invodb/models/collection.cpp index 3806c26..b0597ef 100644 --- a/invodb/models/collection.cpp +++ b/invodb/models/collection.cpp @@ -3,3 +3,84 @@ // #include "collection.h" +#include "utils/uuid.h" + +Collection::Collection(const std::string &name, const int &firstPage) { + Logger::info("load Collection: ", name); +} + +void Collection::insert(JSON &json) { + + if(!json.HasMember(" __Invo_ID__")) { + std::string uuid = generateUUID(); + Document::AllocatorType &allocator = json.GetAllocator(); + Value invoid (kStringType); + invoid.SetString(uuid.c_str(), uuid.size()); + json.AddMember(" __Invo_ID__", invoid, allocator); + } + + Logger::info(json.ToString()); +} + + +std::map Collection::map; +std::set Collection::free; + +void Collection::loadCollections() { + // 前四页为集合信息页 + for (int id = 0; id < 4; id++) { + StoragePage page = PageManager::Instance().getPage(id); + PageManager::Instance().setPage(id, page); + for (int i = 0; i < 32; i++) { + int p = i * 32; + int len = strlen(&page[p]); + std::string name(&page[p], len > 28 ? 28 : len); + int firstPage = page.getIntStartFrom(p + 28); + // if free + if (firstPage == 0) free.insert(id * 32 + i); + // not free + else map.insert(make_pair(name, new Collection(name, firstPage))); + } + } + Logger::info("Successfully load Collections: ", 128 - free.size()); +} + +Collection& Collection::createCollection(const std::string &name) { + // exist + if(map.count(name) != 0) { + throw "collection has already exist"; + } + // no free line + if(free.size() == 0) { + throw "you are reach the max limit count of collections"; + } + int id = *free.begin(); + free.erase(free.begin()); + + StoragePage page = PageManager::Instance().getPage(id / 32); + id %= 32; + + int collectionPage = PageManager::Instance().allocate(); + + if(name.size() > 28) { + throw "too long name of collection"; + } + + page.setStringStartFrom(id*32, name.c_str()); + page.setIntStartFrom(id*32+28, collectionPage); + page.save(); + + Collection *col = new Collection(name, collectionPage); + + map.insert(make_pair(name, col)); + + return *col; +} + +Collection &Collection::getCollection(const std::string &name) { + if(map.count(name) == 0) { + throw "no such collection"; + } + return *map[name]; +} + diff --git a/invodb/models/collection.h b/invodb/models/collection.h index 4d37b8b..08d2b7b 100644 --- a/invodb/models/collection.h +++ b/invodb/models/collection.h @@ -5,9 +5,28 @@ #ifndef INVODB_COLLECTION_H #define INVODB_COLLECTION_H +#include "file/page_manager.h" +#include "utils/logger.h" +#include "json.h" +#include +#include +#include +#include class Collection { - +public: + static void insert(JSON &json); + static void loadCollections(); + static Collection& getCollection(const std::string& name); + static Collection& createCollection(const std::string& name); +private: + static std::map map; + static std::set free; + Collection(const std::string& name,const int& firstPage); + Collection() {} + ~Collection() {} + Collection(const Collection&); + Collection& operator=(const Collection&); }; diff --git a/invodb/models/json.cpp b/invodb/models/json.cpp index ab8fa56..ac3ee63 100644 --- a/invodb/models/json.cpp +++ b/invodb/models/json.cpp @@ -2,4 +2,11 @@ // Created by YuhangQ on 2021/10/9. // -#include "json.h" \ No newline at end of file +#include "json.h" + +std::string JSON::ToString() { + StringBuffer buffer; + Writer writer(buffer); + this->Accept(writer); + return buffer.GetString(); +} diff --git a/invodb/models/json.h b/invodb/models/json.h index 16fbf6c..9dbfe2c 100644 --- a/invodb/models/json.h +++ b/invodb/models/json.h @@ -7,11 +7,18 @@ #include #include "rapidjson/document.h" +#include "rapidjson/writer.h" +#include "rapidjson/stringbuffer.h" using namespace rapidjson; class JSON : public Document { +public: + JSON(std::string json): Document() { this->Parse(json.c_str()); } + JSON(const char *json): Document() { this->Parse(json); } + std::string ToString(); +private: }; diff --git a/invodb/utils/logger.h b/invodb/utils/logger.h new file mode 100644 index 0000000..22d2afd --- /dev/null +++ b/invodb/utils/logger.h @@ -0,0 +1,59 @@ +// +// Created by YuhangQ on 2021/10/22. +// + +#ifndef INVODB_LOGGER_H +#define INVODB_LOGGER_H + +#include +#include + +class Logger { +public: + + template + static void info(const T& msg); + + template + static void info(const T& first, const K& second); + template + static void warn(const T& msg); + template + static void warn(const T& first, const K& second); + template + static void error(const T& msg); + template + static void error(const T& first, const K& second); +}; + +template +void Logger::info(const T &msg) { + std::cout << "[INFO] " << msg << std::endl; +} + +template +void Logger::info(const T &first, const K &second) { + std::cout << "[INFO] " << first << second << std::endl; +} + +template +void Logger::warn(const T &msg) { + std::cout << "[WARN] " << msg << std::endl; +} + +template +void Logger::warn(const T &first, const K &second) { + std::cout << "[WARN] " << first << second << std::endl; +} + +template +void Logger::error(const T &msg) { + std::cout << "[ERROR] " << msg << std::endl; +} + +template +void Logger::error(const T &first, const K &second) { + std::cout << "[ERROR] " << first << second << std::endl; +} + +#endif //INVODB_LOGGER_H diff --git a/invodb/utils/uuid.h b/invodb/utils/uuid.h new file mode 100644 index 0000000..32ab157 --- /dev/null +++ b/invodb/utils/uuid.h @@ -0,0 +1,22 @@ +// +// Created by YuhangQ on 2021/10/23. +// + +#ifndef INVODB_UUID_H +#define INVODB_UUID_H + +#include +#include +#include +#include +#include + +std::string generateUUID() { + boost::uuids::uuid a_uuid = boost::uuids::random_generator()(); + std::string uuid_string = boost::uuids::to_string(a_uuid); + std::remove(uuid_string.begin(), uuid_string.end(), '-'); + uuid_string.resize(32); + return uuid_string; +} + +#endif //INVODB_UUID_H