From 097a09efd9556b22917990ed37fbff3265b24260 Mon Sep 17 00:00:00 2001
From: Ferenc Szontágh <szf@fsociety.hu>
Date: Thu, 27 Jun 2024 18:40:38 +0000
Subject: [PATCH] initial
---
external/tser | 1
.gitignore | 9
main.cpp | 47 +++++
.gitmodules | 6
test_RocksDBWrapper.cpp | 114 ++++++++++++
RocksDBWrapper.cpp | 73 ++++++++
RocksDBWrapper.h | 225 +++++++++++++++++++++++++
CMakeLists.txt | 44 ++++
8 files changed, 516 insertions(+), 3 deletions(-)
diff --git a/.gitignore b/.gitignore
index 0def275..303e39c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,19 +3,22 @@
*.lo
*.o
*.obj
-
+
# Compiled Dynamic libraries
*.so
*.dylib
*.dll
-
+
# Compiled Static libraries
*.lai
*.la
*.a
*.lib
-
+
# Executables
*.exe
*.out
*.app
+
+build
+.vscode
\ No newline at end of file
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..11b2e2d
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "external/tser"]
+ path = external/tser
+ url = https://github.com/KonanM/tser.git
+[submodule "external/wildcards"]
+ path = external/wildcards
+ url = https://github.com/zemasoft/wildcards.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000..8c574b4
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.10)
+project(RocksDBWrapper)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+
+include_directories(${CMAKE_SOURCE_DIR}/external/tser/include)
+
+# Add the source files
+set(SOURCES
+ RocksDBWrapper.cpp
+)
+
+# Find RocksDB
+find_package(RocksDB REQUIRED)
+
+# Create the library target
+add_library(RocksDBWrapper STATIC ${SOURCES})
+
+# Link the RocksDB library
+target_link_libraries(RocksDBWrapper RocksDB::rocksdb)
+
+# Add the executable for testing
+add_executable(RocksDBWrapperExecutable main.cpp)
+target_link_libraries(RocksDBWrapperExecutable RocksDBWrapper)
+
+# Optionally add tests
+option(BUILD_TESTS "Build tests" ON)
+
+if (BUILD_TESTS)
+ include(FetchContent)
+ FetchContent_Declare(
+ googletest
+ URL https://github.com/google/googletest/archive/refs/tags/release-1.11.0.zip
+ )
+ FetchContent_MakeAvailable(googletest)
+
+ enable_testing()
+
+ add_executable(RocksDBWrapperTest test_RocksDBWrapper.cpp)
+ target_link_libraries(RocksDBWrapperTest RocksDBWrapper gtest gtest_main)
+
+ add_test(NAME RocksDBWrapperTest COMMAND RocksDBWrapperTest)
+endif()
diff --git a/RocksDBWrapper.cpp b/RocksDBWrapper.cpp
new file mode 100644
index 0000000..f67139e
--- /dev/null
+++ b/RocksDBWrapper.cpp
@@ -0,0 +1,73 @@
+#include "RocksDBWrapper.h"
+
+RocksDBWrapper::RocksDBWrapper(const std::string &db_path, const std::string &index_path)
+{
+ rocksdb::Options options;
+ options.create_if_missing = true;
+
+ // Open the main RocksDB database
+ rocksdb::Status status = rocksdb::DB::Open(options, db_path, &db_);
+ if (!status.ok())
+ {
+ std::cerr << "Error opening RocksDB database: " << status.ToString() << std::endl;
+ }
+
+ // Open the index RocksDB database
+ status = rocksdb::DB::Open(options, index_path, &index_db_);
+ if (!status.ok())
+ {
+ std::cerr << "Error opening RocksDB index database: " << status.ToString() << std::endl;
+ }
+}
+
+RocksDBWrapper::~RocksDBWrapper()
+{
+ delete db_;
+ delete index_db_;
+}
+
+bool RocksDBWrapper::keyExists(const std::string &key)
+{
+ std::string value;
+ rocksdb::Status status = db_->Get(rocksdb::ReadOptions(), serializeKey(key), &value);
+ return status.ok();
+}
+
+bool RocksDBWrapper::StringMatch(const std::string &text, const std::string &pattern)
+{
+ int n = text.length();
+ int m = pattern.length();
+ int i = 0, j = 0, startIndex = -1, match = 0;
+
+ while (i < n)
+ {
+ if (j < m && (pattern[j] == '?' || pattern[j] == text[i]))
+ {
+ i++;
+ j++;
+ }
+ else if (j < m && pattern[j] == '*')
+ {
+ startIndex = j;
+ match = i;
+ j++;
+ }
+ else if (startIndex != -1)
+ {
+ j = startIndex + 1;
+ match++;
+ i = match;
+ }
+ else
+ {
+ return false;
+ }
+ }
+
+ while (j < m && pattern[j] == '*')
+ {
+ j++;
+ }
+
+ return j == m;
+}
diff --git a/RocksDBWrapper.h b/RocksDBWrapper.h
new file mode 100644
index 0000000..f4993fc
--- /dev/null
+++ b/RocksDBWrapper.h
@@ -0,0 +1,225 @@
+#include <rocksdb/db.h>
+#include <rocksdb/options.h>
+#include <iostream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <regex>
+#include "tser/tser.hpp"
+
+// Wrapper class for RocksDB
+class RocksDBWrapper
+{
+public:
+ RocksDBWrapper(const std::string &db_path, const std::string &index_path);
+
+ ~RocksDBWrapper();
+
+ template <typename T>
+ void store(const std::string &key, const T &value)
+ {
+ tser::BinaryArchive archive(0);
+ archive.save(value);
+
+ rocksdb::Status status = db_->Put(rocksdb::WriteOptions(), serializeKey(key), archive.get_buffer().data());
+ if (!status.ok())
+ {
+ std::cerr << "Error storing value in RocksDB database: " << status.ToString() << std::endl;
+ }
+
+ // Index the struct's members
+ index_members(serializeKey(key), value);
+ }
+
+ template <typename T>
+ bool get(const std::string &key, T &value)
+ {
+ std::string serialized_value;
+ rocksdb::Status status = db_->Get(rocksdb::ReadOptions(), serializeKey(key), &serialized_value);
+ if (!status.ok())
+ {
+ std::cerr << "Error getting value from RocksDB database: " << status.ToString() << std::endl;
+ return false;
+ }
+
+ tser::BinaryArchive archive(0);
+ archive.initialize(serialized_value);
+ value = archive.load<T>();
+ return true;
+ }
+
+ template <typename T>
+ std::vector<std::string> search(const std::string &member_name, const std::string &member_value)
+ {
+ std::vector<std::string> keys;
+ std::string prefix = member_name + ":" + serializeKey(member_value) + ":";
+ rocksdb::Iterator *it = index_db_->NewIterator(rocksdb::ReadOptions());
+ for (it->Seek(prefix); it->Valid() && it->key().ToString().find(prefix) == 0; it->Next())
+ {
+ keys.push_back(it->value().ToString());
+ }
+ delete it;
+ return keys;
+ }
+
+ inline std::vector<std::string> split(const std::string &s, const std::string &delimiter)
+ {
+ size_t last = 0;
+ size_t next = 0;
+ std::vector<std::string> results;
+ while ((next = s.find(delimiter, last)) != std::string::npos)
+ {
+ results.emplace_back(s.substr(last, next - last));
+ last = next + delimiter.length();
+ }
+ results.emplace_back(s.substr(last));
+ return results;
+ }
+
+ template <typename T>
+ void search_text(const std::string &search_value, std::vector<T> &res)
+ {
+ std::vector<std::string> keys;
+ std::vector<std::string> tmp;
+ rocksdb::Iterator *it = index_db_->NewIterator(rocksdb::ReadOptions());
+
+ for (it->SeekToFirst(); it->Valid(); it->Next())
+ {
+ std::string key = it->key().ToString();
+ auto _split = this->split(key, ":");
+ if (this->StringMatch(_split[1], search_value))
+ {
+ keys.push_back(_split[2]);
+ }
+ }
+
+ std::vector<rocksdb::Slice> slices;
+ for (const auto &key : keys)
+ {
+ slices.push_back(rocksdb::Slice(key));
+ }
+
+ std::vector<rocksdb::Status> statuses = db_->MultiGet(rocksdb::ReadOptions(), slices, &tmp);
+
+ for (const auto &_v : tmp)
+ {
+ T _val;
+ this->DeSerialize(_v, _val);
+ res.emplace_back(_val);
+ }
+ delete it;
+ }
+
+ // New keyExists method
+ bool keyExists(const std::string &key);
+
+ // New conditional search method
+ template <typename T>
+ void search_conditional(std::vector<T> &res, std::function<bool(const T &)> condition)
+ {
+ rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions());
+
+ for (it->SeekToFirst(); it->Valid(); it->Next())
+ {
+ std::string key = it->key().ToString();
+ std::string serialized_value = it->value().ToString();
+
+ T _val;
+ this->DeSerialize(serialized_value, _val);
+
+ if (condition(_val))
+ {
+ res.push_back(_val);
+ }
+ }
+ delete it;
+ }
+
+private:
+ rocksdb::DB *db_;
+ rocksdb::DB *index_db_;
+
+ bool StringMatch(const std::string &text, const std::string &pattern);
+
+ template <typename T>
+ inline void DeSerialize(const std::string &data, T &value)
+ {
+ tser::BinaryArchive archive(0);
+ archive.initialize(data);
+ value = archive.load<T>();
+ }
+
+ template <typename T>
+ inline void Serialize(const T &value, std::string &data)
+ {
+ tser::BinaryArchive archive(0);
+ archive.save(value);
+ data = archive.get_buffer().data();
+ }
+
+ template <typename T>
+ void index_single_member(const std::string &key, const std::string &member_name, const T &member_value)
+ {
+ if constexpr (std::is_arithmetic_v<std::decay_t<T>>)
+ {
+ std::string member_value_str = std::to_string(member_value);
+ std::string index_key = member_name + ":" + serializeKey(member_value_str) + ":" + key;
+ rocksdb::Status status = index_db_->Put(rocksdb::WriteOptions(), index_key, key);
+ if (!status.ok())
+ {
+ std::cerr << "Error indexing member in RocksDB index database: " << status.ToString() << std::endl;
+ }
+ }
+ else if constexpr (std::is_same_v<std::decay_t<T>, std::string>)
+ {
+ std::string index_key = member_name + ":" + serializeKey(member_value) + ":" + key;
+ rocksdb::Status status = index_db_->Put(rocksdb::WriteOptions(), index_key, key);
+ if (!status.ok())
+ {
+ std::cerr << "Error indexing member in RocksDB index database: " << status.ToString() << std::endl;
+ }
+ }
+ else if constexpr (tser::is_container_v<std::decay_t<T>>)
+ {
+ for (const auto &elem : member_value)
+ {
+ index_single_member(key, member_name, elem);
+ }
+ }
+ else
+ {
+ // Handle other types if necessary
+ }
+ }
+
+ template <typename T>
+ void index_members(const std::string &key, const T &value)
+ {
+ if constexpr (tser::is_tser_t_v<T>)
+ {
+ auto member_names = T::_memberNames;
+ size_t index = 0;
+ std::apply([&](auto &&...args)
+ { ((index_single_member(key, member_names[index++], args)), ...); },
+ value.members());
+ }
+ }
+
+ std::string serializeKey(const std::string &key)
+ {
+ // Implement a method to serialize keys to handle special characters
+ std::string serialized_key;
+ for (const char c : key)
+ {
+ if (c == ':')
+ {
+ serialized_key += "%3A"; // Percent-encoding for ':'
+ }
+ else
+ {
+ serialized_key += c;
+ }
+ }
+ return serialized_key;
+ }
+};
diff --git a/external/tser b/external/tser
new file mode 160000
index 0000000..9f44236
--- /dev/null
+++ b/external/tser
@@ -0,0 +1 @@
+Subproject commit 9f44236de8fc9e1af5bbea443c8fe8b5a4952ec5
diff --git a/main.cpp b/main.cpp
new file mode 100644
index 0000000..783e261
--- /dev/null
+++ b/main.cpp
@@ -0,0 +1,47 @@
+#include <rocksdb/db.h>
+#include <rocksdb/options.h>
+#include <iostream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <regex>
+#include "tser/tser.hpp"
+#include "RocksDBWrapper.h"
+
+// Define a sample struct with tser serialization
+struct Book
+{
+ std::string title;
+ int pages;
+
+ // Define the members for serialization
+ DEFINE_SERIALIZABLE(Book, title, pages)
+};
+
+int main()
+{
+ RocksDBWrapper db("testdb", "indexdb");
+
+ // Create some sample books
+ Book book1{"Book One", 30};
+ Book book2{"Book Two", 15};
+ Book book3{"Book Three", 50};
+
+ // Store the books in the database
+ db.store("book_1", book1);
+ db.store("book_2", book2);
+ db.store("book_3", book3);
+
+ std::vector<Book> res;
+ db.search_text("*One", res);
+
+ for (const auto &_l : res)
+ {
+ std::cout << _l << std::endl;
+ }
+
+ Book _res;
+ db.get("book_1", _res);
+
+ std::cout << _res << std::endl;
+}
\ No newline at end of file
diff --git a/test_RocksDBWrapper.cpp b/test_RocksDBWrapper.cpp
new file mode 100644
index 0000000..ed76853
--- /dev/null
+++ b/test_RocksDBWrapper.cpp
@@ -0,0 +1,114 @@
+#include "RocksDBWrapper.h"
+#include <gtest/gtest.h>
+#include <string>
+#include <filesystem>
+
+struct TestStruct
+{
+ int id;
+ std::string name;
+ double value;
+
+ DEFINE_SERIALIZABLE(TestStruct, id, name, value);
+};
+
+class RocksDBWrapperTest : public ::testing::Test
+{
+protected:
+ RocksDBWrapper *db;
+ std::string db_path = "/tmp/testdb";
+ std::string index_path = "/tmp/testindex";
+
+ void SetUp() override
+ {
+ db = new RocksDBWrapper(db_path, index_path);
+ }
+
+ void TearDown() override
+ {
+ delete db;
+ // Clean up the database files after each test
+ std::filesystem::remove_all(db_path);
+ std::filesystem::remove_all(index_path);
+ }
+};
+
+TEST_F(RocksDBWrapperTest, StoreAndRetrieve)
+{
+ TestStruct original{1, "Test", 3.14};
+ db->store("test_key", original);
+
+ TestStruct retrieved;
+ bool found = db->get("test_key", retrieved);
+
+ ASSERT_TRUE(found);
+ EXPECT_EQ(original.id, retrieved.id);
+ EXPECT_EQ(original.name, retrieved.name);
+ EXPECT_DOUBLE_EQ(original.value, retrieved.value);
+}
+
+TEST_F(RocksDBWrapperTest, KeyExists)
+{
+ TestStruct original{2, "AnotherTest", 2.718};
+ db->store("another_key", original);
+
+ ASSERT_TRUE(db->keyExists("another_key"));
+ ASSERT_FALSE(db->keyExists("nonexistent_key"));
+}
+
+TEST_F(RocksDBWrapperTest, SearchByMember)
+{
+ TestStruct ts1{1, "Alpha", 1.1};
+ TestStruct ts2{2, "Beta", 2.2};
+ TestStruct ts3{3, "Gamma", 3.3};
+
+ db->store("key1", ts1);
+ db->store("key2", ts2);
+ db->store("key3", ts3);
+
+ auto keys = db->search<TestStruct>("name", "Beta");
+ ASSERT_EQ(keys.size(), 1);
+ EXPECT_EQ(keys[0], "key2");
+}
+
+TEST_F(RocksDBWrapperTest, SearchText)
+{
+ TestStruct ts1{1, "Alpha", 1.1};
+ TestStruct ts2{2, "Beta", 2.2};
+ TestStruct ts3{3, "Gamma", 3.3};
+
+ db->store("key1", ts1);
+ db->store("key2", ts2);
+ db->store("key3", ts3);
+
+ std::vector<TestStruct> results;
+ db->search_text("Bet*", results);
+
+ ASSERT_EQ(results.size(), 1);
+ EXPECT_EQ(results[0].name, "Beta");
+}
+
+TEST_F(RocksDBWrapperTest, ConditionalSearch)
+{
+ TestStruct ts1{1, "Alpha", 1.1};
+ TestStruct ts2{2, "Beta", 2.2};
+ TestStruct ts3{3, "Gamma", 3.3};
+
+ db->store("key1", ts1);
+ db->store("key2", ts2);
+ db->store("key3", ts3);
+
+ std::vector<TestStruct> results;
+ db->search_conditional<TestStruct>(results, [](const TestStruct &s)
+ { return s.value > 2.0; });
+
+ ASSERT_EQ(results.size(), 2);
+ EXPECT_EQ(results[0].name, "Beta");
+ EXPECT_EQ(results[1].name, "Gamma");
+}
+
+int main(int argc, char **argv)
+{
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
--
Gitblit v1.9.3