Merge branch 'OpenAtomFoundation:unstable' into new

.github/pr-title-checker-config.json

@@ -4,10 +4,9 @@
     "color": "B60205"
   },
   "CHECKS": {
-    "regexp": "^(feat|fix|test|refactor|chore|style|docs|perf|build|ci|revert)(\\(.*\\))?:.*",
+    "regexp": "^(feat|fix|test|refactor|chore|upgrade|bump|style|docs|perf|build|ci|revert)(\\(.*\\))?:[^\u4e00-\u9fa5]+$",
     "ignoreLabels": [
       "ignore-title"
     ]
   }
-}
-
+}

.github/workflows/pr-title-checker.yaml

@@ -12,7 +12,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: thehanimo/[email protected].1
+      - uses: thehanimo/[email protected].2
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           pass_on_octokit_error: false

CMakeLists.txt

@@ -79,7 +79,7 @@ ELSEIF (CMAKE_SYSTEM_NAME MATCHES "Linux")
         set(CMAKE_THREAD_LIBS_INIT "-lpthread")
     ELSEIF (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
         SET(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wl,--no-as-needed -ldl")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wl,--no-as-needed -ldl -Wno-restrict")
     ENDIF ()
     ADD_DEFINITIONS(-DOS_LINUX)
 ELSE ()
@@ -150,6 +150,7 @@ INCLUDE(FetchContent)
 INCLUDE(cmake/openssl.cmake)
 INCLUDE(cmake/gflags.cmake)
 INCLUDE(cmake/findTools.cmake)
+INCLUDE(cmake/snappy.cmake)
 INCLUDE(cmake/leveldb.cmake)
 INCLUDE(cmake/libevent.cmake)
 INCLUDE(cmake/llhttp.cmake)

cmake/braft.cmake

@@ -10,7 +10,7 @@ SET(BRAFT_INSTALL_DIR ${THIRD_PARTY_PATH}/install/braft)
 SET(BRAFT_INCLUDE_DIR "${BRAFT_INSTALL_DIR}/include" CACHE PATH "braft include directory." FORCE)
 SET(BRAFT_LIBRARIES "${BRAFT_INSTALL_DIR}/lib/libbraft.a" CACHE FILEPATH "braft library." FORCE)
 
-SET(prefix_path "${THIRD_PARTY_PATH}/install/brpc|${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-build|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-src")
+SET(prefix_path "${THIRD_PARTY_PATH}/install/brpc|${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/leveldb")
 
 ExternalProject_Add(
         extern_braft

cmake/brpc.cmake

@@ -11,7 +11,7 @@ SET(BRPC_INCLUDE_DIR "${BRPC_INSTALL_DIR}/include" CACHE PATH "brpc include dire
 SET(BRPC_LIBRARIES "${BRPC_INSTALL_DIR}/lib/libbrpc.a" CACHE FILEPATH "brpc library." FORCE)
 
 # Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args
-SET(prefix_path "${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-build|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-src|${THIRD_PARTY_PATH}/install/openssl")
+SET(prefix_path "${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/leveldb|${THIRD_PARTY_PATH}/install/openssl")
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
 # If minimal .a is need, you can set  WITH_DEBUG_SYMBOLS=OFF
 EXTERNALPROJECT_ADD( 

cmake/gflags.cmake

@@ -16,7 +16,7 @@ FetchContent_Declare(gflags
 SET(GFLAGS_BUILD_STATIC_LIBS ON CACHE BOOL "" FORCE)
 SET(GFLAGS_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
 SET(GFLAGS_BUILD_gflags_LIB ON CACHE BOOL "" FORCE)
-SET(GFLAGS_BUILD_gflags_nothreads_LIB OFF CACHE BOOL "" FORCE)
+SET(GFLAGS_BUILD_gflags_nothreads_LIB ON CACHE BOOL "" FORCE)
 SET(GFLAGS_BUILD_TESTING OFF CACHE BOOL "" FORCE)
 FETCHCONTENT_MAKEAVAILABLE(gflags)
 

cmake/leveldb.cmake

@@ -3,17 +3,46 @@
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.
 
-INCLUDE_GUARD()
+INCLUDE(ExternalProject)
 
-FETCHCONTENT_DECLARE(
-    leveldb
-    GIT_REPOSITORY https://github.com/google/leveldb.git
-    GIT_TAG main
+SET(LEVELDB_SOURCES_DIR ${THIRD_PARTY_PATH}/leveldb)
+SET(LEVELDB_INSTALL_DIR ${THIRD_PARTY_PATH}/install/leveldb)
+SET(LEVELDB_INCLUDE_DIR "${LEVELDB_INSTALL_DIR}/include" CACHE PATH "leveldb include directory." FORCE)
+SET(LEVELDB_LIBRARIES "${LEVELDB_INSTALL_DIR}/lib/libleveldb.a" CACHE FILEPATH "leveldb library." FORCE)
+
+SET(prefix_path "${THIRD_PARTY_PATH}/install/snappy")
+
+ExternalProject_Add(
+        extern_leveldb
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        DEPENDS snappy
+        GIT_REPOSITORY "https://github.com/google/leveldb.git"
+        GIT_TAG "1.23"
+        PREFIX ${LEVELDB_SOURCES_DIR}
+        UPDATE_COMMAND ""
+        CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+        -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+        -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
+        -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
+        -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+        -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
+        -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
+        -DCMAKE_INSTALL_PREFIX=${LEVELDB_INSTALL_DIR}
+        -DCMAKE_INSTALL_LIBDIR=${LEVELDB_INSTALL_DIR}/lib
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+        -DLEVELDB_BUILD_TESTS=OFF
+        -DLEVELDB_BUILD_BENCHMARKS=OFF
+        -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+        -DCMAKE_PREFIX_PATH=${prefix_path}
+        ${EXTERNAL_OPTIONAL_ARGS}
+        CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${LEVELDB_INSTALL_DIR}
+        -DCMAKE_INSTALL_LIBDIR:PATH=${LEVELDB_INSTALL_DIR}/lib
+        -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+        -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
 )
-SET(LEVELDB_BUILD_TESTS OFF CACHE BOOL "" FORCE)
-SET(LEVELDB_BUILD_BENCHMARKS OFF CACHE BOOL "" FORCE)
-SET(LEVELDB_INSTALL OFF CACHE BOOL "" FORCE)
-FETCHCONTENT_MAKEAVAILABLE(leveldb)
 
-SET(LEVELDB_INCLUDE_PATH ${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-src/include)
-SET(LEVELDB_LIB ${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-build/libleveldb.a)
+ADD_DEPENDENCIES(extern_leveldb snappy)
+ADD_LIBRARY(leveldb STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET leveldb PROPERTY IMPORTED_LOCATION ${LEVELDB_LIBRARIES})
+ADD_DEPENDENCIES(leveldb extern_leveldb)

cmake/snappy.cmake

@@ -0,0 +1,47 @@
+# Copyright (c) 2024-present, Qihoo, Inc.  All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+
+INCLUDE(ExternalProject)
+
+SET(SNAPPY_SOURCES_DIR ${THIRD_PARTY_PATH}/snappy)
+SET(SNAPPY_INSTALL_DIR ${THIRD_PARTY_PATH}/install/snappy)
+SET(SNAPPY_INCLUDE_DIR "${SNAPPY_INSTALL_DIR}/include" CACHE PATH "snappy include directory." FORCE)
+SET(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.a" CACHE FILEPATH "snappy library." FORCE)
+
+SET(prefix_path "${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build")
+
+ExternalProject_Add(
+        extern_snappy
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        DEPENDS gflags
+        GIT_REPOSITORY "https://github.com/google/snappy.git"
+        GIT_TAG "1.2.1"
+        PREFIX ${SNAPPY_SOURCES_DIR}
+        UPDATE_COMMAND ""
+        CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+        -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+        -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
+        -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
+        -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+        -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
+        -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
+        -DCMAKE_INSTALL_PREFIX=${SNAPPY_INSTALL_DIR}
+        -DCMAKE_INSTALL_LIBDIR=${SNAPPY_INSTALL_DIR}/lib
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+        -DSNAPPY_BUILD_TESTS=OFF
+        -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+        -DCMAKE_PREFIX_PATH=${prefix_path}
+        ${EXTERNAL_OPTIONAL_ARGS}
+        CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${SNAPPY_INSTALL_DIR}
+        -DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPY_INSTALL_DIR}/lib
+        -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+        -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+)
+
+ADD_DEPENDENCIES(extern_snappy gflags)
+ADD_LIBRARY(snappy STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET snappy PROPERTY IMPORTED_LOCATION ${SNAPPY_LIBRARIES})
+ADD_DEPENDENCIES(snappy extern_snappy)

src/CMakeLists.txt

@@ -31,7 +31,6 @@ TARGET_INCLUDE_DIRECTORIES(pikiwidb
         PRIVATE ${BRPC_INCLUDE_DIR}
         )
 
-
-TARGET_LINK_LIBRARIES(pikiwidb net; dl; fmt; storage; rocksdb; pstd braft brpc ssl crypto zlib protobuf leveldb gflags  z praft praft_pb "${LIB}")
+TARGET_LINK_LIBRARIES(pikiwidb net; dl; fmt; storage; rocksdb; pstd braft brpc ssl crypto zlib protobuf leveldb gflags z praft praft_pb "${LIB}")
 
 SET_TARGET_PROPERTIES(pikiwidb PROPERTIES LINKER_LANGUAGE CXX)

src/base_cmd.cc

@@ -7,8 +7,10 @@
 
 #include "base_cmd.h"
 #include "common.h"
+#include "config.h"
 #include "log.h"
 #include "pikiwidb.h"
+#include "praft/praft.h"
 
 namespace pikiwidb {
 
@@ -32,6 +34,20 @@ std::vector<std::string> BaseCmd::CurrentKey(PClient* client) const { return std
 void BaseCmd::Execute(PClient* client) {
   DEBUG("execute command: {}", client->CmdName());
 
+  if (g_config.use_raft.load()) {
+    // 1. If PRAFT is not initialized yet, return an error message to the client for both read and write commands.
+    if (!PRAFT.IsInitialized() && (HasFlag(kCmdFlagsReadonly) || HasFlag(kCmdFlagsWrite))) {
+      DEBUG("drop command: {}", client->CmdName());
+      return client->SetRes(CmdRes::kErrOther, "PRAFT is not initialized");
+    }
+
+    // 2. If PRAFT is initialized and the current node is not the leader, return a redirection message for write
+    // commands.
+    if (HasFlag(kCmdFlagsWrite) && !PRAFT.IsLeader()) {
+      return client->SetRes(CmdRes::kErrOther, fmt::format("MOVED {}", PRAFT.GetLeaderAddress()));
+    }
+  }
+
   auto dbIndex = client->GetCurrentDB();
   if (!HasFlag(kCmdFlagsExclusive)) {
     PSTORE.GetBackend(dbIndex)->LockShared();

src/cmd_admin.cc

@@ -129,8 +129,7 @@ void ShutdownCmd::DoCmd(PClient* client) {
   client->SetRes(CmdRes::kNone);
 }
 
-PingCmd::PingCmd(const std::string& name, int16_t arity)
-    : BaseCmd(name, arity, kCmdFlagsWrite, kAclCategoryWrite | kAclCategoryList) {}
+PingCmd::PingCmd(const std::string& name, int16_t arity) : BaseCmd(name, arity, kCmdFlagsFast, kAclCategoryFast) {}
 
 bool PingCmd::DoInitial(PClient* client) { return true; }
 

src/cmd_hash.cc

@@ -93,6 +93,10 @@ HMSetCmd::HMSetCmd(const std::string& name, int16_t arity)
     : BaseCmd(name, arity, kCmdFlagsWrite, kAclCategoryWrite | kAclCategoryHash) {}
 
 bool HMSetCmd::DoInitial(PClient* client) {
+  if (client->argv_.size() % 2 != 0) {
+    client->SetRes(CmdRes::kWrongNum, kCmdNameHMSet);
+    return false;
+  }
   client->SetKey(client->argv_[1]);
   client->ClearFvs();
   // set fvs
@@ -116,6 +120,7 @@ HMGetCmd::HMGetCmd(const std::string& name, int16_t arity)
 
 bool HMGetCmd::DoInitial(PClient* client) {
   client->SetKey(client->argv_[1]);
+  client->ClearFields();
   for (size_t i = 2; i < client->argv_.size(); ++i) {
     client->Fields().push_back(client->argv_[i]);
   }

src/cmd_kv.cc

@@ -532,7 +532,7 @@ void SetNXCmd::DoCmd(PClient* client) {
 }
 
 GetBitCmd::GetBitCmd(const std::string& name, int16_t arity)
-    : BaseCmd(name, arity, kCmdFlagsWrite, kAclCategoryWrite | kAclCategoryString) {}
+    : BaseCmd(name, arity, kCmdFlagsReadonly | kCmdFlagsFast, kAclCategoryRead | kAclCategoryBitmap) {}
 
 bool GetBitCmd::DoInitial(PClient* client) {
   client->SetKey(client->argv_[1]);

src/cmd_zset.cc

@@ -726,7 +726,10 @@ void ZScoreCmd::DoCmd(PClient* client) {
   storage::Status s;
   s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->ZScore(client->Key(), client->argv_[2], &score);
   if (s.ok() || s.IsNotFound()) {
-    client->AppendString(std::to_string(score));
+    char buf[32];
+    int64_t len = pstd::D2string(buf, sizeof(buf), score);
+    client->AppendStringLen(len);
+    client->AppendContent(buf);
   } else {
     client->SetRes(CmdRes::kErrOther, s.ToString());
   }

src/db.cc

@@ -121,6 +121,11 @@ void DB::LoadDBFromCheckpoint(const std::string& checkpoint_path, bool sync [[ma
     abort();
   }
 
+  // in single-mode, pikiwidb will enable wal
+  if (!g_config.use_raft.load(std::memory_order_relaxed)) {
+    storage_->DisableWal(false);
+  }
+
   opened_ = true;
   INFO("DB{} load a checkpoint from {} success!", db_index_, checkpoint_path);
 }

src/pikiwidb.cc

@@ -172,8 +172,6 @@ bool PikiwiDB::Init() {
     PREPL.SetMasterAddr(g_config.master_ip.ToString().c_str(), g_config.master_port.load());
   }
 
-  //  cmd_table_manager_.InitCmdTable();
-
   return true;
 }
 

tests/consistency_test.go

@@ -47,11 +47,11 @@ var _ = Describe("Consistency", Ordered, func() {
 			if i == 0 {
 				leader = s.NewClient()
 				Expect(leader).NotTo(BeNil())
-				Expect(leader.FlushDB(ctx).Err()).NotTo(HaveOccurred())
+				Expect(leader.FlushDB(ctx).Err().Error()).To(Equal("ERR PRAFT is not initialized"))
 			} else {
 				c := s.NewClient()
 				Expect(c).NotTo(BeNil())
-				Expect(c.FlushDB(ctx).Err()).NotTo(HaveOccurred())
+				Expect(c.FlushDB(ctx).Err().Error()).To(Equal("ERR PRAFT is not initialized"))
 				followers = append(followers, c)
 			}
 		}
@@ -96,7 +96,7 @@ var _ = Describe("Consistency", Ordered, func() {
 			} else {
 				c := s.NewClient()
 				Expect(c).NotTo(BeNil())
-				//Expect(c.FlushDB(ctx).Err().Error()).To(Equal("ERR MOVED 127.0.0.1:12111"))
+				Expect(c.FlushDB(ctx).Err().Error()).To(Equal("ERR MOVED 127.0.0.1:12111"))
 				followers = append(followers, c)
 			}
 		}