Feature remove duckdb dependencies and fix some bugs

cpp/pixels-cpp.properties → cpp/etc/pixels-cpp.properties

@@ -21,7 +21,7 @@ pixel.column.size.path=/home/whz/pixels/clickbench-size-e0.csv
 
 # the work thread to run parquet. -1 means using all CPU cores
 parquet.threads=-1
-
+pixels.doublebuffer=true
 # storage device identifier directory depth
 # this parameter defines the directory depth that determines the storage device.
 # for example, we have three SSDs, the path is /data/ssd1, /data/ssd2 and /data/ssd3, so the depth is 2
@@ -63,4 +63,3 @@ pixel.bufferpool.hugepage=true
 # 1 IORING_SETUP_IOPOLL
 # 2 IORING_SETUP_SQPOLL
 pixels.io_uring.mode=0
-

cpp/include/PixelsScanFunction.hpp

@@ -70,10 +70,8 @@
 #include "duckdb/catalog/catalog.hpp"
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/enums/file_compression_type.hpp"
-//#include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/hive_partitioning.hpp"
-//#include "duckdb/common/types/chunk_collection.hpp"
 #include "duckdb/function/copy_function.hpp"
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/main/client_context.hpp"
@@ -87,6 +85,8 @@
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
 #include "duckdb/common/multi_file/multi_file_reader.hpp"
+#include "filter/table_filter.hpp"//转换filter
+#include "filter/constant_filter.hpp"
 
 #endif
 
@@ -117,9 +117,8 @@ namespace duckdb
                                             PixelsReadLocalState &scan_data, PixelsReadGlobalState &parallel_state,
                                             bool is_init_state = false);
 
-        static PixelsReaderOption
-        GetPixelsReaderOption(PixelsReadLocalState &local_state, PixelsReadGlobalState &global_state);
-
+        static PixelsReaderOption GetPixelsReaderOption(PixelsReadLocalState &local_state, PixelsReadGlobalState &global_state);
+        static pixels::TableFilterSet ConvertDuckDBFilter(TableFilterSet *filters);
     private:
         static void TransformDuckdbType(const std::shared_ptr <TypeDescription> &type,
                                         vector <LogicalType> &return_types);
@@ -133,4 +132,4 @@ namespace duckdb
     };
 
 } // namespace duckdb
-#endif // EXAMPLE_C_PIXELSSCANFUNCTION_HPP
+#endif // EXAMPLE_C_PIXELSSCANFUNCTION_HPP

cpp/pixels-common/CMakeLists.txt

@@ -53,4 +53,4 @@ link_directories(${CMAKE_CURRENT_BINARY_DIR}/liburing/src)
 message(${CMAKE_CURRENT_BINARY_DIR}/liburing/src)
 target_link_libraries(pixels-common
         ${Protobuf_LIBRARIES}
-		${CMAKE_CURRENT_BINARY_DIR}/liburing/src/liburing.a)
+		${CMAKE_CURRENT_BINARY_DIR}/liburing/src/liburing.a)

cpp/pixels-common/lib/utils/ConfigFactory.cpp

@@ -53,8 +53,8 @@ ConfigFactory::ConfigFactory()
     {
         pixelsHome += "/";
     }
-    std::ifstream infile(pixelsHome + "etc/pixels-cpp.properties");
-    std::cout << "pixels properties file is " << pixelsHome + "etc/pixels-cpp.properties" << std::endl;
+    std::ifstream infile(pixelsHome + "cpp/etc/pixels-cpp.properties");
+    std::cout << "pixels properties file is " << pixelsHome + "cpp/etc/pixels-cpp.properties" << std::endl;
     std::string line;
     while (std::getline(infile, line))
     {

cpp/pixels-core/include/PixelsBitMask.h

@@ -26,13 +26,6 @@
 #define DUCKDB_PIXELSBITMASK_H
 
 #include <bitset>
-#include "duckdb/planner/table_filter.hpp"
-#include "duckdb/common/vector_size.hpp"
-#include "duckdb/planner/filter/constant_filter.hpp"
-#include "duckdb/planner/filter/null_filter.hpp"
-#include "duckdb/planner/filter/conjunction_filter.hpp"
-#include "duckdb/common/operator/comparison_operators.hpp"
-
 #include "vector/ColumnVector.h"
 #include "TypeDescription.h"
 
@@ -68,4 +61,4 @@ class PixelsBitMask
     uint8_t get(long index);
 };
 
-#endif //DUCKDB_PIXELSBITMASK_H
+#endif //DUCKDB_PIXELSBITMASK_H

cpp/pixels-core/include/PixelsFilter.h

@@ -26,38 +26,54 @@
 #define DUCKDB_PIXELSFILTER_H
 
 #include <bitset>
-#include "duckdb/planner/table_filter.hpp"
-#include "duckdb/common/vector_size.hpp"
-#include "duckdb/planner/filter/constant_filter.hpp"
-#include "duckdb/planner/filter/null_filter.hpp"
-#include "duckdb/planner/filter/conjunction_filter.hpp"
-#include "duckdb/common/operator/comparison_operators.hpp"
 #include "PixelsBitMask.h"
 #include "vector/ColumnVector.h"
 #include "TypeDescription.h"
 #include <immintrin.h>
 #include <avxintrin.h>
+#include "filter/table_filter.hpp"
+#include "filter/constant_filter.hpp"
+#include "filter/conjunction_filter.hpp"
+#include "filter/comparison_operators.hpp"
 
 #define ENABLE_SIMD_FILTER
 
 class PixelsFilter
 {
 public:
-    static void ApplyFilter(std::shared_ptr <ColumnVector> vector, duckdb::TableFilter &filter,
+    static void ApplyFilter(std::shared_ptr <ColumnVector> vector, const pixels::TableFilter &filter,
                             PixelsBitMask &filterMask,
                             std::shared_ptr <TypeDescription> type);
 
     template<class T, class OP>
     static int CompareAvx2(void *data, T constant);
 
-    template<class T, class OP>
+    /*template<class T, class OP>
     static void TemplatedFilterOperation(std::shared_ptr <ColumnVector> vector,
-                                         const duckdb::Value &constant, PixelsBitMask &filter_mask,
-                                         std::shared_ptr <TypeDescription> type);
+                                         const pixels::Scalar &constant, PixelsBitMask &filter_mask,
+                                         std::shared_ptr <TypeDescription> type);*/
+    template<class OP>
+    static void IntFilterOperation(std::shared_ptr <ColumnVector> vector,
+                                    const pixels::Scalar &constant, PixelsBitMask &filter_mask);
+
+    template<class OP>
+    static void LongFilterOperation(std::shared_ptr <ColumnVector> vector,
+                                    const pixels::Scalar &constant, PixelsBitMask &filter_mask);
+    template<class OP>
+    static void DateFilterOperation(std::shared_ptr <ColumnVector> vector,
+                                    const pixels::Scalar &constant, PixelsBitMask &filter_mask);
+
+    template<class OP>
+    static void DecimalFilterOperation(std::shared_ptr <ColumnVector> vector,
+                                    const pixels::Scalar &constant, PixelsBitMask &filter_mask);    
+
+    template<class OP>
+    static void StringFilterOperation(std::shared_ptr <ColumnVector> vector,
+                                    const pixels::Scalar &constant, PixelsBitMask &filter_mask);                                    
 
     template<class OP>
-    static void FilterOperationSwitch(std::shared_ptr <ColumnVector> vector, duckdb::Value &constant,
+    static void FilterOperationSwitch(std::shared_ptr <ColumnVector> vector, const pixels::Scalar &constant,
                                       PixelsBitMask &filter_mask, std::shared_ptr <TypeDescription> type);
 
 };
-#endif //DUCKDB_PIXELSFILTER_H
+#endif //DUCKDB_PIXELSFILTER_H

cpp/pixels-core/include/PixelsTypes.h

@@ -0,0 +1,155 @@
+#pragma once
+//放一些枚举类
+
+#ifndef PIXELS_TYPES_H
+#define PIXELS_TYPES_H
+#include <variant>
+#include <cstdint>
+#include <cassert>
+#include <string>
+using idx_t = uint64_t;
+namespace pixels {
+
+enum class PhysicalType : uint8_t {//decimal的物理存储类型
+    INT16,
+    INT32,
+    INT64,
+    INT128,
+};
+
+struct DecimalConfig {//decimal不同物理存储类型的最大宽度
+        static constexpr int MAX_WIDTH_INT16 = 4;
+        static constexpr int MAX_WIDTH_INT32 = 9;
+        static constexpr int MAX_WIDTH_INT64 = 18;
+        static constexpr int MAX_WIDTH_INT128 = 38;
+};
+
+enum class ComparisonOperator : uint8_t {
+        EQUAL,
+        NOT_EQUAL,
+        GREATER_THAN,
+        GREATER_THAN_OR_EQUAL,
+        LESS_THAN,
+        LESS_THAN_OR_EQUAL,
+        IS_NULL,
+        IS_NOT_NULL
+};
+
+enum class TableFilterType : uint8_t {
+	CONSTANT_COMPARISON = 0, // constant comparison (e.g. =C, >C, >=C, <C, <=C)
+	IS_NULL = 1,             // C IS NULL 不支持
+	IS_NOT_NULL = 2,         // C IS NOT NULL 不支持
+	CONJUNCTION_OR = 3,      // OR of different filters
+	CONJUNCTION_AND = 4,     // AND of different filters
+	STRUCT_EXTRACT = 5,      // filter applies to child-column of struct 不支持
+	OPTIONAL_FILTER = 6,     // executing filter is not required for query correctness 不支持
+	IN_FILTER = 7,           // col IN (C1, C2, C3, ...)    不支持
+	DYNAMIC_FILTER = 8,      // dynamic filters can be updated at run-time  不支持
+	EXPRESSION_FILTER = 9,   // an arbitrary expression     不支持
+    DEFAULT=10               //未定义
+};
+
+struct Scalar {
+    enum class Type {
+        INT32,
+        INT64,
+        FLOAT,
+        DOUBLE,
+        STRING,
+        INVALID
+    };
+
+private:
+    Type type_;
+
+    int32_t i32_;
+    int64_t i64_;
+    float f_;
+    double d_;
+    std::string str_;
+
+public:
+    // ---------- 构造 ----------
+    Scalar() : type_(Type::INVALID) {}
+
+    Scalar(int32_t v) : type_(Type::INT32), i32_(v) {}
+    Scalar(int64_t v) : type_(Type::INT64), i64_(v) {}
+    Scalar(float v) : type_(Type::FLOAT), f_(v) {}
+    Scalar(double v) : type_(Type::DOUBLE), d_(v) {}
+    Scalar(const std::string &v) : type_(Type::STRING), str_(v) {}
+    Scalar(std::string &&v) : type_(Type::STRING), str_(std::move(v)) {}
+
+    // ---------- 类型访问 ----------
+    Type type() const {
+        return type_;
+    }
+
+    bool is_int32() const { return type_ == Type::INT32; }
+    bool is_int64() const { return type_ == Type::INT64; }
+    bool is_float() const { return type_ == Type::FLOAT; }
+    bool is_double() const { return type_ == Type::DOUBLE; }
+    bool is_string() const { return type_ == Type::STRING; }
+
+    // ---------- 读取接口 ----------
+    int32_t get_int32() const {
+        assert(type_ == Type::INT32);
+        return i32_;
+    }
+
+    int64_t get_int64() const {
+        assert(type_ == Type::INT64);
+        return i64_;
+    }
+
+    float get_float() const {
+        assert(type_ == Type::FLOAT);
+        return f_;
+    }
+
+    double get_double() const {
+        assert(type_ == Type::DOUBLE);
+        return d_;
+    }
+
+    const std::string &get_string() const {
+        assert(type_ == Type::STRING);
+        return str_;
+    }
+
+    // ---------- 写入接口 ----------
+    void set(int32_t v) {
+        type_ = Type::INT32;
+        i32_ = v;
+    }
+
+    void set(int64_t v) {
+        type_ = Type::INT64;
+        i64_ = v;
+    }
+
+    void set(float v) {
+        type_ = Type::FLOAT;
+        f_ = v;
+    }
+
+    void set(double v) {
+        type_ = Type::DOUBLE;
+        d_ = v;
+    }
+
+    void set(const std::string &v) {
+        type_ = Type::STRING;
+        str_ = v;
+    }
+
+    void set(std::string &&v) {
+        type_ = Type::STRING;
+        str_ = std::move(v);
+    }
+};
+
+
+
+} // namespace pixels
+
+#endif // PIXELS_TYPES_H

cpp/pixels-core/include/filter/comparison_operators.hpp

@@ -0,0 +1,45 @@
+#pragma once
+namespace pixels{
+    //仿照duckdb的比较符结构体，每个结构体都有一个静态函数Operation，接受两个参数，返回比较结果。PixelsFilter.cpp中会调用这个函数来进行过滤操作。
+
+struct Equals {
+	template <class T>
+	static inline bool Operation(const T &left, const T &right) {
+		return left == right;
+	}
+};
+struct NotEquals {
+	template <class T>
+	static inline bool Operation(const T &left, const T &right) {
+		return !Equals::Operation(left, right);
+	}
+};
+
+struct GreaterThan {
+	template <class T>
+	static inline bool Operation(const T &left, const T &right) {
+		return left > right;
+	}
+};
+
+struct GreaterThanEquals {
+	template <class T>
+	static inline bool Operation(const T &left, const T &right) {
+		return !GreaterThan::Operation(right, left);
+	}
+};
+
+struct LessThan {
+	template <class T>
+	static inline bool Operation(const T &left, const T &right) {
+		return GreaterThan::Operation(right, left);
+	}
+};
+
+struct LessThanEquals {
+	template <class T>
+	static inline bool Operation(const T &left, const T &right) {
+		return !GreaterThan::Operation(left, right);
+	}
+};
+}