From bac29677ed2e392b47825d3e4a0445b3d446c03c Mon Sep 17 00:00:00 2001 From: den818 Date: Thu, 1 Dec 2022 01:02:36 +0400 Subject: [PATCH] Support geo column types --- README.md | 1 + clickhouse/CMakeLists.txt | 2 + clickhouse/client.h | 1 + clickhouse/columns/factory.cpp | 16 ++++- clickhouse/columns/geo.cpp | 108 +++++++++++++++++++++++++++++++ clickhouse/columns/geo.h | 76 ++++++++++++++++++++++ clickhouse/columns/tuple.h | 27 +++++--- clickhouse/types/type_parser.cpp | 6 +- clickhouse/types/types.cpp | 28 ++++++++ clickhouse/types/types.h | 14 +++- ut/client_ut.cpp | 88 +++++++++++++++++++++++++ ut/types_ut.cpp | 6 +- ut/utils.cpp | 15 ++++- 13 files changed, 371 insertions(+), 17 deletions(-) create mode 100644 clickhouse/columns/geo.cpp create mode 100644 clickhouse/columns/geo.h diff --git a/README.md b/README.md index b35d5c55..ce1f68fe 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ C++ client for [ClickHouse](https://clickhouse.com/). * Int128 * UUID * Map +* Point, Ring, Polygon, MultiPolygon ## Building diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 2caeebba..3c8f03a9 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -12,6 +12,7 @@ SET ( clickhouse-cpp-lib-src columns/decimal.cpp columns/enum.cpp columns/factory.cpp + columns/geo.cpp columns/ip4.cpp columns/ip6.cpp columns/lowcardinality.cpp @@ -116,6 +117,7 @@ INSTALL(FILES columns/date.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/decimal.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/enum.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/factory.h DESTINATION include/clickhouse/columns/) +INSTALL(FILES columns/geo.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/ip4.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/ip6.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/itemview.h DESTINATION include/clickhouse/columns/) diff --git a/clickhouse/client.h b/clickhouse/client.h index 2b315b00..63a51c2e 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -7,6 +7,7 @@ #include "columns/date.h" #include "columns/decimal.h" #include "columns/enum.h" +#include "columns/geo.h" #include "columns/ip4.h" #include "columns/ip6.h" #include "columns/lowcardinality.h" diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index 2ea7b6f5..fbd57889 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -4,17 +4,19 @@ #include "date.h" #include "decimal.h" #include "enum.h" +#include "geo.h" #include "ip4.h" #include "ip6.h" #include "lowcardinality.h" #include "lowcardinalityadaptor.h" +#include "map.h" #include "nothing.h" #include "nullable.h" #include "numeric.h" #include "string.h" #include "tuple.h" #include "uuid.h" -#include "map.h" + #include "../types/type_parser.h" @@ -97,6 +99,18 @@ static ColumnRef CreateTerminalColumn(const TypeAst& ast) { case Type::UUID: return std::make_shared(); + case Type::Point: + return std::make_shared(); + + case Type::Ring: + return std::make_shared(); + + case Type::Polygon: + return std::make_shared(); + + case Type::MultiPolygon: + return std::make_shared(); + default: return nullptr; } diff --git a/clickhouse/columns/geo.cpp b/clickhouse/columns/geo.cpp new file mode 100644 index 00000000..5d749374 --- /dev/null +++ b/clickhouse/columns/geo.cpp @@ -0,0 +1,108 @@ +#include "geo.h" + +#include "utils.h" + +namespace { +using namespace ::clickhouse; + +template +TypeRef CreateGeoType() { + if constexpr (type_code == Type::Code::Point) { + return Type::CreatePoint(); + } else if constexpr (type_code == Type::Code::Ring) { + return Type::CreateRing(); + } else if constexpr (type_code == Type::Code::Polygon) { + return Type::CreatePolygon(); + } else if constexpr (type_code == Type::Code::MultiPolygon) { + return Type::CreateMultiPolygon(); + } +} + +template +std::shared_ptr CreateColumn() { + if constexpr (std::is_same_v>) { + return std::make_shared>( + std::make_tuple(std::make_shared(), std::make_shared())); + } else { + return std::make_shared(); + } +} + +} // namespace + +namespace clickhouse { + +template +ColumnGeo::ColumnGeo() + : Column(std::move(CreateGeoType())), + data_(std::move(CreateColumn())) { +} + +template +ColumnGeo::ColumnGeo(ColumnRef data) + : Column(std::move(CreateGeoType())) + , data_(std::move(WrapColumn(std::move(data)))) { +} + +template +void ColumnGeo::Clear() { + data_->Clear(); +} + +template +const typename ColumnGeo::ValueType ColumnGeo::At(size_t n) const { + return data_->At(n); +} + +template +const typename ColumnGeo::ValueType ColumnGeo::operator[](size_t n) const { + return data_->At(n); +} + +template +void ColumnGeo::Append(ColumnRef column) { + if (auto col = column->template As()) { + data_->Append(col->data_->template As()); + } +} + +template +bool ColumnGeo::LoadBody(InputStream* input, size_t rows) { + return data_->LoadBody(input, rows); +} + +template +void ColumnGeo::SaveBody(OutputStream* output) { + data_->SaveBody(output); +} + +template +size_t ColumnGeo::Size() const { + return data_->Size(); +} + +template +ColumnRef ColumnGeo::Slice(size_t begin, size_t len) const { + return std::make_shared(data_->Slice(begin, len)); +} + +template +ColumnRef ColumnGeo::CloneEmpty() const { + return std::make_shared(); +} + +template +void ColumnGeo::Swap(Column& other) { + auto& col = dynamic_cast(other); + data_.swap(col.data_); +} + +template class ColumnGeo, Type::Code::Point>; + +template class ColumnGeo, Type::Code::Ring>; + +template class ColumnGeo, Type::Code::Polygon>; + +template class ColumnGeo, Type::Code::MultiPolygon>; + +} // namespace clickhouse diff --git a/clickhouse/columns/geo.h b/clickhouse/columns/geo.h new file mode 100644 index 00000000..5f3db9b6 --- /dev/null +++ b/clickhouse/columns/geo.h @@ -0,0 +1,76 @@ +#pragma once + +#include "array.h" +#include "column.h" +#include "numeric.h" +#include "tuple.h" + +namespace clickhouse { + +template +class ColumnGeo : public Column { +public: + using ValueType = typename NestedColumnType::ValueType; + + ColumnGeo(); + + explicit ColumnGeo(ColumnRef data); + + /// Appends one element to the end of column. + template + void Append(const T& value) { + data_->Append(value); + } + + /// Returns element at given row number. + const ValueType At(size_t n) const; + + /// Returns element at given row number. + const ValueType operator[](size_t n) const; + +public: + /// Appends content of given column to the end of current one. + void Append(ColumnRef column) override; + + /// Loads column data from input stream. + bool LoadBody(InputStream* input, size_t rows) override; + + /// Saves column data to output stream. + void SaveBody(OutputStream* output) override; + + /// Clear column data . + void Clear() override; + + /// Returns count of rows in the column. + size_t Size() const override; + + /// Makes slice of the current column. + ColumnRef Slice(size_t begin, size_t len) const override; + ColumnRef CloneEmpty() const override; + void Swap(Column& other) override; + +private: + std::shared_ptr data_; +}; + +// /** +// * Represents a Point column. +// */ +using ColumnPoint = ColumnGeo, Type::Code::Point>; + +/** + * Represents a Ring column. + */ +using ColumnRing = ColumnGeo, Type::Code::Ring>; + +/** + * Represents a Polygon column. + */ +using ColumnPolygon = ColumnGeo, Type::Code::Polygon>; + +/** + * Represents a MultiPolygon column. + */ +using ColumnMultiPolygon = ColumnGeo, Type::Code::MultiPolygon>; + +} // namespace clickhouse diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index a5f3a25f..ff33ba02 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -72,16 +72,11 @@ class ColumnTupleT : public ColumnTuple { inline ValueType operator[](size_t index) const { return GetTupleOfValues(index); } - template > - inline void Append([[maybe_unused]] T value) { - static_assert(index <= std::tuple_size_v); - static_assert(std::tuple_size_v == std::tuple_size_v); - if constexpr (index == 0) { - return; - } else { - std::get(typed_columns_)->Append(std::move(std::get(value))); - Append(std::move(value)); - } + using ColumnTuple::Append; + + template + inline void Append(std::tuple value) { + AppendTuple(std::move(value)); } /** Create a ColumnTupleT from a ColumnTuple, without copying data and offsets, but by @@ -118,6 +113,18 @@ class ColumnTupleT : public ColumnTuple { } private: + template > + inline void AppendTuple([[maybe_unused]] T value) { + static_assert(index <= std::tuple_size_v); + static_assert(std::tuple_size_v == std::tuple_size_v); + if constexpr (index == 0) { + return; + } else { + std::get(typed_columns_)->Append(std::move(std::get(value))); + AppendTuple(std::move(value)); + } + } + template > inline static std::vector TupleToVector([[maybe_unused]] const T& value) { static_assert(index <= std::tuple_size_v); diff --git a/clickhouse/types/type_parser.cpp b/clickhouse/types/type_parser.cpp index ee05a244..e16aadb5 100644 --- a/clickhouse/types/type_parser.cpp +++ b/clickhouse/types/type_parser.cpp @@ -46,7 +46,11 @@ static const std::unordered_map kTypeCode = { { "Decimal64", Type::Decimal64 }, { "Decimal128", Type::Decimal128 }, { "LowCardinality", Type::LowCardinality }, - { "Map", Type::Map}, + { "Map", Type::Map}, + { "Point", Type::Point}, + { "Ring", Type::Ring}, + { "Polygon", Type::Polygon}, + { "MultiPolygon", Type::MultiPolygon}, }; static Type::Code GetTypeCode(const std::string& name) { diff --git a/clickhouse/types/types.cpp b/clickhouse/types/types.cpp index 370bc47b..d5093f10 100644 --- a/clickhouse/types/types.cpp +++ b/clickhouse/types/types.cpp @@ -47,6 +47,10 @@ const char* Type::TypeName(Type::Code code) { case Type::Code::DateTime64: return "DateTime64"; case Type::Code::Date32: return "Date32"; case Type::Code::Map: return "Map"; + case Type::Code::Point: return "Point"; + case Type::Code::Ring: return "Ring"; + case Type::Code::Polygon: return "Polygon"; + case Type::Code::MultiPolygon: return "MultiPolygon"; } return "Unknown type"; @@ -72,6 +76,10 @@ std::string Type::GetName() const { case IPv6: case Date: case Date32: + case Point: + case Ring: + case Polygon: + case MultiPolygon: return TypeName(code_); case FixedString: return As()->GetName(); @@ -126,6 +134,10 @@ uint64_t Type::GetTypeUniqueId() const { case IPv6: case Date: case Date32: + case Point: + case Ring: + case Polygon: + case MultiPolygon: // For simple types, unique ID is the same as Type::Code return code_; @@ -233,6 +245,22 @@ TypeRef Type::CreateMap(TypeRef key_type, TypeRef value_type) { return std::make_shared(key_type, value_type); } +TypeRef Type::CreatePoint() { + return TypeRef(new Type(Type::Point)); +} + +TypeRef Type::CreateRing() { + return TypeRef(new Type(Type::Ring)); +} + +TypeRef Type::CreatePolygon() { + return TypeRef(new Type(Type::Polygon)); +} + +TypeRef Type::CreateMultiPolygon() { + return TypeRef(new Type(Type::MultiPolygon)); +} + /// class ArrayType ArrayType::ArrayType(TypeRef item_type) : Type(Array), item_type_(item_type) { diff --git a/clickhouse/types/types.h b/clickhouse/types/types.h index 16570f34..423a6f70 100644 --- a/clickhouse/types/types.h +++ b/clickhouse/types/types.h @@ -50,7 +50,11 @@ class Type { LowCardinality, DateTime64, Date32, - Map + Map, + Point, + Ring, + Polygon, + MultiPolygon }; using EnumItem = std::pair; @@ -128,6 +132,14 @@ class Type { static TypeRef CreateMap(TypeRef key_type, TypeRef value_type); + static TypeRef CreatePoint(); + + static TypeRef CreateRing(); + + static TypeRef CreatePolygon(); + + static TypeRef CreateMultiPolygon(); + private: uint64_t GetTypeUniqueId() const; diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 58d6304b..0278d6d0 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -67,6 +67,21 @@ class ClientCase : public testing::TestWithParam { } } + std::string GetSettingValue(const std::string& name) { + std::string result; + client_->Select("SELECT value FROM system.settings WHERE name = \'" + name + "\'", + [&result](const Block& block) + { + if (block.GetRowCount() == 0) { + return; + } + result = block[0]->AsStrict()->At(0); + } + ); + return result; + } + + std::unique_ptr client_; const std::string table_name = "test_clickhouse_cpp_test_ut_table"; const std::string column_name = "test_column"; @@ -1051,6 +1066,79 @@ TEST_P(ClientCase, RoundtripMapUUID_Tuple_String_Array_Uint64) { EXPECT_TRUE(CompareRecursive(*map, *result_typed)); } +TEST_P(ClientCase, RoundtripPoint) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + col->Append({1.0, 2.0}); + col->Append({0.1, 0.2}); + + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(ClientCase, RoundtripRing) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector ring{{1.0, 2.0}, {3.0, 4.0}}; + col->Append(ring); + } + { + std::vector ring{{0.1, 0.2}, {0.3, 0.4}}; + col->Append(ring); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(ClientCase, RoundtripPolygon) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector> polygon + {{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}}; + col->Append(polygon); + } + { + std::vector> polygon + {{{0.1, 0.2}, {0.3, 0.4}}, {{0.5, 0.6}, {0.7, 0.8}}}; + col->Append(polygon); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(ClientCase, RoundtripMultiPolygon) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector>> multi_polygon + {{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}}, + {{{1.1, 2.2}, {3.3, 4.4}}, {{5.5, 6.6}, {7.7, 8.8}}}}; + col->Append(multi_polygon); + } + { + std::vector>> multi_polygon + {{{{0.1, 0.2}, {0.3, 0.4}}, {{0.5, 0.6}, {0.7, 0.8}}}, + {{{1.1, 1.2}, {1.3, 1.4}}, {{1.5, 1.6}, {1.7, 1.8}}}}; + col->Append(multi_polygon); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + TEST_P(ClientCase, OnProgress) { Block block; createTableWithOneColumn(block); diff --git a/ut/types_ut.cpp b/ut/types_ut.cpp index 8e2f943c..3f795269 100644 --- a/ut/types_ut.cpp +++ b/ut/types_ut.cpp @@ -109,7 +109,11 @@ TEST(TypesCase, IsEqual) { "Map(String, Tuple(String, Int8, Date, DateTime))", "Map(UUID, Array(Tuple(String, Int8, Date, DateTime)))", "Map(String, Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime))))))", - "Map(LowCardinality(FixedString(10)), Array(Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime('UTC'))))))))" + "Map(LowCardinality(FixedString(10)), Array(Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime('UTC'))))))))", + "Point", + "Ring", + "Polygon", + "MultiPolygon" }; // Check that Type::IsEqual returns true only if: diff --git a/ut/utils.cpp b/ut/utils.cpp index 9f8d9cab..1cb5978c 100644 --- a/ut/utils.cpp +++ b/ut/utils.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -47,10 +48,14 @@ std::ostream& operator<<(std::ostream & ostr, const DateTimeValue & time) { return ostr << buffer; } -template +template ().At(0)) > bool doPrintValue(const ColumnRef & c, const size_t row, std::ostream & ostr) { if (const auto & casted_c = c->As()) { - ostr << static_cast(casted_c->At(row)); + if constexpr (is_container_v>) { + ostr << PrintContainer{static_cast(casted_c->At(row))}; + } else { + ostr << static_cast(casted_c->At(row)); + } return true; } return false; @@ -173,7 +178,11 @@ std::ostream & printColumnValue(const ColumnRef& c, const size_t row, std::ostre || doPrintValue(c, row, ostr) || doPrintValue(c, row, ostr) || doPrintValue(c, row, ostr) - || doPrintValue(c, row, ostr); + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr); if (!r) ostr << "Unable to print value of type " << c->GetType().GetName();