Increase max decimal precision to 76 (decimal256)

* Changelog entry Type: feature Component: type system Support for decimals with precision from 35 to 76. This PR adds decimal256 support to YT. Additionally: - Decimal256 is supported in skiff as int256. - ClickHouse decimal precisions from 35 to 76 are supported in CHYT. --- Pull Request resolved: ytsaurus/ytsaurus#798 Co-authored-by: dakovalkov <[email protected]> Co-authored-by: ermolovd <[email protected]> commit_hash:73c4809966cf4c625e6007d31b5dde14bd80e829
ytsaurus · Oct 15, 2024 · 29643c7 · 29643c7
1 parent 49527fd
commit 29643c7
Show file tree

Hide file tree

Showing 14 changed files with 654 additions and 123 deletions.
diff --git a/library/cpp/skiff/public.h b/library/cpp/skiff/public.h
@@ -15,11 +15,13 @@ enum class EWireType
     Int32    /* "int32" */,
     Int64    /* "int64" */,
     Int128   /* "int128" */,
+    Int256   /* "int256" */,
     Uint8    /* "uint8" */,
     Uint16   /* "uint16" */,
     Uint32   /* "uint32" */,
     Uint64   /* "uint64" */,
     Uint128  /* "uint128" */,
+    Uint256  /* "uint256" */,
     Double   /* "double" */,
     Boolean  /* "boolean" */,
     String32 /* "string32" */,

diff --git a/library/cpp/skiff/skiff.cpp b/library/cpp/skiff/skiff.cpp
@@ -32,6 +32,18 @@ bool operator!=(TUint128 lhs, TUint128 rhs)
 
 ////////////////////////////////////////////////////////////////////////////////
 
+bool operator==(const TInt256& lhs, const TInt256& rhs)
+{
+    return lhs.Parts == rhs.Parts;
+}
+
+bool operator==(const TUint256& lhs, const TUint256& rhs)
+{
+    return lhs.Parts == rhs.Parts;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
 TUncheckedSkiffParser::TUncheckedSkiffParser(IZeroCopyInput* underlying)
     : Underlying_(underlying)
     , Buffer_(512 * 1024)
@@ -95,6 +107,26 @@ TUint128 TUncheckedSkiffParser::ParseUint128()
     return {low, high};
 }
 
+TInt256 TUncheckedSkiffParser::ParseInt256()
+{
+    TInt256 result;
+    for (auto& part : result.Parts) {
+        part = ParseSimple<ui64>();
+    }
+
+    return result;
+}
+
+TUint256 TUncheckedSkiffParser::ParseUint256()
+{
+    TUint256 result;
+    for (auto& part : result.Parts) {
+        part  = ParseSimple<ui64>();
+    }
+
+    return result;
+}
+
 double TUncheckedSkiffParser::ParseDouble()
 {
     return ParseSimple<double>();
@@ -274,6 +306,18 @@ TUint128 TCheckedSkiffParser::ParseUint128()
     return Parser_.ParseUint128();
 }
 
+TInt256 TCheckedSkiffParser::ParseInt256()
+{
+    Validator_->OnSimpleType(EWireType::Int256);
+    return Parser_.ParseInt256();
+}
+
+TUint256 TCheckedSkiffParser::ParseUint256()
+{
+    Validator_->OnSimpleType(EWireType::Uint256);
+    return Parser_.ParseUint256();
+}
+
 double TCheckedSkiffParser::ParseDouble()
 {
     Validator_->OnSimpleType(EWireType::Double);
@@ -389,6 +433,20 @@ void TUncheckedSkiffWriter::WriteUint128(TUint128 value)
     WriteSimple<ui64>(value.High);
 }
 
+void TUncheckedSkiffWriter::WriteInt256(const TInt256& value)
+{
+    for (auto part : value.Parts) {
+        WriteSimple<ui64>(part);
+    }
+}
+
+void TUncheckedSkiffWriter::WriteUint256(const TUint256& value)
+{
+    for (auto part : value.Parts) {
+        WriteSimple<ui64>(part);
+    }
+}
+
 void TUncheckedSkiffWriter::WriteUint8(ui8 value)
 {
     WriteSimple<ui8>(value);
@@ -551,6 +609,18 @@ void TCheckedSkiffWriter::WriteUint128(TUint128 value)
     Writer_.WriteUint128(value);
 }
 
+void TCheckedSkiffWriter::WriteInt256(TInt256 value)
+{
+    Validator_->OnSimpleType(EWireType::Int256);
+    Writer_.WriteInt256(std::move(value));
+}
+
+void TCheckedSkiffWriter::WriteUint256(TUint256 value)
+{
+    Validator_->OnSimpleType(EWireType::Uint256);
+    Writer_.WriteUint256(std::move(value));
+}
+
 void TCheckedSkiffWriter::WriteString32(TStringBuf value)
 {
     Validator_->OnSimpleType(EWireType::String32);

diff --git a/library/cpp/skiff/skiff.h b/library/cpp/skiff/skiff.h
@@ -49,6 +49,23 @@ bool operator!=(TUint128 lhs, TUint128 rhs);
 
 ////////////////////////////////////////////////////////////////////////////////
 
+struct TInt256
+{
+    std::array<ui64, 4> Parts;
+};
+
+struct TUint256
+{
+    std::array<ui64, 4> Parts;
+};
+
+// Operator != is synthesized since C++ 20.
+bool operator==(const TInt256& lhs, const TInt256& rhs);
+
+bool operator==(const TUint256& lhs, const TUint256& rhs);
+
+////////////////////////////////////////////////////////////////////////////////
+
 class TUncheckedSkiffParser
 {
 public:
@@ -68,6 +85,9 @@ class TUncheckedSkiffParser
     TInt128 ParseInt128();
     TUint128 ParseUint128();
 
+    TInt256 ParseInt256();
+    TUint256 ParseUint256();
+
     double ParseDouble();
 
     bool ParseBoolean();
@@ -127,6 +147,9 @@ class TCheckedSkiffParser
     TInt128 ParseInt128();
     TUint128 ParseUint128();
 
+    TInt256 ParseInt256();
+    TUint256 ParseUint256();
+
     double ParseDouble();
 
     bool ParseBoolean();
@@ -177,6 +200,9 @@ class TUncheckedSkiffWriter
     void WriteInt128(TInt128 value);
     void WriteUint128(TUint128 value);
 
+    void WriteInt256(const TInt256& value);
+    void WriteUint256(const TUint256& value);
+
     void WriteString32(TStringBuf value);
 
     void WriteYson32(TStringBuf value);
@@ -223,6 +249,9 @@ class TCheckedSkiffWriter
     void WriteInt128(TInt128 value);
     void WriteUint128(TUint128 value);
 
+    void WriteInt256(TInt256 value);
+    void WriteUint256(TUint256 value);
+
     void WriteString32(TStringBuf value);
 
     void WriteYson32(TStringBuf value);

diff --git a/library/cpp/skiff/skiff_schema-inl.h b/library/cpp/skiff/skiff_schema-inl.h
@@ -19,12 +19,14 @@ inline bool IsSimpleType(EWireType type)
         case EWireType::Int32:
         case EWireType::Int64:
         case EWireType::Int128:
+        case EWireType::Int256:
 
         case EWireType::Uint8:
         case EWireType::Uint16:
         case EWireType::Uint32:
         case EWireType::Uint64:
         case EWireType::Uint128:
+        case EWireType::Uint256:
 
         case EWireType::Double:
         case EWireType::Boolean:

diff --git a/library/cpp/skiff/skiff_validator.cpp b/library/cpp/skiff/skiff_validator.cpp
@@ -363,12 +363,14 @@ std::shared_ptr<IValidatorNode> CreateUsageValidatorNode(const std::shared_ptr<T
         case EWireType::Int32:
         case EWireType::Int64:
         case EWireType::Int128:
+        case EWireType::Int256:
 
         case EWireType::Uint8:
         case EWireType::Uint16:
         case EWireType::Uint32:
         case EWireType::Uint64:
         case EWireType::Uint128:
+        case EWireType::Uint256:
 
         case EWireType::Double:
         case EWireType::Boolean:

diff --git a/library/cpp/skiff/unittests/skiff_ut.cpp b/library/cpp/skiff/unittests/skiff_ut.cpp
@@ -209,6 +209,29 @@ Y_UNIT_TEST_SUITE(Skiff)
         UNIT_ASSERT_EQUAL(parser.ParseInt128(), val2);
     }
 
+    Y_UNIT_TEST(TestInt256)
+    {
+        TBufferStream bufferStream;
+
+        auto schema = CreateSimpleTypeSchema(EWireType::Int256);
+
+        const TInt256 val1 = {0x1924cd4aeb9ced82,  0x0885e83f456d6a7e, 0xe9ba36585eccae1a, 0x7854b6f9ce448be9};
+        const TInt256 val2 = {0xe9ba36585eccae1a, 0x1924cd4aeb9ced82, 0x0885e83f456d6a7e, static_cast<ui64>(-0x7854b6f9ce448be9)};
+
+        TCheckedSkiffWriter writer(schema, &bufferStream);
+        writer.WriteInt256(val1);
+        writer.WriteInt256(val2);
+        writer.Finish();
+
+        UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+            "82ed9ceb4acd2419" "7e6a6d453fe88508" "1aaecc5e5836bae9" "e98b44cef9b65478"
+            "1aaecc5e5836bae9" "82ed9ceb4acd2419" "7e6a6d453fe88508" "1774bb310649ab87");
+
+        TCheckedSkiffParser parser(schema, &bufferStream);
+        UNIT_ASSERT_EQUAL(parser.ParseInt256(), val1);
+        UNIT_ASSERT_EQUAL(parser.ParseInt256(), val2);
+    }
+
     Y_UNIT_TEST(TestUint128)
     {
         TBufferStream bufferStream;
@@ -232,6 +255,29 @@ Y_UNIT_TEST_SUITE(Skiff)
         UNIT_ASSERT_EQUAL(parser.ParseUint128(), val2);
     }
 
+    Y_UNIT_TEST(TestUint256)
+    {
+        TBufferStream bufferStream;
+
+        auto schema = CreateSimpleTypeSchema(EWireType::Uint256);
+
+        const auto val1 = TUint256{0x1924cd4aeb9ced82,  0x7854b6f9ce448be9, 0x8854b6f9ce448be9, 0x0885e83f456d6a7e};
+        const auto val2 = TUint256{0xe9ba36585eccae1a,  0x8854b6f9ce448be9, 0x1924cd4aeb9ced82, 0xabacabadabacaba0};
+
+        TCheckedSkiffWriter writer(schema, &bufferStream);
+        writer.WriteUint256(val1);
+        writer.WriteUint256(val2);
+        writer.Finish();
+
+        UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+            "82ed9ceb4acd2419" "e98b44cef9b65478" "e98b44cef9b65488" "7e6a6d453fe88508"
+            "1aaecc5e5836bae9" "e98b44cef9b65488" "82ed9ceb4acd2419" "a0abacabadabacab");
+
+        TCheckedSkiffParser parser(schema, &bufferStream);
+        UNIT_ASSERT_EQUAL(parser.ParseUint256(), val1);
+        UNIT_ASSERT_EQUAL(parser.ParseUint256(), val2);
+    }
+
     Y_UNIT_TEST(TestBoolean)
     {
         auto schema = CreateSimpleTypeSchema(EWireType::Boolean);

diff --git a/yt/yt/client/table_client/logical_type.h b/yt/yt/client/table_client/logical_type.h
@@ -170,7 +170,7 @@ class TDecimalLogicalType
 {
 public:
     static constexpr int MinPrecision = 1;
-    static constexpr int MaxPrecision = 35;
+    static constexpr int MaxPrecision = 76;
 
 public:
     TDecimalLogicalType(int precision, int scale);