From a9f26da6014a39320ff5b0bcee09cf9f32ea89d7 Mon Sep 17 00:00:00 2001 From: arkady-e1ppa Date: Tue, 15 Oct 2024 17:23:48 +0300 Subject: [PATCH] YT-22307: Drop include dependency client -on-> ytlib commit_hash:ccc25ebeed4973b3f8f133835e308fad69589169 --- yt/yt/client/api/distributed_table_session.h | 2 +- yt/yt/client/chunk_client/public.h | 6 + .../table_client/table_upload_options.cpp | 389 ++++++++++++++++++ .../table_client/table_upload_options.h | 2 +- yt/yt/client/ya.make | 1 + yt/yt/ytlib/chunk_client/public.h | 284 ------------- yt/yt/ytlib/misc/public.h | 51 --- yt/yt/ytlib/node_tracker_client/public.h | 89 ---- yt/yt/ytlib/object_client/public.h | 61 --- yt/yt/ytlib/table_client/public.h | 170 -------- 10 files changed, 398 insertions(+), 657 deletions(-) create mode 100644 yt/yt/client/table_client/table_upload_options.cpp rename yt/yt/{ytlib => client}/table_client/table_upload_options.h (98%) delete mode 100644 yt/yt/ytlib/chunk_client/public.h delete mode 100644 yt/yt/ytlib/misc/public.h delete mode 100644 yt/yt/ytlib/node_tracker_client/public.h delete mode 100644 yt/yt/ytlib/object_client/public.h delete mode 100644 yt/yt/ytlib/table_client/public.h diff --git a/yt/yt/client/api/distributed_table_session.h b/yt/yt/client/api/distributed_table_session.h index 193228420..c8c074c84 100644 --- a/yt/yt/client/api/distributed_table_session.h +++ b/yt/yt/client/api/distributed_table_session.h @@ -2,7 +2,7 @@ #include "public.h" -#include +#include #include diff --git a/yt/yt/client/chunk_client/public.h b/yt/yt/client/chunk_client/public.h index 720aeed80..5c93ee55f 100644 --- a/yt/yt/client/chunk_client/public.h +++ b/yt/yt/client/chunk_client/public.h @@ -91,6 +91,12 @@ YT_DEFINE_ERROR_ENUM( ((ForbiddenErasureCodec) (762)) ); +DEFINE_ENUM(EUpdateMode, + ((None) (0)) + ((Append) (1)) + ((Overwrite) (2)) +); + using TChunkId = NObjectClient::TObjectId; extern const TChunkId NullChunkId; diff --git a/yt/yt/client/table_client/table_upload_options.cpp b/yt/yt/client/table_client/table_upload_options.cpp new file mode 100644 index 000000000..6e3bddcf2 --- /dev/null +++ b/yt/yt/client/table_client/table_upload_options.cpp @@ -0,0 +1,389 @@ +#include "table_upload_options.h" +#include "helpers.h" + +#include + +#include + +#include + +namespace NYT::NTableClient { + +using namespace NChunkClient; +using namespace NCompression; +using namespace NCypressClient; +using namespace NYPath; +using namespace NYTree; + +//////////////////////////////////////////////////////////////////////////////// + +TEpochSchema::TEpochSchema(const TEpochSchema& other) +{ + *this = other; +} + +TEpochSchema& TEpochSchema::operator=(const TEpochSchema& other) +{ + TableSchema_ = other.TableSchema_; + Revision_ += other.Revision_ + 1; + return *this; +} + +TEpochSchema& TEpochSchema::operator=(TTableSchemaPtr schema) +{ + Set(schema); + return *this; +} + +const TTableSchema* TEpochSchema::operator->() const +{ + return TableSchema_.Get(); +} + +const TTableSchemaPtr& TEpochSchema::operator*() const +{ + return TableSchema_; +} + +const TTableSchemaPtr& TEpochSchema::Get() const +{ + return TableSchema_; +} + +ui64 TEpochSchema::GetRevision() const +{ + return Revision_; +} + +ui64 TEpochSchema::Set(const TTableSchemaPtr& schema) +{ + TableSchema_ = schema; + return ++Revision_; +} + +void TEpochSchema::Persist(const NPhoenix::TPersistenceContext& context) +{ + using NYT::Persist; + + Persist(context, Revision_); + Persist>(context, TableSchema_); +} + +ui64 TEpochSchema::Reset() +{ + TableSchema_ = New(); + return ++Revision_; +} + +//////////////////////////////////////////////////////////////////////////////// + +TTableSchemaPtr TTableUploadOptions::GetUploadSchema() const +{ + switch (SchemaModification) { + case ETableSchemaModification::None: + return TableSchema.Get(); + + case ETableSchemaModification::UnversionedUpdate: + return TableSchema->ToUnversionedUpdate(/*sorted*/ true); + + default: + YT_ABORT(); + } +} + +void TTableUploadOptions::Persist(const NPhoenix::TPersistenceContext& context) +{ + using NYT::Persist; + + Persist(context, UpdateMode); + Persist(context, LockMode); + // COMPAT(h0pless): NControllerAgent::ESnapshotVersion::AddChunkSchemas + if (context.GetVersion() >= 301300) { + Persist(context, TableSchema); + } else { + TTableSchemaPtr schema; + Persist>(context, schema); + TableSchema.Set(schema); + } + Persist(context, SchemaId); + Persist(context, SchemaModification); + // COMPAT(dave11ar): NControllerAgent::ESnapshotVersion::VersionedMapReduceWrite + if (context.GetVersion() >= 301602) { + Persist(context, VersionedWriteOptions); + } + Persist(context, SchemaMode); + Persist(context, OptimizeFor); + // COMPAT(babenko): NControllerAgent::ESnapshotVersion::ChunkFormat + if (context.GetVersion() >= 301103) { + Persist(context, ChunkFormat); + } + Persist(context, CompressionCodec); + Persist(context, ErasureCodec); + Persist(context, EnableStripedErasure); + Persist(context, SecurityTags); + Persist(context, PartiallySorted); +} + +//////////////////////////////////////////////////////////////////////////////// + +static void ValidateSortColumnsEqual(const TSortColumns& sortColumns, const TTableSchema& schema) +{ + if (sortColumns != schema.GetSortColumns()) { + THROW_ERROR_EXCEPTION("YPath attribute \"sorted_by\" must be compatible with table schema for a \"strong\" schema mode") + << TErrorAttribute("sort_columns", sortColumns) + << TErrorAttribute("table_schema", schema); + } +} + +static void ValidateAppendKeyColumns(const TSortColumns& sortColumns, const TTableSchema& schema, i64 rowCount) +{ + ValidateSortColumns(sortColumns); + + if (rowCount == 0) { + return; + } + + auto tableSortColumns = schema.GetSortColumns(); + bool areKeyColumnsCompatible = true; + if (tableSortColumns.size() < sortColumns.size()) { + areKeyColumnsCompatible = false; + } else { + for (int i = 0; i < std::ssize(sortColumns); ++i) { + if (tableSortColumns[i] != sortColumns[i]) { + areKeyColumnsCompatible = false; + break; + } + } + } + + if (!areKeyColumnsCompatible) { + THROW_ERROR_EXCEPTION("Sort columns mismatch while trying to append sorted data into a non-empty table") + << TErrorAttribute("append_sort_columns", sortColumns) + << TErrorAttribute("table_sort_columns", tableSortColumns); + } +} + +const std::vector& GetTableUploadOptionsAttributeKeys() +{ + static const std::vector Result{ + "schema_mode", + "optimize_for", + "chunk_format", + "compression_codec", + "erasure_codec", + "enable_striped_erasure", + "dynamic" + }; + return Result; +} + +TTableUploadOptions GetTableUploadOptions( + const TRichYPath& path, + const IAttributeDictionary& cypressTableAttributes, + const TTableSchemaPtr& schema, + i64 rowCount) +{ + auto schemaMode = cypressTableAttributes.Get("schema_mode"); + auto optimizeFor = cypressTableAttributes.Get("optimize_for"); + auto chunkFormat = cypressTableAttributes.Find("chunk_format"); + auto compressionCodec = cypressTableAttributes.Get("compression_codec"); + auto erasureCodec = cypressTableAttributes.Get("erasure_codec", NErasure::ECodec::None); + auto enableStripedErasure = cypressTableAttributes.Get("enable_striped_erasure", false); + auto dynamic = cypressTableAttributes.Get("dynamic"); + + // Validate "optimize_for" and "chunk_format" compatibility. + if (chunkFormat) { + ValidateTableChunkFormatAndOptimizeFor(*chunkFormat, optimizeFor); + } + + // Some ypath attributes are not compatible with attribute "schema". + if (path.GetAppend() && path.GetSchema()) { + THROW_ERROR_EXCEPTION("YPath attributes \"append\" and \"schema\" are not compatible") + << TErrorAttribute("path", path); + } + + if (!path.GetSortedBy().empty() && path.GetSchema()) { + THROW_ERROR_EXCEPTION("YPath attributes \"sorted_by\" and \"schema\" are not compatible") + << TErrorAttribute("path", path); + } + + // Dynamic tables have their own requirements as well. + if (dynamic) { + if (path.GetSchema()) { + THROW_ERROR_EXCEPTION("YPath attribute \"schema\" cannot be set on a dynamic table") + << TErrorAttribute("path", path); + } + + if (!path.GetSortedBy().empty()) { + THROW_ERROR_EXCEPTION("YPath attribute \"sorted_by\" cannot be set on a dynamic table") + << TErrorAttribute("path", path); + } + } + + TTableUploadOptions result; + // NB: Saving schema to make sure that if changes are applied to it the schema revision also changes. + result.TableSchema = schema; + auto pathSchema = path.GetSchema(); + if (path.GetAppend() && !path.GetSortedBy().empty() && (schemaMode == ETableSchemaMode::Strong)) { + ValidateSortColumnsEqual(path.GetSortedBy(), *schema); + + result.LockMode = ELockMode::Exclusive; + result.UpdateMode = EUpdateMode::Append; + result.SchemaMode = ETableSchemaMode::Strong; + } else if (path.GetAppend() && !path.GetSortedBy().empty() && (schemaMode == ETableSchemaMode::Weak)) { + // Old behaviour. + ValidateAppendKeyColumns(path.GetSortedBy(), *schema, rowCount); + + result.LockMode = ELockMode::Exclusive; + result.UpdateMode = EUpdateMode::Append; + result.SchemaMode = ETableSchemaMode::Weak; + result.TableSchema = TTableSchema::FromSortColumns(path.GetSortedBy()); + } else if (path.GetAppend() && path.GetSortedBy().empty() && (schemaMode == ETableSchemaMode::Strong)) { + result.LockMode = (schema->IsSorted() && !dynamic) ? ELockMode::Exclusive : ELockMode::Shared; + result.UpdateMode = EUpdateMode::Append; + result.SchemaMode = ETableSchemaMode::Strong; + } else if (path.GetAppend() && path.GetSortedBy().empty() && (schemaMode == ETableSchemaMode::Weak)) { + // Old behaviour - reset key columns if there were any. + result.LockMode = ELockMode::Shared; + result.UpdateMode = EUpdateMode::Append; + result.SchemaMode = ETableSchemaMode::Weak; + result.TableSchema.Reset(); + } else if (!path.GetAppend() && !path.GetSortedBy().empty() && (schemaMode == ETableSchemaMode::Strong)) { + ValidateSortColumnsEqual(path.GetSortedBy(), *schema); + + result.LockMode = ELockMode::Exclusive; + result.UpdateMode = EUpdateMode::Overwrite; + result.SchemaMode = ETableSchemaMode::Strong; + } else if (!path.GetAppend() && !path.GetSortedBy().empty() && (schemaMode == ETableSchemaMode::Weak)) { + result.LockMode = ELockMode::Exclusive; + result.UpdateMode = EUpdateMode::Overwrite; + result.SchemaMode = ETableSchemaMode::Weak; + result.TableSchema = TTableSchema::FromSortColumns(path.GetSortedBy()); + } else if (!path.GetAppend() && pathSchema && (schemaMode == ETableSchemaMode::Strong)) { + result.LockMode = ELockMode::Exclusive; + result.UpdateMode = EUpdateMode::Overwrite; + result.SchemaMode = ETableSchemaMode::Strong; + result.TableSchema = pathSchema; + } else if (!path.GetAppend() && pathSchema && (schemaMode == ETableSchemaMode::Weak)) { + // Change from Weak to Strong schema mode. + result.LockMode = ELockMode::Exclusive; + result.UpdateMode = EUpdateMode::Overwrite; + result.SchemaMode = ETableSchemaMode::Strong; + result.TableSchema = pathSchema; + } else if (!path.GetAppend() && path.GetSortedBy().empty() && (schemaMode == ETableSchemaMode::Strong)) { + result.LockMode = ELockMode::Exclusive; + result.UpdateMode = EUpdateMode::Overwrite; + result.SchemaMode = ETableSchemaMode::Strong; + } else if (!path.GetAppend() && path.GetSortedBy().empty() && (schemaMode == ETableSchemaMode::Weak)) { + result.LockMode = ELockMode::Exclusive; + result.UpdateMode = EUpdateMode::Overwrite; + result.SchemaMode = ETableSchemaMode::Weak; + result.TableSchema.Reset(); + } else { + // Do not use YT_ABORT here, since this code is executed inside scheduler. + THROW_ERROR_EXCEPTION("Failed to define upload parameters") + << TErrorAttribute("path", path) + << TErrorAttribute("schema_mode", schemaMode) + << TErrorAttribute("schema", *schema); + } + + if (path.GetAppend() && path.GetOptimizeFor()) { + THROW_ERROR_EXCEPTION("YPath attributes \"append\" and \"optimize_for\" are not compatible") + << TErrorAttribute("path", path); + } + + result.OptimizeFor = path.GetOptimizeFor() ? *path.GetOptimizeFor() : optimizeFor; + result.ChunkFormat = path.GetChunkFormat() ? *path.GetChunkFormat() : chunkFormat; + + if (path.GetAppend() && path.GetCompressionCodec()) { + THROW_ERROR_EXCEPTION("YPath attributes \"append\" and \"compression_codec\" are not compatible") + << TErrorAttribute("path", path); + } + + if (path.GetCompressionCodec()) { + result.CompressionCodec = *path.GetCompressionCodec(); + } else { + result.CompressionCodec = compressionCodec; + } + + if (path.GetAppend() && path.GetErasureCodec()) { + THROW_ERROR_EXCEPTION("YPath attributes \"append\" and \"erasure_codec\" are not compatible") + << TErrorAttribute("path", path); + } + + result.ErasureCodec = path.GetErasureCodec().value_or(erasureCodec); + result.EnableStripedErasure = enableStripedErasure; + + if (path.GetSchemaModification() == ETableSchemaModification::UnversionedUpdateUnsorted) { + THROW_ERROR_EXCEPTION("YPath attribute \"schema_modification\" cannot have value %Qlv for output tables", + path.GetSchemaModification()) + << TErrorAttribute("path", path); + } else if (!dynamic && path.GetSchemaModification() != ETableSchemaModification::None) { + THROW_ERROR_EXCEPTION("YPath attribute \"schema_modification\" can have value %Qlv only for dynamic tables", + path.GetSchemaModification()) + << TErrorAttribute("path", path); + } + result.SchemaModification = path.GetSchemaModification(); + + auto versionedWriteOptions = path.GetVersionedWriteOptions(); + if (!dynamic && versionedWriteOptions.WriteMode != EVersionedIOMode::Default) { + THROW_ERROR_EXCEPTION("YPath attribute \"versioned_write_options/write_mode\" can have value %Qlv only for dynamic tables", + versionedWriteOptions.WriteMode) + << TErrorAttribute("path", path); + } + if (versionedWriteOptions.WriteMode != EVersionedIOMode::Default && path.GetSchemaModification() != ETableSchemaModification::None) { + THROW_ERROR_EXCEPTION("YPath attributes \"versioned_write_options/write_mode\" and \"schema_modification\"" + "can not be set in non-trivial state together: \"versioned_write_options/write_mode\" is %Qlv, \"schema_modification\" is %Qlv", + versionedWriteOptions.WriteMode, + path.GetSchemaModification()) + << TErrorAttribute("path", path); + } + result.VersionedWriteOptions = versionedWriteOptions; + + if (!dynamic && path.GetPartiallySorted()) { + THROW_ERROR_EXCEPTION("YPath attribute \"partially_sorted\" can be set only for dynamic tables") + << TErrorAttribute("path", path); + } + result.PartiallySorted = path.GetPartiallySorted(); + + result.SecurityTags = path.GetSecurityTags(); + + return result; +} + +TTableUploadOptions GetFileUploadOptions( + const TRichYPath& path, + const IAttributeDictionary& cypressTableAttributes) +{ + auto compressionCodec = cypressTableAttributes.Get("compression_codec"); + auto enableStripedErasure = cypressTableAttributes.Get("enable_striped_erasure", false); + auto erasureCodec = cypressTableAttributes.Get("erasure_codec", NErasure::ECodec::None); + + TTableUploadOptions result; + + if (path.GetAppend()) { + THROW_ERROR_EXCEPTION("Attribute \"append\" is not supported for files") + << TErrorAttribute("path", path); + } + + // NB(coteeq): Fill for sanity. They should not have impact on behaviour, because + // RichYPath's compression_codec & erasure_codec are disallowed in remote copy. + // TODO(coteeq): Make it YT_VERIFY + if (path.GetCompressionCodec() || path.GetErasureCodec()) { + THROW_ERROR_EXCEPTION("\"compression_codec\" and \"erasure_codec\" are disallowed for files") + << TErrorAttribute("path", path); + } + + result.CompressionCodec = compressionCodec; + result.ErasureCodec = erasureCodec; + result.EnableStripedErasure = enableStripedErasure; + result.SecurityTags = path.GetSecurityTags(); + result.LockMode = ELockMode::Exclusive; + result.UpdateMode = EUpdateMode::Overwrite; + + return result; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NTableClient diff --git a/yt/yt/ytlib/table_client/table_upload_options.h b/yt/yt/client/table_client/table_upload_options.h similarity index 98% rename from yt/yt/ytlib/table_client/table_upload_options.h rename to yt/yt/client/table_client/table_upload_options.h index 20cb95440..91bfa5fc8 100644 --- a/yt/yt/ytlib/table_client/table_upload_options.h +++ b/yt/yt/client/table_client/table_upload_options.h @@ -2,7 +2,7 @@ #include "public.h" -#include +#include #include #include diff --git a/yt/yt/client/ya.make b/yt/yt/client/ya.make index 9e468fbf8..a09ba3338 100644 --- a/yt/yt/client/ya.make +++ b/yt/yt/client/ya.make @@ -127,6 +127,7 @@ SRCS( table_client/schemaless_buffered_dynamic_table_writer.cpp table_client/schemaless_dynamic_table_writer.cpp table_client/serialize.cpp + table_client/table_upload_options.cpp table_client/logical_type.cpp table_client/merge_table_schemas.cpp table_client/name_table.cpp diff --git a/yt/yt/ytlib/chunk_client/public.h b/yt/yt/ytlib/chunk_client/public.h deleted file mode 100644 index 0cb2ae86d..000000000 --- a/yt/yt/ytlib/chunk_client/public.h +++ /dev/null @@ -1,284 +0,0 @@ -#pragma once - -#include - -#include - -#include - -#include - -#include - -#include - -#include - -#include - -namespace NYT::NChunkClient { - -//////////////////////////////////////////////////////////////////////////////// - -namespace NProto { - -//////////////////////////////////////////////////////////////////////////////// - -class TReqFetch; - -class TReqExportChunks; -class TRspExportChunks; - -class TReqImportChunks; -class TRspImportChunks; - -class TReqExecuteBatch; -class TRspExecuteBatch; - -class TDataSource; -class TDataSourceDirectoryExt; - -class TDataSink; -class TDataSinkDirectoryExt; - -class TReqGetChunkMeta; - -class TAllyReplicasInfo; -class TChunkReplicaAnnouncement; -class TChunkReplicaAnnouncementRequest; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NProto - -//////////////////////////////////////////////////////////////////////////////// - -using NTableClient::TLoadContext; -using NTableClient::TSaveContext; -using NTableClient::TPersistenceContext; - -constexpr int MaxMasterChunkMetaExtensions = 6; - -struct TBlock; - -using TMediumId = NObjectClient::TObjectId; - -using TReadSessionId = NObjectClient::TObjectId; - -struct TSessionId; - -constexpr NRpc::TRealmId ProxyingDataNodeServiceRealmId = TGuid(0xd452d72f, 0x3142caa3); - -constexpr int DefaultPartIndex = -1; - -//! Estimated memory overhead per chunk reader. -constexpr i64 ChunkReaderMemorySize = 16_KB; - -constexpr int MaxMediumPriority = 10; - -constexpr i64 DefaultMaxBlockSize = 16_MB; -constexpr int MaxInputChunkReplicaCount = 16; - -//! Represents an offset inside a chunk. -using TBlockOffset = i64; - -//! A |(chunkId, blockIndex)| pair. -struct TBlockId; - -using TConsistentReplicaPlacementHash = ui64; -constexpr TConsistentReplicaPlacementHash NullConsistentReplicaPlacementHash = 0; - -//! All chunks are uniformly divided into |ChunkShardCount| shards. -// BEWARE: Changing this value requires reign promotion since rolling update -// is not possible. -constexpr int ChunkShardCount = 60; -static_assert(ChunkShardCount < std::numeric_limits::max(), "ChunkShardCount must fit into i8"); - -//! Typical chunk location count per data node. -constexpr int TypicalChunkLocationCount = 20; - -struct TAllyReplicasInfo; - -constexpr int WholeBlockFragmentRequestLength = -1; - -DEFINE_BIT_ENUM(EBlockType, - ((None) (0x0000)) - //! This basically comprises any block regardless of its semantics (data or some system block). - ((CompressedData) (0x0001)) - //! Uncompressed data block. - ((UncompressedData) (0x0002)) - //! Hash table chunk index system block. - ((HashTableChunkIndex) (0x0004)) - //! Xor filter system block. - ((XorFilter) (0x0008)) - //! Blocks used by chunk fragment reader cache. - ((ChunkFragmentsData) (0x0010)) -); - -DEFINE_ENUM(EChunkType, - ((Unknown) (0)) - ((File) (1)) - ((Table) (2)) - ((Journal) (3)) - ((Hunk) (4)) -); - -//! Values must be contiguous. -DEFINE_ENUM(ESessionType, - ((User) (0)) - ((Replication) (1)) - ((Repair) (2)) -); - -DEFINE_ENUM(EUpdateMode, - ((None) (0)) - ((Append) (1)) - ((Overwrite) (2)) -); - -DEFINE_BIT_ENUM(EChunkFeatures, - ((None) (0x0000)) - ((DescendingSortOrder) (0x0001)) - ((StripedErasure) (0x0002)) - ((IndexedBlockFormat) (0x0004)) - ((SlimBlockFormat) (0x0008)) - ((UnversionedHunks) (0x0010)) - ((CompressedHunkValues) (0x0020)) - ((NoColumnMetaInChunkMeta) (0x0040)) -); - -DEFINE_ENUM(EChunkClientFeature, - // COMPAT(akozhikhov). - ((AllBlocksIndex) (0)) -); - -DEFINE_ENUM(EChunkMergerMode, - ((None) (0)) - ((Shallow) (1)) - ((Deep) (2)) - ((Auto) (3)) -); - -DEFINE_ENUM(EChunkListContentType, - ((Main) (0)) - ((Hunk) (1)) -); - -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_REFCOUNTED_CLASS(TRemoteReaderOptions) -DECLARE_REFCOUNTED_CLASS(TDispatcherConfig) -DECLARE_REFCOUNTED_CLASS(TDispatcherDynamicConfig) -DECLARE_REFCOUNTED_CLASS(TMultiChunkWriterOptions) -DECLARE_REFCOUNTED_CLASS(TMultiChunkReaderOptions) -DECLARE_REFCOUNTED_CLASS(TRemoteWriterOptions) -DECLARE_REFCOUNTED_CLASS(TMetaAggregatingWriterOptions) -DECLARE_REFCOUNTED_CLASS(TBlockCacheConfig) -DECLARE_REFCOUNTED_CLASS(TBlockCacheDynamicConfig) -DECLARE_REFCOUNTED_CLASS(TClientChunkMetaCacheConfig) -DECLARE_REFCOUNTED_CLASS(TChunkScraperConfig) -DECLARE_REFCOUNTED_CLASS(TChunkTeleporterConfig) -DECLARE_REFCOUNTED_CLASS(TMediumDirectorySynchronizerConfig) -DECLARE_REFCOUNTED_CLASS(TChunkReplicaCacheConfig) - -DECLARE_REFCOUNTED_STRUCT(IFetcherChunkScraper) - -DECLARE_REFCOUNTED_CLASS(TEncodingWriter) -DECLARE_REFCOUNTED_CLASS(TEncodingChunkWriter) -DECLARE_REFCOUNTED_CLASS(TBlockFetcher) -DECLARE_REFCOUNTED_CLASS(TSequentialBlockFetcher) - -DECLARE_REFCOUNTED_STRUCT(IChunkReader) -DECLARE_REFCOUNTED_STRUCT(IChunkFragmentReader) -DECLARE_REFCOUNTED_STRUCT(IChunkReaderAllowingRepair) - -DECLARE_REFCOUNTED_STRUCT(IReaderBase) -DECLARE_REFCOUNTED_STRUCT(IReaderFactory) - -DECLARE_REFCOUNTED_STRUCT(IMultiReaderManager) - -DECLARE_REFCOUNTED_CLASS(TTrafficMeter) - -DECLARE_REFCOUNTED_STRUCT(IChunkWriterBase) -DECLARE_REFCOUNTED_STRUCT(IMultiChunkWriter) -DECLARE_REFCOUNTED_STRUCT(IChunkWriter) -DECLARE_REFCOUNTED_STRUCT(IMetaAggregatingWriter) - -DECLARE_REFCOUNTED_STRUCT(IBlockCache) -DECLARE_REFCOUNTED_STRUCT(IClientBlockCache) - -DECLARE_REFCOUNTED_CLASS(TMemoryWriter) - -DECLARE_REFCOUNTED_CLASS(TInputChunk) -DECLARE_REFCOUNTED_CLASS(TInputChunkSlice) -DECLARE_REFCOUNTED_CLASS(TWeightedInputChunk) - -DECLARE_REFCOUNTED_STRUCT(TLegacyDataSlice) - -DECLARE_REFCOUNTED_CLASS(TDataSourceDirectory) -DECLARE_REFCOUNTED_CLASS(TDataSinkDirectory) - -DECLARE_REFCOUNTED_CLASS(TChunkScraper) -DECLARE_REFCOUNTED_CLASS(TScraperTask) -DECLARE_REFCOUNTED_CLASS(TThrottlerManager) -DECLARE_REFCOUNTED_CLASS(TChunkTeleporter) -DECLARE_REFCOUNTED_CLASS(TMediumDirectory) -DECLARE_REFCOUNTED_CLASS(TMediumDirectorySynchronizer) - -DECLARE_REFCOUNTED_CLASS(TChunkMetaFetcher) - -DECLARE_REFCOUNTED_CLASS(TMasterChunkSpecFetcher) -DECLARE_REFCOUNTED_CLASS(TTabletChunkSpecFetcher) - -DECLARE_REFCOUNTED_STRUCT(TChunkReaderStatistics) - -DECLARE_REFCOUNTED_STRUCT(IReaderMemoryManager) -DECLARE_REFCOUNTED_CLASS(TChunkReaderMemoryManager) - -DECLARE_REFCOUNTED_STRUCT(IChunkReplicaCache) - -DECLARE_REFCOUNTED_STRUCT(TChunkReaderHost) - -struct TChunkReaderMemoryManagerOptions; - -struct TUserObject; - -using TRefCountedChunkMeta = TRefCountedProto; -DECLARE_REFCOUNTED_TYPE(TRefCountedChunkMeta) - -DECLARE_REFCOUNTED_CLASS(TDeferredChunkMeta) - -DECLARE_REFCOUNTED_CLASS(TMemoryTrackedDeferredChunkMeta) - -// NB: TRefCountedBlocksExt needs weak pointers support. -using TRefCountedBlocksExt = TRefCountedProto; -DECLARE_REFCOUNTED_TYPE(TRefCountedBlocksExt) - -using TRefCountedMiscExt = TRefCountedProto; -DECLARE_REFCOUNTED_TYPE(TRefCountedMiscExt) - -using TPlacementId = TGuid; - -struct TDataSliceDescriptor; - -struct TInterruptDescriptor; - -class TCodecStatistics; - -struct TClientChunkReadOptions; - -using TDataCenterName = std::optional; - -DECLARE_REFCOUNTED_CLASS(TMemoryUsageGuard) - -DECLARE_REFCOUNTED_CLASS(TChunkReaderMemoryManagerHolder) - -DECLARE_REFCOUNTED_STRUCT(IMultiReaderMemoryManager) -DECLARE_REFCOUNTED_STRUCT(IReaderMemoryManagerHost) - -DECLARE_REFCOUNTED_STRUCT(ICachedChunkMeta) -DECLARE_REFCOUNTED_STRUCT(IClientChunkMetaCache) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NChunkClient diff --git a/yt/yt/ytlib/misc/public.h b/yt/yt/ytlib/misc/public.h deleted file mode 100644 index a25d6f9ae..000000000 --- a/yt/yt/ytlib/misc/public.h +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_REFCOUNTED_STRUCT(INodeMemoryTracker) - -//////////////////////////////////////////////////////////////////////////////// - -DEFINE_ENUM(EMemoryCategory, - ((Footprint) (0)) - ((BlockCache) (1)) - ((ChunkMeta) (2)) - ((ChunkBlockMeta) (10)) - ((ChunkBlocksExt) (16)) - ((ChunkJournalIndex) (20)) - ((Rpc) (21)) - ((UserJobs) (3)) - ((TabletStatic) (4)) - ((TabletDynamic) (5)) - // COMPAT(babenko): drop - ((BlobSession) (6)) - ((PendingDiskRead) (22)) - ((PendingDiskWrite) (23)) - ((VersionedChunkMeta) (7)) - ((SystemJobs) (8)) - ((Query) (9)) - ((TmpfsLayers) (11)) - ((MasterCache) (12)) - ((Lookup) (24)) - ((LookupRowsCache) (13)) - ((AllocFragmentation) (14)) - ((P2P) (15)) - ((Unknown) (17)) - ((Mixed) (18)) - ((TabletBackground) (19)) - ((JobInputBlockCache) (25)) - ((JobInputChunkMetaCache) (26)) - ((TableReplication) (27)) - ((ChaosReplicationIncoming) (28)) - ((ChaosReplicationOutgoing) (29)) - ((ReadTable) (30)) - ((Other) (31)) -); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/ytlib/node_tracker_client/public.h b/yt/yt/ytlib/node_tracker_client/public.h deleted file mode 100644 index c642a3c53..000000000 --- a/yt/yt/ytlib/node_tracker_client/public.h +++ /dev/null @@ -1,89 +0,0 @@ -#pragma once - -#include - -#include - -#include - -namespace NYT::NNodeTrackerClient { - -//////////////////////////////////////////////////////////////////////////////// - -namespace NProto { - -class TReqRegisterNode; -class TRspRegisterNode; - -} // namespace NProto - -//////////////////////////////////////////////////////////////////////////////// - -class TNodeDirectoryBuilder; - -DECLARE_REFCOUNTED_STRUCT(INodeDirectorySynchronizer) - -DECLARE_REFCOUNTED_CLASS(TNodeDirectorySynchronizerConfig) - -DECLARE_REFCOUNTED_STRUCT(INodeChannelFactory) - -DECLARE_REFCOUNTED_STRUCT(INodeStatusDirectory) - -DEFINE_ENUM(EMemoryLimitType, - ((None) (0)) - ((Static) (1)) - ((Dynamic) (2)) -); - -DEFINE_ENUM(ENodeState, - // Used internally. - ((Unknown) (-1)) - // Not registered. - ((Offline) (0)) - // Registered but did not report some of the heartbeats. - ((Registered) (1)) - // Registered and reported all the expected types of heartbeats - // at least once. - ((Online) (2)) - // Unregistered and placed into disposal queue. - ((Unregistered) (3)) - // Indicates that state varies across cells. - ((Mixed) (4)) - // Unregistered and ongoing disposal. - ((BeingDisposed) (5)) -); - -DEFINE_ENUM(ECellAggregatedStateReliability, - // Used internally. - ((Unknown) (0)) - // Node knows about this cell from config. - ((StaticallyKnown) (1)) - // Indicates that node will receive information about this cell dynamically, - // no need to take into account information about node from cell, - // marked as ECellAggregatedStateReliability::DuringPropagation during computing aggregated state on primary master. - ((DuringPropagation) (2)) - // Indicates that node already received information about this cell dynamically. - ((DynamicallyDiscovered) (3)) -); - -DEFINE_ENUM(ENodeRole, - ((MasterCache) (0)) - ((TimestampProvider) (1)) -); - -DEFINE_ENUM(ENodeFlavor, - // COMPAT(gritukan) - ((Cluster) (0)) - // Node that is used to store chunks. - ((Data) (1)) - // Node that is used to execute jobs. - ((Exec) (2)) - // Node that is used to host dynamic tables tablets. - ((Tablet) (3)) - // Node that is used to host chaos cells. - ((Chaos) (4)) -); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NNodeTrackerClient diff --git a/yt/yt/ytlib/object_client/public.h b/yt/yt/ytlib/object_client/public.h deleted file mode 100644 index d524b2c2a..000000000 --- a/yt/yt/ytlib/object_client/public.h +++ /dev/null @@ -1,61 +0,0 @@ -#pragma once - -#include - -#include - -namespace NYT::NObjectClient { - -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_REFCOUNTED_CLASS(TReqExecuteBatchWithRetriesConfig) - -DECLARE_REFCOUNTED_CLASS(TObjectAttributeCacheConfig) -DECLARE_REFCOUNTED_CLASS(TObjectAttributeCache) - -DECLARE_REFCOUNTED_CLASS(TObjectServiceCacheConfig) -DECLARE_REFCOUNTED_CLASS(TObjectServiceCacheDynamicConfig) -DECLARE_REFCOUNTED_CLASS(TCachingObjectServiceConfig) -DECLARE_REFCOUNTED_CLASS(TCachingObjectServiceDynamicConfig) - -DECLARE_REFCOUNTED_CLASS(TObjectServiceCacheEntry) -DECLARE_REFCOUNTED_CLASS(TObjectServiceCache) - -DECLARE_REFCOUNTED_CLASS(TAbcConfig) - -DECLARE_REFCOUNTED_STRUCT(ICachingObjectService) - -//////////////////////////////////////////////////////////////////////////////// - -DEFINE_ENUM(EMasterFeature, - ((OverlayedJournals) (0)) - ((Portals) (1)) - ((PortalExitSynchronization) (2)) -); - -// Some objects must be created and removed atomically. -// -// Let's consider accounts. In the absence of an atomic commit, it's possible -// that some cell knows about an account, and some other cell doesn't. Then, the -// former cell sending a chunk requisition update to the latter will cause -// trouble. -// -// Removal also needs two-phase (and even more!) locking since otherwise a primary master -// is unable to command the destruction of an object to its secondaries without risking -// that some secondary still holds a reference to the object. -DEFINE_ENUM_WITH_UNDERLYING_TYPE(EObjectLifeStage, ui8, - // Creation workflow - ((CreationStarted) (0)) - ((CreationPreCommitted) (1)) - ((CreationCommitted) (2)) - - // Removal workflow - ((RemovalStarted) (3)) - ((RemovalPreCommitted) (4)) - ((RemovalAwaitingCellsSync)(5)) - ((RemovalCommitted) (6)) -); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NObjectClient diff --git a/yt/yt/ytlib/table_client/public.h b/yt/yt/ytlib/table_client/public.h deleted file mode 100644 index aa337eec6..000000000 --- a/yt/yt/ytlib/table_client/public.h +++ /dev/null @@ -1,170 +0,0 @@ -#pragma once - -#include - -namespace NYT::NTableClient { - -//////////////////////////////////////////////////////////////////////////////// - -namespace NProto { - -class TVirtualValueDirectory; - -} // namespace NProto - -constexpr int DefaultPartitionTag = -1; - -// TODO(ifsmirnov): calculate actual estimates. -constexpr i64 DefaultRemoteDynamicStoreReaderMemoryEstimate = 64_MB; - -DECLARE_REFCOUNTED_CLASS(TChunkColumnMapping) - -struct TColumnIdMapping -{ - int ChunkSchemaIndex; - int ReaderSchemaIndex; -}; - -DECLARE_REFCOUNTED_CLASS(TTableSchema) - -class TSchemaDictionary; - -template -class TGenericColumnFilterDictionary; - -using TColumnFilterDictionary = TGenericColumnFilterDictionary; -using TColumnStableNameFilterDictionary = TGenericColumnFilterDictionary; - -class THorizontalBlockReader; - -struct THunkChunkRef; -struct THunkChunksInfo; -struct THunkChunkMeta; - -struct TTableReadSpec; -struct TFetchSingleTableReadSpecOptions; - -struct TBoundaryKeysExtension; -struct TColumnMetaExtension; -struct TKeyColumnsExtension; -struct TSamplesExtension; - -DECLARE_REFCOUNTED_STRUCT(TOffloadingReaderOptions) -DECLARE_REFCOUNTED_STRUCT(IOffloadingReader) - -DECLARE_REFCOUNTED_CLASS(TSamplesFetcher) - -DECLARE_REFCOUNTED_STRUCT(IChunkSliceFetcher) - -DECLARE_REFCOUNTED_CLASS(TChunkSliceSizeFetcher) - -DECLARE_REFCOUNTED_CLASS(TKeySetWriter) - -DECLARE_REFCOUNTED_STRUCT(ISchemalessChunkReader) -DECLARE_REFCOUNTED_STRUCT(ISchemalessChunkWriter) - -DECLARE_REFCOUNTED_STRUCT(ISchemalessMultiChunkReader) -DECLARE_REFCOUNTED_STRUCT(ISchemalessMultiChunkWriter) - -DECLARE_REFCOUNTED_CLASS(TPartitionChunkReader) -DECLARE_REFCOUNTED_CLASS(TPartitionMultiChunkReader) - -DECLARE_REFCOUNTED_STRUCT(IVersionedChunkWriter) -DECLARE_REFCOUNTED_STRUCT(IVersionedMultiChunkWriter) - -DECLARE_REFCOUNTED_STRUCT(IHunkChunkPayloadWriter) - -DECLARE_REFCOUNTED_STRUCT(ITimingReader) - -DECLARE_REFCOUNTED_STRUCT(IPartitioner) - -DECLARE_REFCOUNTED_CLASS(TVersionedRowsetReader) - -DECLARE_REFCOUNTED_STRUCT(TXorFilterMeta) -DECLARE_REFCOUNTED_CLASS(TColumnarChunkMeta) -DECLARE_REFCOUNTED_CLASS(TCachedVersionedChunkMeta) - -DECLARE_REFCOUNTED_CLASS(TColumnarStatisticsFetcher) - -DECLARE_REFCOUNTED_STRUCT(TChunkReaderPerformanceCounters) - -DECLARE_REFCOUNTED_STRUCT(TChunkLookupHashTable) - -DECLARE_REFCOUNTED_STRUCT(TChunkState) - -DECLARE_REFCOUNTED_STRUCT(TTabletSnapshot) - -DECLARE_REFCOUNTED_STRUCT(TVirtualValueDirectory) - -DECLARE_REFCOUNTED_STRUCT(IVersionedRowDigestBuilder) - -struct TOwningBoundaryKeys; - -struct TBlobTableSchema; -class TBlobTableWriter; - -struct TChunkTimestamps; - -DECLARE_REFCOUNTED_CLASS(TSkynetColumnEvaluator) - -DECLARE_REFCOUNTED_CLASS(TCachedBlockMeta) -DECLARE_REFCOUNTED_CLASS(TBlockMetaCache) - -DECLARE_REFCOUNTED_CLASS(TTableColumnarStatisticsCache) - -class TSchemafulRowMerger; -class TUnversionedRowMerger; -class TSamplingRowMerger; - -struct IVersionedRowMerger; - -DECLARE_REFCOUNTED_CLASS(TTableWriterOptions) -DECLARE_REFCOUNTED_CLASS(TTableReaderOptions) - -DECLARE_REFCOUNTED_CLASS(TBlobTableWriterConfig) -DECLARE_REFCOUNTED_CLASS(TBufferedTableWriterConfig) -DECLARE_REFCOUNTED_CLASS(TPartitionConfig) -DECLARE_REFCOUNTED_CLASS(TTableColumnarStatisticsCacheConfig) -DECLARE_REFCOUNTED_CLASS(THunkChunkPayloadWriterConfig) - -DECLARE_REFCOUNTED_STRUCT(IHunkChunkReaderStatistics) -DECLARE_REFCOUNTED_STRUCT(IHunkChunkWriterStatistics) -class THunkChunkReaderCounters; -class THunkChunkWriterCounters; - -class TSliceBoundaryKey; - -DEFINE_ENUM(ETableCollocationType, - ((Replication) (0)) -); - -DECLARE_REFCOUNTED_STRUCT(IChunkIndexBuilder) - -DECLARE_REFCOUNTED_STRUCT(IKeyFilter) -DECLARE_REFCOUNTED_STRUCT(IKeyFilterBuilder) - -DECLARE_REFCOUNTED_STRUCT(TKeyFilterStatistics) - -constexpr int VersionedBlockValueSize = 16; - -constexpr int IndexedRowTypicalGroupCount = 1; - -class TIndexedVersionedBlockFormatDetail; - -DECLARE_REFCOUNTED_STRUCT(IChunkIndexReadController) - -DECLARE_REFCOUNTED_STRUCT(TTabletPerformanceCounters) - -DECLARE_REFCOUNTED_STRUCT(IGranuleFilter) - -struct TVersionedRowDigest; - -DECLARE_REFCOUNTED_STRUCT(IDictionaryCompressionSession) -DECLARE_REFCOUNTED_STRUCT(IDictionaryDecompressionSession) -DECLARE_REFCOUNTED_STRUCT(IDictionaryCompressionFactory) - -struct TTimestampReadOptions; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NTableClient