Merged PR 730058: Remove UsePhysicalSizeInQuotaKeeper

Remove UsePhysicalSizeInQuotaKeeper Related work items: #2086569
microsoft · Aug 1, 2023 · fdee73b · fdee73b
1 parent 2667877
commit fdee73b
Show file tree

Hide file tree

Showing 10 changed files with 16 additions and 26 deletions.
diff --git a/Public/Src/Cache/ContentStore/Distributed/NuCache/CachingCentralStorage.cs b/Public/Src/Cache/ContentStore/Distributed/NuCache/CachingCentralStorage.cs
@@ -73,7 +73,6 @@ public CachingCentralStorage(
                     {
                         TraceFileSystemContentStoreDiagnosticMessages = Configuration.TraceFileSystemContentStoreDiagnosticMessages,
                         SelfCheckSettings = Configuration.SelfCheckSettings,
-                        UsePhysicalSizeInQuotaKeeper = Configuration.UsePhysicalSizeInQuotaKeeper,
                     });
             }
 

diff --git a/Public/Src/Cache/ContentStore/Distributed/NuCache/LocalLocationStoreConfiguration.cs b/Public/Src/Cache/ContentStore/Distributed/NuCache/LocalLocationStoreConfiguration.cs
@@ -392,11 +392,6 @@ public record DistributedCentralStoreConfiguration(AbsolutePath CacheRoot)
         /// </summary>
         public bool InlineCheckpointProactiveCopies { get; set; } = false;
 
-        /// <summary>
-        /// Whether the physical size is used for quota purposes.
-        /// </summary>
-        public bool UsePhysicalSizeInQuotaKeeper { get; set; } = false;
-
         /// <summary>
         ///  Whether to use the primary CAS in the DistributedCentralStorage.
         /// </summary>

diff --git a/Public/Src/Cache/ContentStore/DistributedTest/Stores/GrpcCopyContentTests.cs b/Public/Src/Cache/ContentStore/DistributedTest/Stores/GrpcCopyContentTests.cs
@@ -419,7 +419,7 @@ private async Task RunTestCase(Func<LocalContentServer, AbsolutePath, IContentSe
                                         CopyRequestHandlingCountLimit = _copyToLimit, ProactivePushCountLimit = _proactivePushCountLimit
                                     };
 
-                var storeConfig = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
+                var storeConfig = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(7);
                 Func<AbsolutePath, IContentStore> contentStoreFactory = (path) =>
                     new FileSystemContentStore(
                         FileSystem,

diff --git a/Public/Src/Cache/ContentStore/GrpcTest/GrpcDeleteContentTests.cs b/Public/Src/Cache/ContentStore/GrpcTest/GrpcDeleteContentTests.cs
@@ -76,12 +76,14 @@ public Task SendReceiveDeletion(long size)
                 var placeResult = await rpcClient.PlaceFileAsync(context, putResult.ContentHash, new AbsolutePath(fileName.Path + "place"), FileAccessMode.None, FileReplacementMode.None, FileRealizationMode.Copy);
                 placeResult.ShouldBeSuccess();
 
+                var clusterSize = FileSystem.GetClusterSize(fileName);
+
                 // Delete content
                 var deleteResult = await rpcClient.DeleteContentAsync(context, putResult.ContentHash, deleteLocalOnly: false);
                 deleteResult.ShouldBeSuccess();
                 deleteResult.ContentHash.Equals(putResult.ContentHash).Should().BeTrue();
                 string.IsNullOrEmpty(deleteResult.ErrorMessage).Should().BeTrue();
-                deleteResult.ContentSize.Should().Be(size);
+                deleteResult.ContentSize.Should().Be(ContentFileInfo.GetPhysicalSize(size,clusterSize));
 
                 // Fail to place content
                 var failPlaceResult = await rpcClient.PlaceFileAsync(context, putResult.ContentHash, new AbsolutePath(fileName.Path + "fail"), FileAccessMode.None, FileReplacementMode.None, FileRealizationMode.Copy);

diff --git a/Public/Src/Cache/ContentStore/Library/Stores/ContentStoreSettings.cs b/Public/Src/Cache/ContentStore/Library/Stores/ContentStoreSettings.cs
@@ -31,11 +31,6 @@ public sealed class ContentStoreSettings
         /// </summary>
         public bool UseEmptyContentShortcut { get; set; } = true;
 
-        /// <summary>
-        /// Whether the physical size is used for quota purposes.
-        /// </summary>
-        public bool UsePhysicalSizeInQuotaKeeper { get; set; } = false;
-
         /// <summary>
         /// A timeout for space reservation operation.
         /// </summary>

diff --git a/Public/Src/Cache/ContentStore/Library/Stores/FileSystemContentStoreInternal.cs b/Public/Src/Cache/ContentStore/Library/Stores/FileSystemContentStoreInternal.cs
@@ -203,8 +203,7 @@ public FileSystemContentStoreInternal(
 
             _settings = settings ?? ContentStoreSettings.DefaultSettings;
 
-            // Use logical size in QuotaKeeper is equivalent to use clusterSize = 1
-            _clusterSize = _settings.UsePhysicalSizeInQuotaKeeper ? FileSystem.GetClusterSize(rootPath) : 1;
+            _clusterSize = FileSystem.GetClusterSize(rootPath);
 
             // MemoryContentDirectory requires for the root path to exist. Making sure this is the case.
             FileSystem.CreateDirectory(RootPath);

diff --git a/Public/Src/Cache/ContentStore/Test/Sessions/ServiceClientContentSessionTestBase.cs b/Public/Src/Cache/ContentStore/Test/Sessions/ServiceClientContentSessionTestBase.cs
@@ -34,7 +34,7 @@ protected ServiceClientContentStoreConfiguration CreateConfiguration()
 
         protected const int ContentByteCount = 100;
         protected const HashType ContentHashType = HashType.Vso0;
-        private const long DefaultMaxSize = 1 * 1024 * 1024;
+        protected const long DefaultMaxSize = 1 * 1024 * 1024;
         protected static readonly CancellationToken Token = CancellationToken.None;
         protected string Scenario;
 
@@ -65,14 +65,14 @@ protected async Task RunSessionTestAsync(
             }
         }
 
-        protected async Task RunStoreTestAsync(Func<Context, IContentStore, Task> funcAsync, LocalServerConfiguration localContentServerConfiguration = null, TimeSpan? heartbeatOverride = null)
+        protected async Task RunStoreTestAsync(Func<Context, IContentStore, Task> funcAsync, LocalServerConfiguration localContentServerConfiguration = null, TimeSpan? heartbeatOverride = null, long cacheSize = DefaultMaxSize)
         {
             var context = new Context(Logger);
             // Using unique scenario to avoid flakiness when running the tests in parallel
             Scenario += Guid.NewGuid();
             using (var directory = new DisposableDirectory(FileSystem))
             {
-                var config = new ContentStoreConfiguration(new MaxSizeQuota($"{DefaultMaxSize}"));
+                var config = new ContentStoreConfiguration(new MaxSizeQuota($"{cacheSize}"));
 
                 using (var store = CreateStore(directory.Path, config, localContentServerConfiguration ?? TestConfigurationHelper.LocalContentServerConfiguration, heartbeatOverride))
                 {
@@ -92,11 +92,13 @@ protected async Task RunStoreTestAsync(Func<Context, IContentStore, Task> funcAs
         protected Task RunSessionTestAsync(
             ImplicitPin implicitPin,
             Func<Context, IContentSession, Task> funcAsync,
-            LocalServerConfiguration localContentServerConfiguration = null)
+            LocalServerConfiguration localContentServerConfiguration = null,
+            long cacheSize = DefaultMaxSize)
         {
             return RunStoreTestAsync(
                 (context, store) => RunSessionTestAsync(context, store, implicitPin, funcAsync),
-                localContentServerConfiguration);
+                localContentServerConfiguration,
+                cacheSize: cacheSize);
         }
 
         protected abstract T CreateStore(AbsolutePath rootPath, ContentStoreConfiguration configuration, LocalServerConfiguration localContentServerConfiguration, TimeSpan? heartbeatOverride);

diff --git a/Public/Src/Cache/ContentStore/Test/Sessions/ServiceRequestsWorkAcrossServerRestartTests.cs b/Public/Src/Cache/ContentStore/Test/Sessions/ServiceRequestsWorkAcrossServerRestartTests.cs
@@ -191,6 +191,8 @@ private Task RunManyForStoreAcrossServerRestartAsync(Func<Context, IContentStore
 
         private Task RunManyForSessionAcrossServerRestartAsync(Func<Context, IContentSession, ContentHash, int, Task> requestFunc)
         {
+            // Set the physical size manually because getting it dinamically for each test requires a lot of changes
+            long cacheSize = DefaultMaxSize * 8;
             // Scenario must be unique for different test cases to avoid getting CacheException like:
             // BuildXL.Cache.ContentStore.Exceptions.CacheException : Shutdown event name=[InProcessServiceRequestsWorkAcrossServerRestartTestsDEBUGDEBUG] already exists
             Scenario += Guid.NewGuid().ToString();
@@ -217,7 +219,8 @@ private Task RunManyForSessionAcrossServerRestartAsync(Func<Context, IContentSes
                     string failureMessage = string.Join(",", singleException.InnerExceptions.Select(x => x.Message));
                     Assert.True(false, failureMessage);
                 }
-            });
+            },
+            cacheSize: cacheSize);
         }
 
         private class MockLogger : IOperationLogger

diff --git a/Public/Src/Cache/DistributedCache.Host/Configuration/DistributedContentSettings.cs b/Public/Src/Cache/DistributedCache.Host/Configuration/DistributedContentSettings.cs
@@ -785,9 +785,6 @@ public bool UseGrpcDotNetForCopies()
         [DataMember]
         public bool UseRedundantPutFileShortcut { get; set; } = false;
 
-        [DataMember]
-        public bool UsePhysicalSizeInQuotaKeeper { get; set; } = false;
-
         [DataMember]
         public bool UsePrimaryCasInDcs { get; set; } = false;
 

diff --git a/Public/Src/Cache/DistributedCache.Host/Service/Internal/DistributedContentStoreFactory.cs b/Public/Src/Cache/DistributedCache.Host/Service/Internal/DistributedContentStoreFactory.cs
@@ -477,7 +477,6 @@ private static ContentStoreSettings FromDistributedSettings(DistributedContentSe
                 SelfCheckSettings = CreateSelfCheckSettings(settings),
                 OverrideUnixFileAccessMode = settings.OverrideUnixFileAccessMode,
                 UseRedundantPutFileShortcut = settings.UseRedundantPutFileShortcut,
-                UsePhysicalSizeInQuotaKeeper = settings.UsePhysicalSizeInQuotaKeeper,
                 TraceFileSystemContentStoreDiagnosticMessages = settings.TraceFileSystemContentStoreDiagnosticMessages,
 
                 SkipTouchAndLockAcquisitionWhenPinningFromHibernation = settings.UseFastHibernationPin,
@@ -614,7 +613,6 @@ private void ApplySecretSettingsForLls(
                     MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies,
                     ProactiveCopyCheckpointFiles = _distributedSettings.ProactiveCopyCheckpointFiles,
                     InlineCheckpointProactiveCopies = _distributedSettings.InlineCheckpointProactiveCopies,
-                    UsePhysicalSizeInQuotaKeeper = _distributedSettings.UsePhysicalSizeInQuotaKeeper,
                     UsePrimaryCasInDcs = _distributedSettings.UsePrimaryCasInDcs,
                 };