From 8847bfab30b4220faa139b8f0c3244538888172d Mon Sep 17 00:00:00 2001 From: David Jimenez Date: Thu, 29 Aug 2024 10:16:55 -0600 Subject: [PATCH 1/8] SPIKE: Find approach for reverse paging --- .../Properties/launchSettings.json | 8 ++++++ .../apiPublisherSettings.json | 3 ++- src/EdFi.Tools.ApiPublisher.Cli/logging.json | 2 +- ...PagingStreamResourcePageMessageProducer.cs | 27 ++++++++++++++++--- 4 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json diff --git a/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json b/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json new file mode 100644 index 0000000..b6eddd5 --- /dev/null +++ b/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "EdFi.Tools.ApiPublisher.Cli": { + "commandName": "Project", + "commandLineArgs": ".\\EdFiApiPublisher.exe --sourceName=AzureDavid --targetName=dockerDavid --sourceUrl=https://api.ed-fi.org/v7.1/api/ --sourceKey=RvcohKz9zHI4 --sourceSecret=E1iEFusaNf81xzCxwHfbolkC --targetUrl=https://localhost/api/ --targetKey=DbXHLFvuRgSK --targetSecret=KpMOlOoD9FzdKiIM2qrMnfBK --ignoreIsolation=true --maxDegreeOfParallelismForPostResourceItem=1 --maxDegreeOfParallelismForStreamResourcePages=1 --includeDescriptors=true --ignoreSslErrors=true --includeOnly=/ed-fi/people --useChangeVersionPaging=true" + } + } +} \ No newline at end of file diff --git a/src/EdFi.Tools.ApiPublisher.Cli/apiPublisherSettings.json b/src/EdFi.Tools.ApiPublisher.Cli/apiPublisherSettings.json index bf02363..a421266 100644 --- a/src/EdFi.Tools.ApiPublisher.Cli/apiPublisherSettings.json +++ b/src/EdFi.Tools.ApiPublisher.Cli/apiPublisherSettings.json @@ -14,7 +14,8 @@ "changeVersionPagingWindowSize": 25000, "enableRateLimit": false, "rateLimitNumberExecutions": 100, - "rateLimitTimeLimitMinutes": 1 + "rateLimitTimeLimitMinutes": 1, + "useReversePaging": false }, "authorizationFailureHandling": [ { diff --git a/src/EdFi.Tools.ApiPublisher.Cli/logging.json b/src/EdFi.Tools.ApiPublisher.Cli/logging.json index fabb9da..1c1f62e 100644 --- a/src/EdFi.Tools.ApiPublisher.Cli/logging.json +++ b/src/EdFi.Tools.ApiPublisher.Cli/logging.json @@ -1,7 +1,7 @@ { "Serilog": { "MinimumLevel": { - "Default": "Information" + "Default": "Debug" }, "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ], "WriteTo:Console": { diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs index fb98a7f..f585508 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs @@ -97,8 +97,19 @@ public async Task>> P continue; } - int offsetOnWindow = 0; - while (offsetOnWindow < totalCountOnWindow) + //int offsetOnWindow = 0; + + /// APIPUB-68 + bool isLastOne = false; + long offsetOnWindow = totalCountOnWindow - limit; + if (offsetOnWindow < 0) + { + offsetOnWindow = 0; + isLastOne = true; + } + + int limitOnWindow = totalCountOnWindow < limit ? (int)totalCountOnWindow : limit; + while ((offsetOnWindow > 0 || isLastOne == true) && totalCountOnWindow > 0) { var pageMessage = new StreamResourcePageMessage { @@ -107,7 +118,7 @@ public async Task>> P PostAuthorizationFailureRetry = message.PostAuthorizationFailureRetry, // Page-strategy specific context - Limit = limit, + Limit = limitOnWindow, Offset = offsetOnWindow, // Global processing context @@ -118,7 +129,15 @@ public async Task>> P }; pageMessages.Add(pageMessage); - offsetOnWindow += limit; + offsetOnWindow -= limit; + if (isLastOne) + break; + if (offsetOnWindow < 0) + { + limitOnWindow = limit + (int)offsetOnWindow; + offsetOnWindow = 0; + isLastOne = true; + } } changeVersionWindow++; From 4502c942a4fbad29d840bd813eb251ac1c9f44ca Mon Sep 17 00:00:00 2001 From: David Jimenez Date: Thu, 29 Aug 2024 14:20:16 -0600 Subject: [PATCH 2/8] fixes --- .../Properties/launchSettings.json | 2 +- ...EdFiApiStreamResourcePageMessageHandler.cs | 40 +++++++++++++++---- .../Configuration/ApiPublisherSettings.cs | 2 + 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json b/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json index b6eddd5..ba3b12e 100644 --- a/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json +++ b/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "EdFi.Tools.ApiPublisher.Cli": { "commandName": "Project", - "commandLineArgs": ".\\EdFiApiPublisher.exe --sourceName=AzureDavid --targetName=dockerDavid --sourceUrl=https://api.ed-fi.org/v7.1/api/ --sourceKey=RvcohKz9zHI4 --sourceSecret=E1iEFusaNf81xzCxwHfbolkC --targetUrl=https://localhost/api/ --targetKey=DbXHLFvuRgSK --targetSecret=KpMOlOoD9FzdKiIM2qrMnfBK --ignoreIsolation=true --maxDegreeOfParallelismForPostResourceItem=1 --maxDegreeOfParallelismForStreamResourcePages=1 --includeDescriptors=true --ignoreSslErrors=true --includeOnly=/ed-fi/people --useChangeVersionPaging=true" + "commandLineArgs": ".\\EdFiApiPublisher.exe --sourceName=AzureDavid --targetName=dockerDavid --sourceUrl=https://tools-apipub01.southcentralus.cloudapp.azure.com/WebApi/ --sourceKey=rVzVM8ibVA4v --sourceSecret=hARh1t7SuP3WValGrx68s0YE --targetUrl=https://localhost/api/ --targetKey=Wa9SzHlESvI9 --targetSecret=hWrndZ8qBxZy6XLoUi0o8TcY --ignoreIsolation=true --maxDegreeOfParallelismForPostResourceItem=1 --maxDegreeOfParallelismForStreamResourcePages=1 --includeDescriptors=true --ignoreSslErrors=true --useChangeVersionPaging=true --exclude=/ed-fi/studentParentAssociations" } } } \ No newline at end of file diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs index 7e4ef1c..42f7680 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs @@ -177,18 +177,25 @@ public async Task> HandleStreamResourcePageAsyn break; } - // Perform limit/offset final page check (for need for possible continuation) - if (message.IsFinalPage && JArray.Parse(responseContent).Count == limit) + if (!options.useReversePaging) { - if (_logger.IsEnabled(LogEventLevel.Debug)) + // Perform limit/offset final page check (for need for possible continuation) + if (message.IsFinalPage && JArray.Parse(responseContent).Count == limit) { - _logger.Debug($"{message.ResourceUrl}: Final page was full. Attempting to retrieve more data."); - } + if (_logger.IsEnabled(LogEventLevel.Debug)) + { + _logger.Debug($"{message.ResourceUrl}: Final page was full. Attempting to retrieve more data."); + } - // Looks like there could be more data - offset += limit; + // Looks like there could be more data + offset += limit; - continue; + continue; + } + } + else + { + break; } } @@ -205,7 +212,24 @@ public async Task> HandleStreamResourcePageAsyn _logger.Warning($"{message.ResourceUrl}: Rate limit exceeded. Please try again later."); } } + break; + + //// Perform limit/offset final page check (for need for possible continuation) + //if (message.IsFinalPage && JArray.Parse(responseContent).Count == limit) + //{ + // if (_logger.IsEnabled(LogEventLevel.Debug)) + // { + // _logger.Debug($"{message.ResourceUrl}: Final page was full. Attempting to retrieve more data."); + // } + + // // Looks like there could be more data + // offset += limit; + + // continue; + //} + + //break; } while (true); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs index 19613b0..8f2ab91 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs @@ -94,5 +94,7 @@ public int MaxDegreeOfParallelismForPostResourceItem public int RateLimitNumberExecutions { get; set; } = 100; public int RateLimitTimeLimitMinutes { get; set; } = 1; + + public bool useReversePaging { get; set; } = false; } } From 18b8457da82e5f97f1653a441aadebcbe89bb896 Mon Sep 17 00:00:00 2001 From: David Jimenez Date: Fri, 30 Aug 2024 08:36:17 -0600 Subject: [PATCH 3/8] Code organization --- .../Properties/launchSettings.json | 3 +- .../Modules/EdFiApiAsSourceModule.cs | 11 +- ...EdFiApiStreamResourcePageMessageHandler.cs | 18 --- ...PagingStreamResourcePageMessageProducer.cs | 35 +--- ...PagingStreamResourcePageMessageProducer.cs | 153 ++++++++++++++++++ .../ConfigurationBuilderFactory.cs | 1 + 6 files changed, 173 insertions(+), 48 deletions(-) create mode 100644 src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs diff --git a/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json b/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json index ba3b12e..7e9124a 100644 --- a/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json +++ b/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json @@ -1,8 +1,7 @@ { "profiles": { "EdFi.Tools.ApiPublisher.Cli": { - "commandName": "Project", - "commandLineArgs": ".\\EdFiApiPublisher.exe --sourceName=AzureDavid --targetName=dockerDavid --sourceUrl=https://tools-apipub01.southcentralus.cloudapp.azure.com/WebApi/ --sourceKey=rVzVM8ibVA4v --sourceSecret=hARh1t7SuP3WValGrx68s0YE --targetUrl=https://localhost/api/ --targetKey=Wa9SzHlESvI9 --targetSecret=hWrndZ8qBxZy6XLoUi0o8TcY --ignoreIsolation=true --maxDegreeOfParallelismForPostResourceItem=1 --maxDegreeOfParallelismForStreamResourcePages=1 --includeDescriptors=true --ignoreSslErrors=true --useChangeVersionPaging=true --exclude=/ed-fi/studentParentAssociations" + "commandName": "Project" } } } \ No newline at end of file diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs index 869a6f6..365b678 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs @@ -85,9 +85,18 @@ protected override void Load(ContainerBuilder builder) // Register resource page message producer using a ChangeVersion paging strategy if (options.UseChangeVersionPaging) { - builder.RegisterType() + if (options.useReversePaging) + { + builder.RegisterType() .As() .SingleInstance(); + } + else + { + builder.RegisterType() + .As() + .SingleInstance(); + } } // Register resource page message producer using a limit/offset paging strategy else diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs index 42f7680..176e6d5 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs @@ -212,24 +212,6 @@ public async Task> HandleStreamResourcePageAsyn _logger.Warning($"{message.ResourceUrl}: Rate limit exceeded. Please try again later."); } } - - break; - - //// Perform limit/offset final page check (for need for possible continuation) - //if (message.IsFinalPage && JArray.Parse(responseContent).Count == limit) - //{ - // if (_logger.IsEnabled(LogEventLevel.Debug)) - // { - // _logger.Debug($"{message.ResourceUrl}: Final page was full. Attempting to retrieve more data."); - // } - - // // Looks like there could be more data - // offset += limit; - - // continue; - //} - - //break; } while (true); diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs index f585508..836a54d 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs @@ -36,7 +36,7 @@ public async Task>> P $"{message.ResourceUrl}: Retrieving total count of items in change versions {message.ChangeWindow.MinChangeVersion} to {message.ChangeWindow.MaxChangeVersion}."); } else - { + { _logger.Information($"{message.ResourceUrl}: Retrieving total count of items."); } @@ -59,7 +59,7 @@ public async Task>> P int limit = message.PageSize; var pageMessages = new List>(); - + if (totalCount > 0) { var noOfPartitions = Math.Ceiling((decimal)(message.ChangeWindow.MaxChangeVersion - message.ChangeWindow.MinChangeVersion) @@ -72,7 +72,7 @@ public async Task>> P { long changeVersionWindowEndValue = (changeVersionWindowStartValue > 0 ? changeVersionWindowStartValue - 1 : changeVersionWindowStartValue) + options.ChangeVersionPagingWindowSize; - + if (changeVersionWindowEndValue > message.ChangeWindow.MaxChangeVersion) { changeVersionWindowEndValue = message.ChangeWindow.MaxChangeVersion; @@ -97,19 +97,8 @@ public async Task>> P continue; } - //int offsetOnWindow = 0; - - /// APIPUB-68 - bool isLastOne = false; - long offsetOnWindow = totalCountOnWindow - limit; - if (offsetOnWindow < 0) - { - offsetOnWindow = 0; - isLastOne = true; - } - - int limitOnWindow = totalCountOnWindow < limit ? (int)totalCountOnWindow : limit; - while ((offsetOnWindow > 0 || isLastOne == true) && totalCountOnWindow > 0) + int offsetOnWindow = 0; + while (offsetOnWindow < totalCountOnWindow) { var pageMessage = new StreamResourcePageMessage { @@ -118,7 +107,7 @@ public async Task>> P PostAuthorizationFailureRetry = message.PostAuthorizationFailureRetry, // Page-strategy specific context - Limit = limitOnWindow, + Limit = limit, Offset = offsetOnWindow, // Global processing context @@ -129,15 +118,7 @@ public async Task>> P }; pageMessages.Add(pageMessage); - offsetOnWindow -= limit; - if (isLastOne) - break; - if (offsetOnWindow < 0) - { - limitOnWindow = limit + (int)offsetOnWindow; - offsetOnWindow = 0; - isLastOne = true; - } + offsetOnWindow += limit; } changeVersionWindow++; @@ -153,4 +134,4 @@ public async Task>> P return pageMessages; } -} +} \ No newline at end of file diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs new file mode 100644 index 0000000..10be3b9 --- /dev/null +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +using EdFi.Tools.ApiPublisher.Core.Configuration; +using EdFi.Tools.ApiPublisher.Core.Counting; +using EdFi.Tools.ApiPublisher.Core.Processing; +using EdFi.Tools.ApiPublisher.Core.Processing.Handlers; +using EdFi.Tools.ApiPublisher.Core.Processing.Messages; +using Serilog; +using System.Threading.Tasks.Dataflow; + +namespace EdFi.Tools.ApiPublisher.Connections.Api.Processing.Source.MessageProducers; + +public class EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer : IStreamResourcePageMessageProducer +{ + private readonly ISourceTotalCountProvider _sourceTotalCountProvider; + private readonly ILogger _logger = Log.ForContext(typeof(EdFiApiLimitOffsetPagingStreamResourcePageMessageProducer)); + + public EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer(ISourceTotalCountProvider sourceTotalCountProvider) + { + _sourceTotalCountProvider = sourceTotalCountProvider; + } + + public async Task>> ProduceMessagesAsync( + StreamResourceMessage message, + Options options, + ITargetBlock errorHandlingBlock, + Func, string, IEnumerable> createProcessDataMessages, + CancellationToken cancellationToken) + { + if (message.ChangeWindow?.MaxChangeVersion != default(long) && message.ChangeWindow?.MaxChangeVersion != null) + { + _logger.Information( + $"{message.ResourceUrl}: Retrieving total count of items in change versions {message.ChangeWindow.MinChangeVersion} to {message.ChangeWindow.MaxChangeVersion}."); + } + else + { + _logger.Information($"{message.ResourceUrl}: Retrieving total count of items."); + } + + // Get total count of items in source resource for change window (if applicable) + var (totalCountSuccess, totalCount) = await _sourceTotalCountProvider.TryGetTotalCountAsync( + message.ResourceUrl, + options, + message.ChangeWindow, + errorHandlingBlock, + cancellationToken); + + if (!totalCountSuccess) + { + // Allow processing to continue without performing additional work on this resource. + return Enumerable.Empty>(); + } + + _logger.Information($"{message.ResourceUrl}: Total count = {totalCount}"); + + int limit = message.PageSize; + + var pageMessages = new List>(); + + if (totalCount > 0) + { + var noOfPartitions = Math.Ceiling((decimal)(message.ChangeWindow.MaxChangeVersion - message.ChangeWindow.MinChangeVersion) + / options.ChangeVersionPagingWindowSize); + + int changeVersionWindow = 0; + long changeVersionWindowStartValue = message.ChangeWindow.MinChangeVersion; + + while (changeVersionWindow < noOfPartitions) + { + long changeVersionWindowEndValue = (changeVersionWindowStartValue > 0 ? + changeVersionWindowStartValue - 1 : changeVersionWindowStartValue) + options.ChangeVersionPagingWindowSize; + + if (changeVersionWindowEndValue > message.ChangeWindow.MaxChangeVersion) + { + changeVersionWindowEndValue = message.ChangeWindow.MaxChangeVersion; + } + var changeWindow = new ChangeWindow + { + MinChangeVersion = changeVersionWindowStartValue, + MaxChangeVersion = changeVersionWindowEndValue + }; + changeVersionWindowStartValue = changeVersionWindowEndValue + 1; + + // Get total count of items in source resource for change window (if applicable) + var (totalCountOnWindowSuccess, totalCountOnWindow) = await _sourceTotalCountProvider.TryGetTotalCountAsync( + message.ResourceUrl, + options, + changeWindow, + errorHandlingBlock, + cancellationToken); + + if (!totalCountOnWindowSuccess) + { + continue; + } + + bool isLastOne = false; + long offsetOnWindow = totalCountOnWindow - limit; + if (offsetOnWindow < 0) + { + offsetOnWindow = 0; + isLastOne = true; + } + + int limitOnWindow = totalCountOnWindow < limit ? (int)totalCountOnWindow : limit; + while ((offsetOnWindow > 0 || isLastOne == true) && totalCountOnWindow > 0) + { + var pageMessage = new StreamResourcePageMessage + { + // Resource-specific context + ResourceUrl = message.ResourceUrl, + PostAuthorizationFailureRetry = message.PostAuthorizationFailureRetry, + + // Page-strategy specific context + Limit = limitOnWindow, + Offset = offsetOnWindow, + + // Global processing context + ChangeWindow = changeWindow, + CreateProcessDataMessages = createProcessDataMessages, + + CancellationSource = message.CancellationSource, + }; + + pageMessages.Add(pageMessage); + offsetOnWindow -= limit; + if (isLastOne) + break; + if (offsetOnWindow < 0) + { + limitOnWindow = limit + (int)offsetOnWindow; + offsetOnWindow = 0; + isLastOne = true; + } + } + changeVersionWindow++; + + } + } + + // Flag the last page for special "continuation" processing + if (pageMessages.Any()) + { + // Page-strategy specific context + pageMessages.Last().IsFinalPage = true; + } + + return pageMessages; + } +} diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationBuilderFactory.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationBuilderFactory.cs index 285fa85..37852b9 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationBuilderFactory.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationBuilderFactory.cs @@ -75,6 +75,7 @@ public IConfigurationBuilder Create(string[] commandLineArgs) ["--enableRateLimit"] = "Options:EnableRateLimit", ["--rateLimitNumberExecutions"] = "Options:RateLimitNumberExecutions", ["--rateLimitTimeLimitMinutes"] = "Options:RateLimitTimeLimitMinutes", + ["--useReversePaging"] = "Options:UseReversePaging", // Resource selection (comma delimited paths - e.g. "/ed-fi/students,/ed-fi/studentSchoolAssociations") ["--include"] = "Connections:Source:Include", From 06dd9bbbe5695519c18bab836a35e0cb2a08208e Mon Sep 17 00:00:00 2001 From: David Jimenez Date: Fri, 30 Aug 2024 08:39:06 -0600 Subject: [PATCH 4/8] Code cleanup --- .../Properties/launchSettings.json | 7 ------- src/EdFi.Tools.ApiPublisher.Cli/logging.json | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json b/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json index 7e9124a..e69de29 100644 --- a/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json +++ b/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json @@ -1,7 +0,0 @@ -{ - "profiles": { - "EdFi.Tools.ApiPublisher.Cli": { - "commandName": "Project" - } - } -} \ No newline at end of file diff --git a/src/EdFi.Tools.ApiPublisher.Cli/logging.json b/src/EdFi.Tools.ApiPublisher.Cli/logging.json index 1c1f62e..fabb9da 100644 --- a/src/EdFi.Tools.ApiPublisher.Cli/logging.json +++ b/src/EdFi.Tools.ApiPublisher.Cli/logging.json @@ -1,7 +1,7 @@ { "Serilog": { "MinimumLevel": { - "Default": "Debug" + "Default": "Information" }, "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ], "WriteTo:Console": { From c322855c37bacad7f637c6ae03d0cec60cabe0a9 Mon Sep 17 00:00:00 2001 From: David Jimenez Date: Mon, 2 Sep 2024 15:38:40 -0600 Subject: [PATCH 5/8] Trying to implement tests --- ...PagingStreamResourcePageMessageProducer.cs | 2 +- .../Helpers/TestHelpers.cs | 11 +- .../Processing/ReversePagingTests.cs | 224 ++++++++++++++++++ 3 files changed, 233 insertions(+), 4 deletions(-) create mode 100644 src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs index 10be3b9..e90c518 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs @@ -106,7 +106,7 @@ public async Task>> P } int limitOnWindow = totalCountOnWindow < limit ? (int)totalCountOnWindow : limit; - while ((offsetOnWindow > 0 || isLastOne == true) && totalCountOnWindow > 0) + while ((offsetOnWindow >= 0 || isLastOne == true) && totalCountOnWindow > 0 && limitOnWindow > 0) { var pageMessage = new StreamResourcePageMessage { diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Helpers/TestHelpers.cs b/src/EdFi.Tools.ApiPublisher.Tests/Helpers/TestHelpers.cs index b564f27..010b5cc 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Helpers/TestHelpers.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Helpers/TestHelpers.cs @@ -260,7 +260,8 @@ public static ChangeProcessor CreateChangeProcessorWithDefaultDependencies( IFakeHttpRequestHandler fakeSourceRequestHandler, ApiConnectionDetails targetApiConnectionDetails, IFakeHttpRequestHandler fakeTargetRequestHandler, - INodeJSService nodeJsService = null) + INodeJSService nodeJsService = null, + bool withReversePaging = false) { EdFiApiClient SourceApiClientFactory() => new EdFiApiClient( @@ -300,8 +301,12 @@ EdFiApiClient TargetApiClientFactory() => var streamingResourceProcessor = new StreamingResourceProcessor( new StreamResourceBlockFactory( - new EdFiApiLimitOffsetPagingStreamResourcePageMessageProducer( - new EdFiApiSourceTotalCountProvider(sourceEdFiApiClientProvider))), + (withReversePaging) ? + new EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer( + new EdFiApiSourceTotalCountProvider(sourceEdFiApiClientProvider)) : + new EdFiApiLimitOffsetPagingStreamResourcePageMessageProducer( + new EdFiApiSourceTotalCountProvider(sourceEdFiApiClientProvider)) + ), new StreamResourcePagesBlockFactory(new EdFiApiStreamResourcePageMessageHandler(sourceEdFiApiClientProvider)), sourceApiConnectionDetails); diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs new file mode 100644 index 0000000..6e60e40 --- /dev/null +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +using EdFi.Tools.ApiPublisher.Connections.Api.ApiClientManagement; +using EdFi.Tools.ApiPublisher.Connections.Api.Processing.Source.Counting; +using EdFi.Tools.ApiPublisher.Connections.Api.Processing.Source.MessageProducers; +using EdFi.Tools.ApiPublisher.Core.Configuration; +using EdFi.Tools.ApiPublisher.Core.Processing; +using EdFi.Tools.ApiPublisher.Core.Processing.Messages; +using EdFi.Tools.ApiPublisher.Tests.Helpers; +using NUnit.Framework; +using Shouldly; +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace EdFi.Tools.ApiPublisher.Tests.Processing +{ + [TestFixture] + public class ReversePagingTests + { + public class When_producing_messages_with_reverse_paging + { + [TestFixture] + public class Given_a_MaxChangeVersion_equal_to_100_and_ChangeVersion_page_size_equal_40_and_pageSize_equal_to_40 : TestFixtureAsyncBase + { + private ChangeProcessor _changeProcessor; + private IFakeHttpRequestHandler _fakeTargetRequestHandler; + private IFakeHttpRequestHandler _fakeSourceRequestHandler; + private ChangeProcessorConfiguration _changeProcessorConfiguration; + private EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer _messageProducer; + private Options _options; + + protected override async Task ArrangeAsync() + { + // ----------------------------------------------------------------- + // Source Requests + // ----------------------------------------------------------------- + var sourceResourceFaker = TestHelpers.GetGenericResourceFaker(); + + var suppliedSourceResources = sourceResourceFaker.Generate(100); + + // Prepare the fake source API endpoint + _fakeSourceRequestHandler = TestHelpers.GetFakeBaselineSourceApiRequestHandler() + + // Test-specific mocks + .AvailableChangeVersions(1100) + .ResourceCount(responseTotalCountHeader: 100) + .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}", suppliedSourceResources) + .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}/deletes", Array.Empty()); + + var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( + includeOnly: new[] { "schools" }); + + _options = TestHelpers.GetOptions(); + _options.IncludeDescriptors = false; // Shorten test execution time + _options.UseChangeVersionPaging = true; + _options.useReversePaging = true; + _options.StreamingPageSize = 40; + _options.ChangeVersionPagingWindowSize = 40; + + await Task.Yield(); + + EdFiApiClient SourceApiClientFactory() => + new EdFiApiClient( + "TestSource", + sourceApiConnectionDetails, + bearerTokenRefreshMinutes: 27, + ignoreSslErrors: true, + httpClientHandler: new HttpClientHandlerFakeBridge(_fakeSourceRequestHandler)); + + + var sourceEdFiApiClientProvider = new EdFiApiClientProvider(new Lazy(SourceApiClientFactory)); + + _messageProducer = new EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer( + new EdFiApiSourceTotalCountProvider(sourceEdFiApiClientProvider)); + } + + [TestCase("schools")] + public async Task Should_produce_messages_correctly(string resourceCollectionName) + { + StreamResourceMessage msg = new StreamResourceMessage + { + ChangeWindow = new ChangeWindow() + { + MaxChangeVersion = 100, + MinChangeVersion = 0 + + }, + ResourceUrl = $"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}", + PageSize = 40 + }; + + var pageMessages = await _messageProducer.ProduceMessagesAsync(msg, _options, null, null, new CancellationToken()); + + pageMessages.ElementAt(0).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 40 }); + pageMessages.ElementAt(3).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 41, MaxChangeVersion = 80 }); + pageMessages.ElementAt(6).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 81, MaxChangeVersion = 100 }); + + pageMessages.ElementAt(0).Offset.ShouldBe(60); + pageMessages.ElementAt(0).Limit.ShouldBe(40); + + pageMessages.ElementAt(1).Offset.ShouldBe(20); + pageMessages.ElementAt(1).Limit.ShouldBe(40); + + pageMessages.ElementAt(2).Offset.ShouldBe(0); + pageMessages.ElementAt(2).Limit.ShouldBe(20); + + pageMessages.ElementAt(3).Offset.ShouldBe(60); + pageMessages.ElementAt(3).Limit.ShouldBe(40); + + pageMessages.ElementAt(4).Offset.ShouldBe(20); + pageMessages.ElementAt(4).Limit.ShouldBe(40); + + pageMessages.ElementAt(5).Offset.ShouldBe(0); + pageMessages.ElementAt(5).Limit.ShouldBe(20); + + pageMessages.ElementAt(6).Offset.ShouldBe(60); + pageMessages.ElementAt(6).Limit.ShouldBe(40); + + pageMessages.ElementAt(7).Offset.ShouldBe(20); + pageMessages.ElementAt(7).Limit.ShouldBe(40); + + pageMessages.ElementAt(8).Offset.ShouldBe(0); + pageMessages.ElementAt(8).Limit.ShouldBe(20); + } + } + + + [TestFixture] + public class Given_a_MaxChangeVersion_equal_to_100_and_ChangeVersion_page_size_equal_40_and_pageSize_equal_to_50 : TestFixtureAsyncBase + { + private ChangeProcessor _changeProcessor; + private IFakeHttpRequestHandler _fakeTargetRequestHandler; + private IFakeHttpRequestHandler _fakeSourceRequestHandler; + private ChangeProcessorConfiguration _changeProcessorConfiguration; + private EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer _messageProducer; + private Options _options; + + protected override async Task ArrangeAsync() + { + // ----------------------------------------------------------------- + // Source Requests + // ----------------------------------------------------------------- + var sourceResourceFaker = TestHelpers.GetGenericResourceFaker(); + + var suppliedSourceResources = sourceResourceFaker.Generate(100); + + // Prepare the fake source API endpoint + _fakeSourceRequestHandler = TestHelpers.GetFakeBaselineSourceApiRequestHandler() + + // Test-specific mocks + .AvailableChangeVersions(1100) + .ResourceCount(responseTotalCountHeader: 100) + .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}", suppliedSourceResources) + .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}/deletes", Array.Empty()); + + var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( + includeOnly: new[] { "schools" }); + + _options = TestHelpers.GetOptions(); + _options.IncludeDescriptors = false; // Shorten test execution time + _options.UseChangeVersionPaging = true; + _options.useReversePaging = true; + _options.StreamingPageSize = 50; + _options.ChangeVersionPagingWindowSize = 50; + + await Task.Yield(); + + EdFiApiClient SourceApiClientFactory() => + new EdFiApiClient( + "TestSource", + sourceApiConnectionDetails, + bearerTokenRefreshMinutes: 27, + ignoreSslErrors: true, + httpClientHandler: new HttpClientHandlerFakeBridge(_fakeSourceRequestHandler)); + + + var sourceEdFiApiClientProvider = new EdFiApiClientProvider(new Lazy(SourceApiClientFactory)); + + _messageProducer = new EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer( + new EdFiApiSourceTotalCountProvider(sourceEdFiApiClientProvider)); + } + + [TestCase("schools")] + public async Task Should_produce_messages_correctly(string resourceCollectionName) + { + var pageSize = 50; + StreamResourceMessage msg = new StreamResourceMessage + { + ChangeWindow = new ChangeWindow() + { + MaxChangeVersion = 100, + MinChangeVersion = 0 + + }, + ResourceUrl = $"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}", + PageSize = pageSize + }; + + var pageMessages = await _messageProducer.ProduceMessagesAsync(msg, _options, null, null, new CancellationToken()); + + pageMessages.ElementAt(0).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 50 }); + pageMessages.ElementAt(2).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 51, MaxChangeVersion = 100 }); + + pageMessages.ElementAt(0).Offset.ShouldBe(50); + pageMessages.ElementAt(0).Limit.ShouldBe(pageSize); + + pageMessages.ElementAt(1).Offset.ShouldBe(0); + pageMessages.ElementAt(1).Limit.ShouldBe(pageSize); + + pageMessages.ElementAt(2).Offset.ShouldBe(50); + pageMessages.ElementAt(2).Limit.ShouldBe(pageSize); + + pageMessages.ElementAt(3).Offset.ShouldBe(0); + pageMessages.ElementAt(3).Limit.ShouldBe(pageSize); + } + } + } + } +} From 70584352965505099f5973fd1fef2c55785ea3e0 Mon Sep 17 00:00:00 2001 From: David Jimenez Date: Tue, 3 Sep 2024 13:46:59 -0600 Subject: [PATCH 6/8] More tests --- .../Processing/ReversePagingTests.cs | 200 ++++++++++++++---- 1 file changed, 164 insertions(+), 36 deletions(-) diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs index 6e60e40..fd8725e 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs @@ -22,17 +22,16 @@ namespace EdFi.Tools.ApiPublisher.Tests.Processing [TestFixture] public class ReversePagingTests { + [TestFixture] public class When_producing_messages_with_reverse_paging { [TestFixture] - public class Given_a_MaxChangeVersion_equal_to_100_and_ChangeVersion_page_size_equal_40_and_pageSize_equal_to_40 : TestFixtureAsyncBase + public class Given_100_records_and_ChangeVersion_page_size_equal_40_and_pageSize_equal_to_40 : TestFixtureAsyncBase { - private ChangeProcessor _changeProcessor; - private IFakeHttpRequestHandler _fakeTargetRequestHandler; private IFakeHttpRequestHandler _fakeSourceRequestHandler; - private ChangeProcessorConfiguration _changeProcessorConfiguration; private EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer _messageProducer; private Options _options; + private int _pageSize = 40; protected override async Task ArrangeAsync() { @@ -48,9 +47,7 @@ protected override async Task ArrangeAsync() // Test-specific mocks .AvailableChangeVersions(1100) - .ResourceCount(responseTotalCountHeader: 100) - .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}", suppliedSourceResources) - .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}/deletes", Array.Empty()); + .ResourceCount(responseTotalCountHeader: 100); var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( includeOnly: new[] { "schools" }); @@ -59,7 +56,7 @@ protected override async Task ArrangeAsync() _options.IncludeDescriptors = false; // Shorten test execution time _options.UseChangeVersionPaging = true; _options.useReversePaging = true; - _options.StreamingPageSize = 40; + _options.StreamingPageSize = _pageSize; _options.ChangeVersionPagingWindowSize = 40; await Task.Yield(); @@ -72,7 +69,6 @@ EdFiApiClient SourceApiClientFactory() => ignoreSslErrors: true, httpClientHandler: new HttpClientHandlerFakeBridge(_fakeSourceRequestHandler)); - var sourceEdFiApiClientProvider = new EdFiApiClientProvider(new Lazy(SourceApiClientFactory)); _messageProducer = new EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer( @@ -80,7 +76,7 @@ EdFiApiClient SourceApiClientFactory() => } [TestCase("schools")] - public async Task Should_produce_messages_correctly(string resourceCollectionName) + public async Task Should_produce_9_pages_with_3_change_query_windows(string resourceCollectionName) { StreamResourceMessage msg = new StreamResourceMessage { @@ -91,54 +87,57 @@ public async Task Should_produce_messages_correctly(string resourceCollectionNam }, ResourceUrl = $"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}", - PageSize = 40 + PageSize = _pageSize }; var pageMessages = await _messageProducer.ProduceMessagesAsync(msg, _options, null, null, new CancellationToken()); pageMessages.ElementAt(0).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 40 }); + pageMessages.ElementAt(1).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 40 }); + pageMessages.ElementAt(2).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 40 }); pageMessages.ElementAt(3).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 41, MaxChangeVersion = 80 }); + pageMessages.ElementAt(4).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 41, MaxChangeVersion = 80 }); + pageMessages.ElementAt(5).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 41, MaxChangeVersion = 80 }); pageMessages.ElementAt(6).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 81, MaxChangeVersion = 100 }); + pageMessages.ElementAt(7).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 81, MaxChangeVersion = 100 }); + pageMessages.ElementAt(8).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 81, MaxChangeVersion = 100 }); pageMessages.ElementAt(0).Offset.ShouldBe(60); - pageMessages.ElementAt(0).Limit.ShouldBe(40); + pageMessages.ElementAt(0).Limit.ShouldBe(_pageSize); pageMessages.ElementAt(1).Offset.ShouldBe(20); - pageMessages.ElementAt(1).Limit.ShouldBe(40); + pageMessages.ElementAt(1).Limit.ShouldBe(_pageSize); pageMessages.ElementAt(2).Offset.ShouldBe(0); pageMessages.ElementAt(2).Limit.ShouldBe(20); pageMessages.ElementAt(3).Offset.ShouldBe(60); - pageMessages.ElementAt(3).Limit.ShouldBe(40); + pageMessages.ElementAt(3).Limit.ShouldBe(_pageSize); pageMessages.ElementAt(4).Offset.ShouldBe(20); - pageMessages.ElementAt(4).Limit.ShouldBe(40); + pageMessages.ElementAt(4).Limit.ShouldBe(_pageSize); pageMessages.ElementAt(5).Offset.ShouldBe(0); pageMessages.ElementAt(5).Limit.ShouldBe(20); pageMessages.ElementAt(6).Offset.ShouldBe(60); - pageMessages.ElementAt(6).Limit.ShouldBe(40); + pageMessages.ElementAt(6).Limit.ShouldBe(_pageSize); pageMessages.ElementAt(7).Offset.ShouldBe(20); - pageMessages.ElementAt(7).Limit.ShouldBe(40); + pageMessages.ElementAt(7).Limit.ShouldBe(_pageSize); pageMessages.ElementAt(8).Offset.ShouldBe(0); pageMessages.ElementAt(8).Limit.ShouldBe(20); } } - [TestFixture] - public class Given_a_MaxChangeVersion_equal_to_100_and_ChangeVersion_page_size_equal_40_and_pageSize_equal_to_50 : TestFixtureAsyncBase + public class Given_100_records_and_ChangeVersion_page_size_equal_40_and_pageSize_equal_to_50 : TestFixtureAsyncBase { - private ChangeProcessor _changeProcessor; - private IFakeHttpRequestHandler _fakeTargetRequestHandler; private IFakeHttpRequestHandler _fakeSourceRequestHandler; - private ChangeProcessorConfiguration _changeProcessorConfiguration; private EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer _messageProducer; private Options _options; + private int _pageSize = 50; protected override async Task ArrangeAsync() { @@ -154,9 +153,7 @@ protected override async Task ArrangeAsync() // Test-specific mocks .AvailableChangeVersions(1100) - .ResourceCount(responseTotalCountHeader: 100) - .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}", suppliedSourceResources) - .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}/deletes", Array.Empty()); + .ResourceCount(responseTotalCountHeader: 100); var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( includeOnly: new[] { "schools" }); @@ -165,8 +162,8 @@ protected override async Task ArrangeAsync() _options.IncludeDescriptors = false; // Shorten test execution time _options.UseChangeVersionPaging = true; _options.useReversePaging = true; - _options.StreamingPageSize = 50; - _options.ChangeVersionPagingWindowSize = 50; + _options.StreamingPageSize = _pageSize; + _options.ChangeVersionPagingWindowSize = 40; await Task.Yield(); @@ -186,9 +183,8 @@ EdFiApiClient SourceApiClientFactory() => } [TestCase("schools")] - public async Task Should_produce_messages_correctly(string resourceCollectionName) + public async Task Should_produce_6_messages_with_3_change_query_windows(string resourceCollectionName) { - var pageSize = 50; StreamResourceMessage msg = new StreamResourceMessage { ChangeWindow = new ChangeWindow() @@ -198,27 +194,159 @@ public async Task Should_produce_messages_correctly(string resourceCollectionNam }, ResourceUrl = $"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}", - PageSize = pageSize + PageSize = _pageSize }; var pageMessages = await _messageProducer.ProduceMessagesAsync(msg, _options, null, null, new CancellationToken()); - pageMessages.ElementAt(0).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 50 }); - pageMessages.ElementAt(2).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 51, MaxChangeVersion = 100 }); + pageMessages.ElementAt(0).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 40 }); + pageMessages.ElementAt(1).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 40 }); + pageMessages.ElementAt(2).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 41, MaxChangeVersion = 80 }); + pageMessages.ElementAt(3).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 41, MaxChangeVersion = 80 }); + pageMessages.ElementAt(4).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 81, MaxChangeVersion = 100 }); + pageMessages.ElementAt(5).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 81, MaxChangeVersion = 100 }); pageMessages.ElementAt(0).Offset.ShouldBe(50); - pageMessages.ElementAt(0).Limit.ShouldBe(pageSize); + pageMessages.ElementAt(0).Limit.ShouldBe(_pageSize); pageMessages.ElementAt(1).Offset.ShouldBe(0); - pageMessages.ElementAt(1).Limit.ShouldBe(pageSize); + pageMessages.ElementAt(1).Limit.ShouldBe(_pageSize); pageMessages.ElementAt(2).Offset.ShouldBe(50); - pageMessages.ElementAt(2).Limit.ShouldBe(pageSize); + pageMessages.ElementAt(2).Limit.ShouldBe(_pageSize); + + pageMessages.ElementAt(3).Offset.ShouldBe(0); + pageMessages.ElementAt(3).Limit.ShouldBe(_pageSize); + + pageMessages.ElementAt(4).Offset.ShouldBe(50); + pageMessages.ElementAt(4).Limit.ShouldBe(_pageSize); + + pageMessages.ElementAt(5).Offset.ShouldBe(0); + pageMessages.ElementAt(5).Limit.ShouldBe(_pageSize); + } + } + + [TestFixture] + public class Given_100_records_and_ChangeVersion_page_size_equal_40_and_pageSize_equal_to_30 : TestFixtureAsyncBase + { + private IFakeHttpRequestHandler _fakeSourceRequestHandler; + private EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer _messageProducer; + private Options _options; + private int _pageSize = 30; + + protected override async Task ArrangeAsync() + { + // ----------------------------------------------------------------- + // Source Requests + // ----------------------------------------------------------------- + var sourceResourceFaker = TestHelpers.GetGenericResourceFaker(); + + var suppliedSourceResources = sourceResourceFaker.Generate(100); + + // Prepare the fake source API endpoint + _fakeSourceRequestHandler = TestHelpers.GetFakeBaselineSourceApiRequestHandler() + + // Test-specific mocks + .AvailableChangeVersions(1100) + .ResourceCount(responseTotalCountHeader: 100); + + var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( + includeOnly: new[] { "schools" }); + + _options = TestHelpers.GetOptions(); + _options.IncludeDescriptors = false; // Shorten test execution time + _options.UseChangeVersionPaging = true; + _options.useReversePaging = true; + _options.StreamingPageSize = _pageSize; + _options.ChangeVersionPagingWindowSize = 40; + + await Task.Yield(); + + EdFiApiClient SourceApiClientFactory() => + new EdFiApiClient( + "TestSource", + sourceApiConnectionDetails, + bearerTokenRefreshMinutes: 27, + ignoreSslErrors: true, + httpClientHandler: new HttpClientHandlerFakeBridge(_fakeSourceRequestHandler)); + + + var sourceEdFiApiClientProvider = new EdFiApiClientProvider(new Lazy(SourceApiClientFactory)); + + _messageProducer = new EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer( + new EdFiApiSourceTotalCountProvider(sourceEdFiApiClientProvider)); + } + + [TestCase("schools")] + public async Task Should_produce_12_messages_with_3_change_query_windows(string resourceCollectionName) + { + StreamResourceMessage msg = new StreamResourceMessage + { + ChangeWindow = new ChangeWindow() + { + MaxChangeVersion = 100, + MinChangeVersion = 0 + + }, + ResourceUrl = $"{EdFiApiConstants.DataManagementApiSegment}{TestHelpers.AnyResourcePattern}", + PageSize = _pageSize + }; + + var pageMessages = await _messageProducer.ProduceMessagesAsync(msg, _options, null, null, new CancellationToken()); + + pageMessages.ElementAt(0).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 40 }); + pageMessages.ElementAt(1).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 40 }); + pageMessages.ElementAt(2).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 40 }); + pageMessages.ElementAt(3).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 0, MaxChangeVersion = 40 }); + + pageMessages.ElementAt(4).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 41, MaxChangeVersion = 80 }); + pageMessages.ElementAt(5).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 41, MaxChangeVersion = 80 }); + pageMessages.ElementAt(6).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 41, MaxChangeVersion = 80 }); + pageMessages.ElementAt(7).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 41, MaxChangeVersion = 80 }); + + pageMessages.ElementAt(8).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 81, MaxChangeVersion = 100 }); + pageMessages.ElementAt(9).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 81, MaxChangeVersion = 100 }); + pageMessages.ElementAt(10).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 81, MaxChangeVersion = 100 }); + pageMessages.ElementAt(11).ChangeWindow.ShouldBeEquivalentTo(new ChangeWindow() { MinChangeVersion = 81, MaxChangeVersion = 100 }); + + pageMessages.ElementAt(0).Offset.ShouldBe(70); + pageMessages.ElementAt(0).Limit.ShouldBe(_pageSize); + + pageMessages.ElementAt(1).Offset.ShouldBe(40); + pageMessages.ElementAt(1).Limit.ShouldBe(_pageSize); + + pageMessages.ElementAt(2).Offset.ShouldBe(10); + pageMessages.ElementAt(2).Limit.ShouldBe(_pageSize); pageMessages.ElementAt(3).Offset.ShouldBe(0); - pageMessages.ElementAt(3).Limit.ShouldBe(pageSize); + pageMessages.ElementAt(3).Limit.ShouldBe(10); + + pageMessages.ElementAt(4).Offset.ShouldBe(70); + pageMessages.ElementAt(4).Limit.ShouldBe(_pageSize); + + pageMessages.ElementAt(5).Offset.ShouldBe(40); + pageMessages.ElementAt(5).Limit.ShouldBe(_pageSize); + + pageMessages.ElementAt(6).Offset.ShouldBe(10); + pageMessages.ElementAt(6).Limit.ShouldBe(_pageSize); + + pageMessages.ElementAt(7).Offset.ShouldBe(0); + pageMessages.ElementAt(7).Limit.ShouldBe(10); + + pageMessages.ElementAt(8).Offset.ShouldBe(70); + pageMessages.ElementAt(8).Limit.ShouldBe(_pageSize); + + pageMessages.ElementAt(9).Offset.ShouldBe(40); + pageMessages.ElementAt(9).Limit.ShouldBe(_pageSize); + + pageMessages.ElementAt(10).Offset.ShouldBe(10); + pageMessages.ElementAt(10).Limit.ShouldBe(_pageSize); + + pageMessages.ElementAt(11).Offset.ShouldBe(0); + pageMessages.ElementAt(11).Limit.ShouldBe(10); } } + } } } From dc5581eeb71412a6750dd020946cfab4c6643c25 Mon Sep 17 00:00:00 2001 From: David Jimenez Date: Wed, 4 Sep 2024 14:59:10 -0600 Subject: [PATCH 7/8] Fixes after rebase --- src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json | 0 .../MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json diff --git a/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json b/src/EdFi.Tools.ApiPublisher.Cli/Properties/launchSettings.json deleted file mode 100644 index e69de29..0000000 diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs index 176e6d5..c20f1f1 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs @@ -197,7 +197,6 @@ public async Task> HandleStreamResourcePageAsyn { break; } - } catch (RateLimiterRejectedException ex) { @@ -212,6 +211,7 @@ public async Task> HandleStreamResourcePageAsyn _logger.Warning($"{message.ResourceUrl}: Rate limit exceeded. Please try again later."); } } + break; } while (true); From bd5fc04f5cff0622b70a801062db113fcd63c94d Mon Sep 17 00:00:00 2001 From: David Jimenez Date: Wed, 4 Sep 2024 16:09:13 -0600 Subject: [PATCH 8/8] Camel case on ApiPublisherSettings --- .../Modules/EdFiApiAsSourceModule.cs | 2 +- .../EdFiApiStreamResourcePageMessageHandler.cs | 2 +- .../Configuration/ApiPublisherSettings.cs | 2 +- .../Processing/ReversePagingTests.cs | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs index 365b678..14efce3 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs @@ -85,7 +85,7 @@ protected override void Load(ContainerBuilder builder) // Register resource page message producer using a ChangeVersion paging strategy if (options.UseChangeVersionPaging) { - if (options.useReversePaging) + if (options.UseReversePaging) { builder.RegisterType() .As() diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs index c20f1f1..03a9e18 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs @@ -177,7 +177,7 @@ public async Task> HandleStreamResourcePageAsyn break; } - if (!options.useReversePaging) + if (!options.UseReversePaging) { // Perform limit/offset final page check (for need for possible continuation) if (message.IsFinalPage && JArray.Parse(responseContent).Count == limit) diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs index 8f2ab91..8e7d85d 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs @@ -95,6 +95,6 @@ public int MaxDegreeOfParallelismForPostResourceItem public int RateLimitTimeLimitMinutes { get; set; } = 1; - public bool useReversePaging { get; set; } = false; + public bool UseReversePaging { get; set; } = false; } } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs index fd8725e..eb44a7d 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs @@ -55,7 +55,7 @@ protected override async Task ArrangeAsync() _options = TestHelpers.GetOptions(); _options.IncludeDescriptors = false; // Shorten test execution time _options.UseChangeVersionPaging = true; - _options.useReversePaging = true; + _options.UseReversePaging = true; _options.StreamingPageSize = _pageSize; _options.ChangeVersionPagingWindowSize = 40; @@ -161,7 +161,7 @@ protected override async Task ArrangeAsync() _options = TestHelpers.GetOptions(); _options.IncludeDescriptors = false; // Shorten test execution time _options.UseChangeVersionPaging = true; - _options.useReversePaging = true; + _options.UseReversePaging = true; _options.StreamingPageSize = _pageSize; _options.ChangeVersionPagingWindowSize = 40; @@ -256,7 +256,7 @@ protected override async Task ArrangeAsync() _options = TestHelpers.GetOptions(); _options.IncludeDescriptors = false; // Shorten test execution time _options.UseChangeVersionPaging = true; - _options.useReversePaging = true; + _options.UseReversePaging = true; _options.StreamingPageSize = _pageSize; _options.ChangeVersionPagingWindowSize = 40;