From b7e10585c67be617d6cb349cf213cd63f7598053 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 11 Oct 2024 11:12:29 +0100 Subject: [PATCH 1/2] EES-5492 - broke orchestrations into their own simplified files and made static, to avoid issues whereby exceptions thrown during the construction of Function classes caused the ImportMetadata stage to lock up --- ...teNextDataSetVersionImportFunctionTests.cs | 4 +- .../Functions/CreateDataSetFunctionTests.cs | 6 +- ...NextDataSetVersionMappingsFunctionTests.cs | 3 +- ...OfNextDataSetVersionImportFunctionTests.cs | 98 +--------------- ...tDataSetVersionImportOrchestrationTests.cs | 105 ++++++++++++++++++ ...ocessInitialDataSetVersionFunctionTests.cs | 104 +---------------- ...InitialDataSetVersionOrchestrationTests.cs | 103 +++++++++++++++++ ...NextDataSetVersionMappingsFunctionTests.cs | 95 +--------------- ...ataSetVersionMappingsOrchestrationTests.cs | 92 +++++++++++++++ .../ProcessorFunctionsIntegrationTest.cs | 6 +- .../Functions/ActivityNames.cs | 14 +-- ...InitialDataSetVersionProcessingFunction.cs | 30 +++++ ...ompleteNextDataSetVersionImportFunction.cs | 2 +- .../Functions/CreateDataSetFunction.cs | 3 +- ...reateNextDataSetVersionMappingsFunction.cs | 2 +- ...ompletionOfNextDataSetVersionFunctions.cs} | 38 +------ ...letionOfNextDataSetVersionOrchestration.cs | 44 ++++++++ ...cessInitialDataSetVersionOrchestration.cs} | 28 +---- ...essNextDataSetVersionMappingsFunctions.cs} | 36 +----- ...NextDataSetVersionMappingsOrchestration.cs | 40 +++++++ 20 files changed, 447 insertions(+), 406 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionOrchestrationTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteInitialDataSetVersionProcessingFunction.cs rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/{ProcessCompletionOfNextDataSetVersionFunction.cs => ProcessCompletionOfNextDataSetVersionFunctions.cs} (64%) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/{ProcessInitialDataSetVersionFunction.cs => ProcessInitialDataSetVersionOrchestration.cs} (57%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/{ProcessNextDataSetVersionMappingsFunction.cs => ProcessNextDataSetVersionMappingsFunctions.cs} (56%) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsOrchestration.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs index 25b2a937bf1..a51130b16ac 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs @@ -48,8 +48,8 @@ public async Task Success() StartOrchestrationOptions? startOrchestrationOptions = null; durableTaskClientMock.Setup(client => client.ScheduleNewOrchestrationInstanceAsync( - nameof(ProcessCompletionOfNextDataSetVersionFunction - .ProcessCompletionOfNextDataSetVersion), + nameof(ProcessCompletionOfNextDataSetVersionOrchestration + .ProcessCompletionOfNextDataSetVersionImport), It.IsAny(), It.IsAny(), It.IsAny())) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs index b6d7e00d5a9..697206ad12b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs @@ -58,7 +58,7 @@ await AddTestData(context => StartOrchestrationOptions? startOrchestrationOptions = null; durableTaskClientMock.Setup(client => client.ScheduleNewOrchestrationInstanceAsync( - nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion), + nameof(ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion), It.IsAny(), It.IsAny(), It.IsAny())) @@ -163,8 +163,8 @@ public async Task ReleaseFileIdHasDataSetVersion_ReturnsValidationProblem() DataSet dataSet = DataFixture.DefaultDataSet(); DataSetVersion dataSetVersion = DataFixture.DefaultDataSetVersion() - .WithRelease(DataFixture.DefaultDataSetVersionRelease() - .WithReleaseFileId(releaseFile.Id)) + .WithRelease(DataFixture.DefaultDataSetVersionRelease() + .WithReleaseFileId(releaseFile.Id)) .WithDataSet(dataSet); await AddTestData(context => diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs index 0cb063daf22..24c47a0b78e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs @@ -41,7 +41,8 @@ public async Task Success() StartOrchestrationOptions? startOrchestrationOptions = null; durableTaskClientMock.Setup(client => client.ScheduleNewOrchestrationInstanceAsync( - nameof(ProcessNextDataSetVersionMappingsFunction.ProcessNextDataSetVersionMappings), + nameof(ProcessNextDataSetVersionMappingsFunctionOrchestration + .ProcessNextDataSetVersionMappings), It.IsAny(), It.IsAny(), It.IsAny())) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs index 382b84d6d4b..466bc9c2dc4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs @@ -7,14 +7,9 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; -using Microsoft.DurableTask; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; using FilterMeta = GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; @@ -37,93 +32,6 @@ public abstract class ProcessCompletionOfNextDataSetVersionImportFunctionTests( TimePeriodsTable.ParquetFile ]; - public class ProcessCompletionOfNextDataSetVersionImportTests( - ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) - { - [Fact] - public async Task Success() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - var activitySequence = new MockSequence(); - - string[] expectedActivitySequence = - [ - ActivityNames.UpdateFileStoragePath, - ActivityNames.ImportMetadata, - ActivityNames.CreateChanges, - ActivityNames.ImportData, - ActivityNames.WriteDataFiles, - ActivityNames.CompleteNextDataSetVersionImportProcessing - ]; - - foreach (var activityName in expectedActivitySequence) - { - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => context.CallActivityAsync(activityName, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - } - - await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - [Fact] - public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - var activitySequence = new MockSequence(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.UpdateFileStoragePath, - mockOrchestrationContext.Object.InstanceId, - null)) - .Throws(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.HandleProcessingFailure, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - - await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - private async Task ProcessCompletionOfNextDataSetVersionImport(TaskOrchestrationContext orchestrationContext) - { - var function = GetRequiredService(); - await function.ProcessCompletionOfNextDataSetVersion( - orchestrationContext, - new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); - } - - private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) - { - var mock = new Mock(MockBehavior.Strict); - - mock.Setup(context => - context.CreateReplaySafeLogger( - nameof(ProcessCompletionOfNextDataSetVersionFunction.ProcessCompletionOfNextDataSetVersion))) - .Returns(NullLogger.Instance); - - mock.SetupGet(context => context.InstanceId) - .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); - - return mock; - } - } - public abstract class CreateChangesTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) @@ -132,7 +40,7 @@ public abstract class CreateChangesTests( protected async Task CreateChanges(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CreateChanges(instanceId, CancellationToken.None); } } @@ -3132,7 +3040,7 @@ public async Task Success_PathNotUpdated() private async Task UpdateFileStoragePath(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.UpdateFileStoragePath(instanceId, CancellationToken.None); } } @@ -3190,7 +3098,7 @@ public async Task DuckDbFileIsDeleted() private async Task CompleteProcessing(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CompleteNextDataSetVersionImportProcessing(instanceId, CancellationToken.None); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs new file mode 100644 index 00000000000..e2250fb24e1 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs @@ -0,0 +1,105 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class ProcessCompletionOfNextDataSetVersionImportOrchestrationTests +{ + public class ProcessCompletionOfNextDataSetVersionImportTests + { + [Fact] + public async Task Success() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + var activitySequence = new MockSequence(); + + // Expect an entity lock to be acquired for calling the ImportMetadata activity + var mockEntityFeature = new Mock(MockBehavior.Strict); + mockEntityFeature.SetupLockForActivity(ActivityNames.ImportMetadata); + mockOrchestrationContext.SetupGet(context => context.Entities) + .Returns(mockEntityFeature.Object); + + string[] expectedActivitySequence = + [ + ActivityNames.UpdateFileStoragePath, + ActivityNames.ImportMetadata, + ActivityNames.CreateChanges, + ActivityNames.ImportData, + ActivityNames.WriteDataFiles, + ActivityNames.CompleteNextDataSetVersionImportProcessing + ]; + + foreach (var activityName in expectedActivitySequence) + { + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => context.CallActivityAsync(activityName, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + } + + await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext, mockEntityFeature); + } + + [Fact] + public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + var activitySequence = new MockSequence(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.UpdateFileStoragePath, + mockOrchestrationContext.Object.InstanceId, + null)) + .Throws(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.HandleProcessingFailure, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + + await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + private static async Task ProcessCompletionOfNextDataSetVersionImport( + TaskOrchestrationContext orchestrationContext) + { + await ProcessCompletionOfNextDataSetVersionOrchestration.ProcessCompletionOfNextDataSetVersionImport( + orchestrationContext, + new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); + } + + private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) + { + var mock = new Mock(MockBehavior.Strict); + + mock.Setup(context => + context.CreateReplaySafeLogger( + nameof(ProcessCompletionOfNextDataSetVersionOrchestration + .ProcessCompletionOfNextDataSetVersionImport))) + .Returns(NullLogger.Instance); + + mock.SetupGet(context => context.InstanceId) + .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); + + return mock; + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs index b42bea901ab..ce333b07ee1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs @@ -3,15 +3,8 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; -using Microsoft.DurableTask; -using Microsoft.DurableTask.Entities; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; @@ -33,99 +26,6 @@ public abstract class ProcessInitialDataSetVersionFunctionTests( TimePeriodsTable.ParquetFile ]; - public class ProcessInitialDataSetVersionTests( - ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessInitialDataSetVersionFunctionTests(fixture) - { - [Fact] - public async Task Success() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - // Expect an entity lock to be acquired for calling the ImportMetadata activity - var mockEntityFeature = new Mock(MockBehavior.Strict); - mockEntityFeature.SetupLockForActivity(ActivityNames.ImportMetadata); - mockOrchestrationContext.SetupGet(context => context.Entities) - .Returns(mockEntityFeature.Object); - - var activitySequence = new MockSequence(); - - string[] expectedActivitySequence = - [ - ActivityNames.CopyCsvFiles, - ActivityNames.ImportMetadata, - ActivityNames.ImportData, - ActivityNames.WriteDataFiles, - ActivityNames.CompleteInitialDataSetVersionProcessing - ]; - - foreach (var activityName in expectedActivitySequence) - { - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => context.CallActivityAsync(activityName, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - } - - await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext, mockEntityFeature); - } - - [Fact] - public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - var activitySequence = new MockSequence(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.CopyCsvFiles, - mockOrchestrationContext.Object.InstanceId, - null)) - .Throws(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.HandleProcessingFailure, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - - await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - private async Task ProcessInitialDataSetVersion(TaskOrchestrationContext orchestrationContext) - { - var function = GetRequiredService(); - await function.ProcessInitialDataSetVersion( - orchestrationContext, - new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); - } - - private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) - { - var mock = new Mock(); - - mock.Setup(context => - context.CreateReplaySafeLogger( - nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion))) - .Returns(NullLogger.Instance); - - mock.SetupGet(context => context.InstanceId) - .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); - - return mock; - } - } - public class CompleteInitialDataSetVersionProcessingTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessInitialDataSetVersionFunctionTests(fixture) @@ -179,8 +79,8 @@ public async Task DuckDbFileIsDeleted() private async Task CompleteProcessing(Guid instanceId) { - var function = GetRequiredService(); - await function.CompleteNextDataSetVersionImportProcessing(instanceId, CancellationToken.None); + var function = GetRequiredService(); + await function.CompleteInitialDataSetVersionProcessing(instanceId, CancellationToken.None); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionOrchestrationTests.cs new file mode 100644 index 00000000000..97f1e35de16 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionOrchestrationTests.cs @@ -0,0 +1,103 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class ProcessInitialDataSetVersionOrchestrationTests +{ + public class ProcessInitialDataSetVersionTests + { + [Fact] + public async Task Success() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + // Expect an entity lock to be acquired for calling the ImportMetadata activity + var mockEntityFeature = new Mock(MockBehavior.Strict); + mockEntityFeature.SetupLockForActivity(ActivityNames.ImportMetadata); + mockOrchestrationContext.SetupGet(context => context.Entities) + .Returns(mockEntityFeature.Object); + + var activitySequence = new MockSequence(); + + string[] expectedActivitySequence = + [ + ActivityNames.CopyCsvFiles, + ActivityNames.ImportMetadata, + ActivityNames.ImportData, + ActivityNames.WriteDataFiles, + ActivityNames.CompleteInitialDataSetVersionProcessing + ]; + + foreach (var activityName in expectedActivitySequence) + { + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => context.CallActivityAsync(activityName, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + } + + await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext, mockEntityFeature); + } + + [Fact] + public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + var activitySequence = new MockSequence(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.CopyCsvFiles, + mockOrchestrationContext.Object.InstanceId, + null)) + .Throws(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.HandleProcessingFailure, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + + await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + private static async Task ProcessInitialDataSetVersion(TaskOrchestrationContext orchestrationContext) + { + await ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion( + orchestrationContext, + new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); + } + + private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) + { + var mock = new Mock(); + + mock.Setup(context => + context.CreateReplaySafeLogger( + nameof(ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion))) + .Returns(NullLogger.Instance); + + mock.SetupGet(context => context.InstanceId) + .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); + + return mock; + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs index 77d1b1f78b4..688ac38c48b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs @@ -9,13 +9,8 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Utils; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; -using Microsoft.DurableTask; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; @@ -23,90 +18,6 @@ public abstract class ProcessNextDataSetVersionMappingsFunctionTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { - public class ProcessNextDataSetVersionMappingsTests( - ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessNextDataSetVersionMappingsFunctionTests(fixture) - { - [Fact] - public async Task Success() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - var activitySequence = new MockSequence(); - - string[] expectedActivitySequence = - [ - ActivityNames.CopyCsvFiles, - ActivityNames.CreateMappings, - ActivityNames.ApplyAutoMappings, - ActivityNames.CompleteNextDataSetVersionMappingProcessing, - ]; - - foreach (var activityName in expectedActivitySequence) - { - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => context.CallActivityAsync(activityName, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - } - - await ProcessNextDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - [Fact] - public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - var activitySequence = new MockSequence(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.CopyCsvFiles, - mockOrchestrationContext.Object.InstanceId, - null)) - .Throws(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.HandleProcessingFailure, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - - await ProcessNextDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - private async Task ProcessNextDataSetVersion(TaskOrchestrationContext orchestrationContext) - { - var function = GetRequiredService(); - await function.ProcessNextDataSetVersionMappings( - orchestrationContext, - new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); - } - - private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) - { - var mock = new Mock(MockBehavior.Strict); - - mock.Setup(context => context.CreateReplaySafeLogger( - nameof(ProcessNextDataSetVersionMappingsFunction.ProcessNextDataSetVersionMappings))) - .Returns(NullLogger.Instance); - - mock.SetupGet(context => context.InstanceId) - .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); - - return mock; - } - } - public abstract class CreateMappingsTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessNextDataSetVersionMappingsFunctionTests(fixture) @@ -115,7 +26,7 @@ public abstract class CreateMappingsTests( protected async Task CreateMappings(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CreateMappings(instanceId, CancellationToken.None); } } @@ -629,7 +540,7 @@ public abstract class ApplyAutoMappingsTests( protected async Task ApplyAutoMappings(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.ApplyAutoMappings(instanceId, CancellationToken.None); } } @@ -1790,7 +1701,7 @@ public async Task Success() private async Task CompleteProcessing(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CompleteNextDataSetVersionMappingProcessing(instanceId, CancellationToken.None); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs new file mode 100644 index 00000000000..252deb849d0 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs @@ -0,0 +1,92 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class ProcessNextDataSetVersionMappingsOrchestrationTests +{ + public class ProcessNextDataSetVersionMappingsTests + { + [Fact] + public async Task Success() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + var activitySequence = new MockSequence(); + + string[] expectedActivitySequence = + [ + ActivityNames.CopyCsvFiles, + ActivityNames.CreateMappings, + ActivityNames.ApplyAutoMappings, + ActivityNames.CompleteNextDataSetVersionMappingProcessing, + ]; + + foreach (var activityName in expectedActivitySequence) + { + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => context.CallActivityAsync(activityName, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + } + + await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + [Fact] + public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + var activitySequence = new MockSequence(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.CopyCsvFiles, + mockOrchestrationContext.Object.InstanceId, + null)) + .Throws(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.HandleProcessingFailure, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + + await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + private async Task ProcessNextDataSetVersion(TaskOrchestrationContext orchestrationContext) + { + await ProcessNextDataSetVersionMappingsFunctionOrchestration.ProcessNextDataSetVersionMappings( + orchestrationContext, + new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); + } + + private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) + { + var mock = new Mock(MockBehavior.Strict); + + mock.Setup(context => context.CreateReplaySafeLogger( + nameof(ProcessNextDataSetVersionMappingsFunctionOrchestration.ProcessNextDataSetVersionMappings))) + .Returns(NullLogger.Instance); + + mock.SetupGet(context => context.InstanceId) + .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); + + return mock; + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs index e9d51bad276..b15821ee4c5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs @@ -288,11 +288,11 @@ protected override IEnumerable GetFunctionTypes() return [ typeof(CreateDataSetFunction), - typeof(ProcessInitialDataSetVersionFunction), + typeof(CompleteInitialDataSetVersionProcessingFunction), typeof(CreateNextDataSetVersionMappingsFunction), - typeof(ProcessNextDataSetVersionMappingsFunction), + typeof(ProcessNextDataSetVersionMappingsFunctions), typeof(CompleteNextDataSetVersionImportFunction), - typeof(ProcessCompletionOfNextDataSetVersionFunction), + typeof(ProcessCompletionOfNextDataSetVersionFunctions), typeof(DeleteDataSetVersionFunction), typeof(CopyCsvFilesFunction), typeof(ImportMetadataFunction), diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs index b990d7e2b45..f1f002e15c4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs @@ -9,17 +9,17 @@ internal static class ActivityNames public const string HandleProcessingFailure = nameof(HandleProcessingFailureFunction.HandleProcessingFailure); public const string CompleteInitialDataSetVersionProcessing = - nameof(ProcessInitialDataSetVersionFunction.CompleteInitialDataSetVersionProcessing); + nameof(CompleteInitialDataSetVersionProcessingFunction.CompleteInitialDataSetVersionProcessing); - public const string CreateMappings = nameof(ProcessNextDataSetVersionMappingsFunction.CreateMappings); - public const string ApplyAutoMappings = nameof(ProcessNextDataSetVersionMappingsFunction.ApplyAutoMappings); + public const string CreateMappings = nameof(ProcessNextDataSetVersionMappingsFunctions.CreateMappings); + public const string ApplyAutoMappings = nameof(ProcessNextDataSetVersionMappingsFunctions.ApplyAutoMappings); public const string CompleteNextDataSetVersionMappingProcessing = - nameof(ProcessNextDataSetVersionMappingsFunction.CompleteNextDataSetVersionMappingProcessing); + nameof(ProcessNextDataSetVersionMappingsFunctions.CompleteNextDataSetVersionMappingProcessing); public const string CreateChanges = - nameof(ProcessCompletionOfNextDataSetVersionFunction.CreateChanges); + nameof(ProcessCompletionOfNextDataSetVersionFunctions.CreateChanges); public const string UpdateFileStoragePath = - nameof(ProcessCompletionOfNextDataSetVersionFunction.UpdateFileStoragePath); + nameof(ProcessCompletionOfNextDataSetVersionFunctions.UpdateFileStoragePath); public const string CompleteNextDataSetVersionImportProcessing = - nameof(ProcessCompletionOfNextDataSetVersionFunction.CompleteNextDataSetVersionImportProcessing); + nameof(ProcessCompletionOfNextDataSetVersionFunctions.CompleteNextDataSetVersionImportProcessing); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteInitialDataSetVersionProcessingFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteInitialDataSetVersionProcessingFunction.cs new file mode 100644 index 00000000000..b0d20ac0fbb --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteInitialDataSetVersionProcessingFunction.cs @@ -0,0 +1,30 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using Microsoft.Azure.Functions.Worker; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class CompleteInitialDataSetVersionProcessingFunction( + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver dataSetVersionPathResolver) : BaseProcessDataSetVersionFunction(publicDataDbContext) +{ + [Function(ActivityNames.CompleteInitialDataSetVersionProcessing)] + public async Task CompleteInitialDataSetVersionProcessing( + [ActivityTrigger] Guid instanceId, + CancellationToken cancellationToken) + { + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.Completing, cancellationToken); + + var dataSetVersion = dataSetVersionImport.DataSetVersion; + + // Delete the DuckDb database file as it is no longer needed + File.Delete(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); + + dataSetVersion.Status = DataSetVersionStatus.Draft; + + dataSetVersionImport.Completed = DateTimeOffset.UtcNow; + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs index 3351d89d3ec..8273ec0ef3e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs @@ -62,7 +62,7 @@ private async Task ProcessCompletionOfNextDataSetVersionImport( CancellationToken cancellationToken) { const string orchestratorName = - nameof(ProcessCompletionOfNextDataSetVersionFunction.ProcessCompletionOfNextDataSetVersion); + nameof(ProcessCompletionOfNextDataSetVersionOrchestration.ProcessCompletionOfNextDataSetVersionImport); var input = new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersionId }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs index aed6af84bed..5660fdfab27 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs @@ -59,7 +59,8 @@ private async Task ProcessInitialDataSetVersion( Guid instanceId, CancellationToken cancellationToken) { - const string orchestratorName = nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion); + const string orchestratorName = + nameof(ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion); var input = new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersionId }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs index 81a6b6058c7..f253abffc88 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs @@ -64,7 +64,7 @@ private async Task ProcessNextDataSetVersion( CancellationToken cancellationToken) { const string orchestratorName = - nameof(ProcessNextDataSetVersionMappingsFunction.ProcessNextDataSetVersionMappings); + nameof(ProcessNextDataSetVersionMappingsFunctionOrchestration.ProcessNextDataSetVersionMappings); var input = new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersionId }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs similarity index 64% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs index a71de9a718c..e9bae2c8a76 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs @@ -5,51 +5,15 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.Azure.Functions.Worker; -using Microsoft.DurableTask; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class ProcessCompletionOfNextDataSetVersionFunction( +public class ProcessCompletionOfNextDataSetVersionFunctions( PublicDataDbContext publicDataDbContext, IDataSetVersionPathResolver dataSetVersionPathResolver, IDataSetVersionChangeService dataSetVersionChangeService) : BaseProcessDataSetVersionFunction(publicDataDbContext) { - [Function(nameof(ProcessCompletionOfNextDataSetVersion))] - public async Task ProcessCompletionOfNextDataSetVersion( - [OrchestrationTrigger] TaskOrchestrationContext context, - ProcessDataSetVersionContext input) - { - var logger = context.CreateReplaySafeLogger(nameof(ProcessCompletionOfNextDataSetVersion)); - - logger.LogInformation( - "Processing completion of import for next data set version (InstanceId={InstanceId}, " + - "DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - try - { - await context.CallActivity(ActivityNames.UpdateFileStoragePath, logger, context.InstanceId); - await context.CallActivity(ActivityNames.ImportMetadata, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CreateChanges, logger, context.InstanceId); - await context.CallActivity(ActivityNames.ImportData, logger, context.InstanceId); - await context.CallActivity(ActivityNames.WriteDataFiles, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CompleteNextDataSetVersionImportProcessing, logger, - context.InstanceId); - } - catch (Exception e) - { - logger.LogError(e, - "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); - } - } - [Function(ActivityNames.CreateChanges)] public async Task CreateChanges( [ActivityTrigger] Guid instanceId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs new file mode 100644 index 00000000000..57db11c7180 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs @@ -0,0 +1,44 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public static class ProcessCompletionOfNextDataSetVersionOrchestration +{ + [Function(nameof(ProcessCompletionOfNextDataSetVersionImport))] + public static async Task ProcessCompletionOfNextDataSetVersionImport( + [OrchestrationTrigger] TaskOrchestrationContext context, + ProcessDataSetVersionContext input) + { + var logger = context.CreateReplaySafeLogger(nameof(ProcessCompletionOfNextDataSetVersionImport)); + + logger.LogInformation( + "Processing completion of import for next data set version (InstanceId={InstanceId}, " + + "DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + try + { + await context.CallActivity(ActivityNames.UpdateFileStoragePath, logger, context.InstanceId); + await context.CallActivityExclusively(ActivityNames.ImportMetadata, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CreateChanges, logger, context.InstanceId); + await context.CallActivity(ActivityNames.ImportData, logger, context.InstanceId); + await context.CallActivity(ActivityNames.WriteDataFiles, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CompleteNextDataSetVersionImportProcessing, logger, + context.InstanceId); + } + catch (Exception e) + { + logger.LogError(e, + "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionOrchestration.cs similarity index 57% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionOrchestration.cs index 9ba387dc7e9..f677a68986c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionOrchestration.cs @@ -1,20 +1,15 @@ -using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.Azure.Functions.Worker; using Microsoft.DurableTask; using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class ProcessInitialDataSetVersionFunction( - PublicDataDbContext publicDataDbContext, - IDataSetVersionPathResolver dataSetVersionPathResolver) : BaseProcessDataSetVersionFunction(publicDataDbContext) +public static class ProcessInitialDataSetVersionOrchestration { [Function(nameof(ProcessInitialDataSetVersion))] - public async Task ProcessInitialDataSetVersion( + public static async Task ProcessInitialDataSetVersion( [OrchestrationTrigger] TaskOrchestrationContext context, ProcessDataSetVersionContext input) { @@ -44,23 +39,4 @@ await context.CallActivity(ActivityNames.CompleteInitialDataSetVersionProcessing await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); } } - - [Function(ActivityNames.CompleteInitialDataSetVersionProcessing)] - public async Task CompleteInitialDataSetVersionProcessing( - [ActivityTrigger] Guid instanceId, - CancellationToken cancellationToken) - { - var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); - await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.Completing, cancellationToken); - - var dataSetVersion = dataSetVersionImport.DataSetVersion; - - // Delete the DuckDb database file as it is no longer needed - File.Delete(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); - - dataSetVersion.Status = DataSetVersionStatus.Draft; - - dataSetVersionImport.Completed = DateTimeOffset.UtcNow; - await publicDataDbContext.SaveChangesAsync(cancellationToken); - } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunctions.cs similarity index 56% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunctions.cs index c1507e6adc3..a24c1189444 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunctions.cs @@ -1,48 +1,14 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; using Microsoft.Azure.Functions.Worker; -using Microsoft.DurableTask; -using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class ProcessNextDataSetVersionMappingsFunction( +public class ProcessNextDataSetVersionMappingsFunctions( PublicDataDbContext publicDataDbContext, IDataSetVersionMappingService mappingService) : BaseProcessDataSetVersionFunction(publicDataDbContext) { - [Function(nameof(ProcessNextDataSetVersionMappings))] - public async Task ProcessNextDataSetVersionMappings([OrchestrationTrigger] TaskOrchestrationContext context, - ProcessDataSetVersionContext input) - { - var logger = context.CreateReplaySafeLogger(nameof(ProcessNextDataSetVersionMappings)); - - logger.LogInformation( - "Processing next data set version (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - try - { - await context.CallActivity(ActivityNames.CopyCsvFiles, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CreateMappings, logger, context.InstanceId); - await context.CallActivity(ActivityNames.ApplyAutoMappings, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CompleteNextDataSetVersionMappingProcessing, logger, - context.InstanceId); - } - catch (Exception e) - { - logger.LogError(e, - "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); - } - } - [Function(ActivityNames.CreateMappings)] public async Task CreateMappings( [ActivityTrigger] Guid instanceId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsOrchestration.cs new file mode 100644 index 00000000000..c2dc922c9d9 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsOrchestration.cs @@ -0,0 +1,40 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public static class ProcessNextDataSetVersionMappingsFunctionOrchestration +{ + [Function(nameof(ProcessNextDataSetVersionMappings))] + public static async Task ProcessNextDataSetVersionMappings([OrchestrationTrigger] TaskOrchestrationContext context, + ProcessDataSetVersionContext input) + { + var logger = context.CreateReplaySafeLogger(nameof(ProcessNextDataSetVersionMappings)); + + logger.LogInformation( + "Processing next data set version (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + try + { + await context.CallActivity(ActivityNames.CopyCsvFiles, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CreateMappings, logger, context.InstanceId); + await context.CallActivity(ActivityNames.ApplyAutoMappings, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CompleteNextDataSetVersionMappingProcessing, logger, + context.InstanceId); + } + catch (Exception e) + { + logger.LogError(e, + "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); + } + } +} From 93b5d5e585ce252324c1ae480be7e8732a41e9d7 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 30 Sep 2024 09:45:57 +0100 Subject: [PATCH 2/2] EES-5492 - renamed a lot of test classes and method names in response to PR comments! --- ...nitialDataSetVersionProcessingFunctionTests.cs} | 12 ++++++------ ...ompletionOfNextDataSetVersionFunctionsTests.cs} | 14 +++++++------- ...etionOfNextDataSetVersionOrchestrationTests.cs} | 2 +- ...essNextDataSetVersionMappingsFunctionsTests.cs} | 8 ++++---- ...NextDataSetVersionMappingsOrchestrationTests.cs | 6 +++--- .../public_api_minor_manual_changes.robot | 8 ++++---- 6 files changed, 25 insertions(+), 25 deletions(-) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/{ProcessInitialDataSetVersionFunctionTests.cs => CompleteInitialDataSetVersionProcessingFunctionTests.cs} (88%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/{ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs => ProcessCompletionOfNextDataSetVersionFunctionsTests.cs} (99%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/{ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs => ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs} (98%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/{ProcessNextDataSetVersionMappingsFunctionTests.cs => ProcessNextDataSetVersionMappingsFunctionsTests.cs} (99%) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteInitialDataSetVersionProcessingFunctionTests.cs similarity index 88% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteInitialDataSetVersionProcessingFunctionTests.cs index ce333b07ee1..17ec1689a0f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteInitialDataSetVersionProcessingFunctionTests.cs @@ -8,7 +8,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; -public abstract class ProcessInitialDataSetVersionFunctionTests( +public abstract class CompleteInitialDataSetVersionProcessingFunctionTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { @@ -26,9 +26,9 @@ public abstract class ProcessInitialDataSetVersionFunctionTests( TimePeriodsTable.ParquetFile ]; - public class CompleteInitialDataSetVersionProcessingTests( + public class CompleteInitialDataSetVersionProcessingProcessingTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessInitialDataSetVersionFunctionTests(fixture) + : CompleteInitialDataSetVersionProcessingFunctionTests(fixture) { private const DataSetVersionImportStage Stage = DataSetVersionImportStage.Completing; @@ -40,7 +40,7 @@ public async Task Success() var dataSetVersionPathResolver = GetRequiredService(); Directory.CreateDirectory(dataSetVersionPathResolver.DirectoryPath(dataSetVersion)); - await CompleteProcessing(instanceId); + await CompleteInitialDataSetVersionProcessing(instanceId); await using var publicDataDbContext = GetDbContext(); @@ -68,7 +68,7 @@ public async Task DuckDbFileIsDeleted() await File.Create(Path.Combine(directoryPath, filename)).DisposeAsync(); } - await CompleteProcessing(instanceId); + await CompleteInitialDataSetVersionProcessing(instanceId); // Ensure the duck db database file is the only file that was deleted AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion, @@ -77,7 +77,7 @@ public async Task DuckDbFileIsDeleted() .ToArray()); } - private async Task CompleteProcessing(Guid instanceId) + private async Task CompleteInitialDataSetVersionProcessing(Guid instanceId) { var function = GetRequiredService(); await function.CompleteInitialDataSetVersionProcessing(instanceId, CancellationToken.None); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionFunctionsTests.cs similarity index 99% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionFunctionsTests.cs index 466bc9c2dc4..ad18ad3daa2 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionFunctionsTests.cs @@ -14,7 +14,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; -public abstract class ProcessCompletionOfNextDataSetVersionImportFunctionTests( +public abstract class ProcessCompletionOfNextDataSetVersionFunctionsTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { @@ -34,7 +34,7 @@ public abstract class ProcessCompletionOfNextDataSetVersionImportFunctionTests( public abstract class CreateChangesTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) + : ProcessCompletionOfNextDataSetVersionFunctionsTests(fixture) { protected const DataSetVersionImportStage Stage = DataSetVersionImportStage.CreatingChanges; @@ -2992,7 +2992,7 @@ .. DataFixture.DefaultTimePeriodMeta() public class UpdateFileStoragePathTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) + : ProcessCompletionOfNextDataSetVersionFunctionsTests(fixture) { private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ManualMapping; @@ -3047,7 +3047,7 @@ private async Task UpdateFileStoragePath(Guid instanceId) public class CompleteNextDataSetVersionImportProcessingTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) + : ProcessCompletionOfNextDataSetVersionFunctionsTests(fixture) { private const DataSetVersionImportStage Stage = DataSetVersionImportStage.Completing; @@ -3059,7 +3059,7 @@ public async Task Success() var dataSetVersionPathResolver = GetRequiredService(); Directory.CreateDirectory(dataSetVersionPathResolver.DirectoryPath(dataSetVersion)); - await CompleteProcessing(instanceId); + await CompleteNextDataSetVersionImportProcessing(instanceId); await using var publicDataDbContext = GetDbContext(); @@ -3087,7 +3087,7 @@ public async Task DuckDbFileIsDeleted() await File.Create(Path.Combine(directoryPath, filename)).DisposeAsync(); } - await CompleteProcessing(instanceId); + await CompleteNextDataSetVersionImportProcessing(instanceId); // Ensure the duck db database file is the only file that was deleted AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion, @@ -3096,7 +3096,7 @@ public async Task DuckDbFileIsDeleted() .ToArray()); } - private async Task CompleteProcessing(Guid instanceId) + private async Task CompleteNextDataSetVersionImportProcessing(Guid instanceId) { var function = GetRequiredService(); await function.CompleteNextDataSetVersionImportProcessing(instanceId, CancellationToken.None); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs similarity index 98% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs index e2250fb24e1..4cf6c417a07 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs @@ -9,7 +9,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; -public abstract class ProcessCompletionOfNextDataSetVersionImportOrchestrationTests +public abstract class ProcessCompletionOfNextDataSetVersionOrchestrationTests { public class ProcessCompletionOfNextDataSetVersionImportTests { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionsTests.cs similarity index 99% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionsTests.cs index 688ac38c48b..ad71c0bd3a4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionsTests.cs @@ -14,13 +14,13 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; -public abstract class ProcessNextDataSetVersionMappingsFunctionTests( +public abstract class ProcessNextDataSetVersionMappingsFunctionsTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { public abstract class CreateMappingsTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessNextDataSetVersionMappingsFunctionTests(fixture) + : ProcessNextDataSetVersionMappingsFunctionsTests(fixture) { protected const DataSetVersionImportStage Stage = DataSetVersionImportStage.CreatingMappings; @@ -534,7 +534,7 @@ public async Task Success_Candidates() public abstract class ApplyAutoMappingsTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessNextDataSetVersionMappingsFunctionTests(fixture) + : ProcessNextDataSetVersionMappingsFunctionsTests(fixture) { protected const DataSetVersionImportStage Stage = DataSetVersionImportStage.AutoMapping; @@ -1673,7 +1673,7 @@ public async Task Complete_HasDeletedTimePeriods_MajorUpdate() public class CompleteNextDataSetVersionMappingsMappingProcessingTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessNextDataSetVersionMappingsFunctionTests(fixture) + : ProcessNextDataSetVersionMappingsFunctionsTests(fixture) { private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ManualMapping; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs index 252deb849d0..a2d17dc1616 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs @@ -35,7 +35,7 @@ public async Task Success() .Returns(Task.CompletedTask); } - await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + await ProcessNextDataSetVersionMappings(mockOrchestrationContext.Object); VerifyAllMocks(mockOrchestrationContext); } @@ -63,12 +63,12 @@ public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() null)) .Returns(Task.CompletedTask); - await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + await ProcessNextDataSetVersionMappings(mockOrchestrationContext.Object); VerifyAllMocks(mockOrchestrationContext); } - private async Task ProcessNextDataSetVersion(TaskOrchestrationContext orchestrationContext) + private async Task ProcessNextDataSetVersionMappings(TaskOrchestrationContext orchestrationContext) { await ProcessNextDataSetVersionMappingsFunctionOrchestration.ProcessNextDataSetVersionMappings( orchestrationContext, diff --git a/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot b/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot index 75d4c186f76..6d9e96e4e72 100644 --- a/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot +++ b/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot @@ -193,7 +193,7 @@ Verify location mapping changes user waits until element contains xpath://table[@data-testid='mappable-table-region']/caption//strong[1] ... 1 mapped location %{WAIT_LONG} -Validate the row headings and its contents in the 'Regions' section(after mapping) +Validate the row headings and its contents in the 'Regions' section after mapping user waits until h3 is visible Locations not found in new data set user checks table column heading contains 1 1 Current data set user checks table column heading contains 1 2 New data set @@ -207,12 +207,12 @@ Validate the row headings and its contents in the 'Regions' section(after mappin user clicks link Back -Validate the version status of location task +Validate the version status of location task is now complete user waits until h3 is visible Draft version tasks user waits until parent contains element testid:map-locations-task link:Map locations user waits until parent contains element id:map-locations-task-status text:Complete user waits until parent contains element testid:map-filters-task link:Map filters - user waits until parent contains element id:map-filters-task-status text:Complete + user waits until parent contains element id:map-filters-task-status text:Incomplete User clicks on Map filters link user clicks link Map filters @@ -247,7 +247,7 @@ Verify filter mapping changes user waits until element contains xpath://table[@data-testid='mappable-table-school_type']/caption//strong[1] ... 1 mapped filter option %{WAIT_LONG} -Validate the row headings and its contents in the 'filters options' section(after mapping) +Validate the row headings and its contents in the 'filters options' section after mapping user waits until h3 is visible Filter options not found in new data set user checks table column heading contains 1 1 Current data set user checks table column heading contains 1 2 New data set