diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs index 25b2a937bf1..a51130b16ac 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs @@ -48,8 +48,8 @@ public async Task Success() StartOrchestrationOptions? startOrchestrationOptions = null; durableTaskClientMock.Setup(client => client.ScheduleNewOrchestrationInstanceAsync( - nameof(ProcessCompletionOfNextDataSetVersionFunction - .ProcessCompletionOfNextDataSetVersion), + nameof(ProcessCompletionOfNextDataSetVersionOrchestration + .ProcessCompletionOfNextDataSetVersionImport), It.IsAny(), It.IsAny(), It.IsAny())) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs index b6d7e00d5a9..697206ad12b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs @@ -58,7 +58,7 @@ await AddTestData(context => StartOrchestrationOptions? startOrchestrationOptions = null; durableTaskClientMock.Setup(client => client.ScheduleNewOrchestrationInstanceAsync( - nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion), + nameof(ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion), It.IsAny(), It.IsAny(), It.IsAny())) @@ -163,8 +163,8 @@ public async Task ReleaseFileIdHasDataSetVersion_ReturnsValidationProblem() DataSet dataSet = DataFixture.DefaultDataSet(); DataSetVersion dataSetVersion = DataFixture.DefaultDataSetVersion() - .WithRelease(DataFixture.DefaultDataSetVersionRelease() - .WithReleaseFileId(releaseFile.Id)) + .WithRelease(DataFixture.DefaultDataSetVersionRelease() + .WithReleaseFileId(releaseFile.Id)) .WithDataSet(dataSet); await AddTestData(context => diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs index 0cb063daf22..24c47a0b78e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs @@ -41,7 +41,8 @@ public async Task Success() StartOrchestrationOptions? startOrchestrationOptions = null; durableTaskClientMock.Setup(client => client.ScheduleNewOrchestrationInstanceAsync( - nameof(ProcessNextDataSetVersionMappingsFunction.ProcessNextDataSetVersionMappings), + nameof(ProcessNextDataSetVersionMappingsFunctionOrchestration + .ProcessNextDataSetVersionMappings), It.IsAny(), It.IsAny(), It.IsAny())) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs index 382b84d6d4b..466bc9c2dc4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs @@ -7,14 +7,9 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; -using Microsoft.DurableTask; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; using FilterMeta = GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; @@ -37,93 +32,6 @@ public abstract class ProcessCompletionOfNextDataSetVersionImportFunctionTests( TimePeriodsTable.ParquetFile ]; - public class ProcessCompletionOfNextDataSetVersionImportTests( - ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) - { - [Fact] - public async Task Success() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - var activitySequence = new MockSequence(); - - string[] expectedActivitySequence = - [ - ActivityNames.UpdateFileStoragePath, - ActivityNames.ImportMetadata, - ActivityNames.CreateChanges, - ActivityNames.ImportData, - ActivityNames.WriteDataFiles, - ActivityNames.CompleteNextDataSetVersionImportProcessing - ]; - - foreach (var activityName in expectedActivitySequence) - { - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => context.CallActivityAsync(activityName, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - } - - await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - [Fact] - public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - var activitySequence = new MockSequence(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.UpdateFileStoragePath, - mockOrchestrationContext.Object.InstanceId, - null)) - .Throws(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.HandleProcessingFailure, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - - await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - private async Task ProcessCompletionOfNextDataSetVersionImport(TaskOrchestrationContext orchestrationContext) - { - var function = GetRequiredService(); - await function.ProcessCompletionOfNextDataSetVersion( - orchestrationContext, - new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); - } - - private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) - { - var mock = new Mock(MockBehavior.Strict); - - mock.Setup(context => - context.CreateReplaySafeLogger( - nameof(ProcessCompletionOfNextDataSetVersionFunction.ProcessCompletionOfNextDataSetVersion))) - .Returns(NullLogger.Instance); - - mock.SetupGet(context => context.InstanceId) - .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); - - return mock; - } - } - public abstract class CreateChangesTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) @@ -132,7 +40,7 @@ public abstract class CreateChangesTests( protected async Task CreateChanges(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CreateChanges(instanceId, CancellationToken.None); } } @@ -3132,7 +3040,7 @@ public async Task Success_PathNotUpdated() private async Task UpdateFileStoragePath(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.UpdateFileStoragePath(instanceId, CancellationToken.None); } } @@ -3190,7 +3098,7 @@ public async Task DuckDbFileIsDeleted() private async Task CompleteProcessing(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CompleteNextDataSetVersionImportProcessing(instanceId, CancellationToken.None); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs new file mode 100644 index 00000000000..e2250fb24e1 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs @@ -0,0 +1,105 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class ProcessCompletionOfNextDataSetVersionImportOrchestrationTests +{ + public class ProcessCompletionOfNextDataSetVersionImportTests + { + [Fact] + public async Task Success() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + var activitySequence = new MockSequence(); + + // Expect an entity lock to be acquired for calling the ImportMetadata activity + var mockEntityFeature = new Mock(MockBehavior.Strict); + mockEntityFeature.SetupLockForActivity(ActivityNames.ImportMetadata); + mockOrchestrationContext.SetupGet(context => context.Entities) + .Returns(mockEntityFeature.Object); + + string[] expectedActivitySequence = + [ + ActivityNames.UpdateFileStoragePath, + ActivityNames.ImportMetadata, + ActivityNames.CreateChanges, + ActivityNames.ImportData, + ActivityNames.WriteDataFiles, + ActivityNames.CompleteNextDataSetVersionImportProcessing + ]; + + foreach (var activityName in expectedActivitySequence) + { + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => context.CallActivityAsync(activityName, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + } + + await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext, mockEntityFeature); + } + + [Fact] + public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + var activitySequence = new MockSequence(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.UpdateFileStoragePath, + mockOrchestrationContext.Object.InstanceId, + null)) + .Throws(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.HandleProcessingFailure, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + + await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + private static async Task ProcessCompletionOfNextDataSetVersionImport( + TaskOrchestrationContext orchestrationContext) + { + await ProcessCompletionOfNextDataSetVersionOrchestration.ProcessCompletionOfNextDataSetVersionImport( + orchestrationContext, + new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); + } + + private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) + { + var mock = new Mock(MockBehavior.Strict); + + mock.Setup(context => + context.CreateReplaySafeLogger( + nameof(ProcessCompletionOfNextDataSetVersionOrchestration + .ProcessCompletionOfNextDataSetVersionImport))) + .Returns(NullLogger.Instance); + + mock.SetupGet(context => context.InstanceId) + .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); + + return mock; + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs index b42bea901ab..ce333b07ee1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs @@ -3,15 +3,8 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; -using Microsoft.DurableTask; -using Microsoft.DurableTask.Entities; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; @@ -33,99 +26,6 @@ public abstract class ProcessInitialDataSetVersionFunctionTests( TimePeriodsTable.ParquetFile ]; - public class ProcessInitialDataSetVersionTests( - ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessInitialDataSetVersionFunctionTests(fixture) - { - [Fact] - public async Task Success() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - // Expect an entity lock to be acquired for calling the ImportMetadata activity - var mockEntityFeature = new Mock(MockBehavior.Strict); - mockEntityFeature.SetupLockForActivity(ActivityNames.ImportMetadata); - mockOrchestrationContext.SetupGet(context => context.Entities) - .Returns(mockEntityFeature.Object); - - var activitySequence = new MockSequence(); - - string[] expectedActivitySequence = - [ - ActivityNames.CopyCsvFiles, - ActivityNames.ImportMetadata, - ActivityNames.ImportData, - ActivityNames.WriteDataFiles, - ActivityNames.CompleteInitialDataSetVersionProcessing - ]; - - foreach (var activityName in expectedActivitySequence) - { - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => context.CallActivityAsync(activityName, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - } - - await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext, mockEntityFeature); - } - - [Fact] - public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - var activitySequence = new MockSequence(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.CopyCsvFiles, - mockOrchestrationContext.Object.InstanceId, - null)) - .Throws(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.HandleProcessingFailure, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - - await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - private async Task ProcessInitialDataSetVersion(TaskOrchestrationContext orchestrationContext) - { - var function = GetRequiredService(); - await function.ProcessInitialDataSetVersion( - orchestrationContext, - new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); - } - - private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) - { - var mock = new Mock(); - - mock.Setup(context => - context.CreateReplaySafeLogger( - nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion))) - .Returns(NullLogger.Instance); - - mock.SetupGet(context => context.InstanceId) - .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); - - return mock; - } - } - public class CompleteInitialDataSetVersionProcessingTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessInitialDataSetVersionFunctionTests(fixture) @@ -179,8 +79,8 @@ public async Task DuckDbFileIsDeleted() private async Task CompleteProcessing(Guid instanceId) { - var function = GetRequiredService(); - await function.CompleteNextDataSetVersionImportProcessing(instanceId, CancellationToken.None); + var function = GetRequiredService(); + await function.CompleteInitialDataSetVersionProcessing(instanceId, CancellationToken.None); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionOrchestrationTests.cs new file mode 100644 index 00000000000..97f1e35de16 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionOrchestrationTests.cs @@ -0,0 +1,103 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class ProcessInitialDataSetVersionOrchestrationTests +{ + public class ProcessInitialDataSetVersionTests + { + [Fact] + public async Task Success() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + // Expect an entity lock to be acquired for calling the ImportMetadata activity + var mockEntityFeature = new Mock(MockBehavior.Strict); + mockEntityFeature.SetupLockForActivity(ActivityNames.ImportMetadata); + mockOrchestrationContext.SetupGet(context => context.Entities) + .Returns(mockEntityFeature.Object); + + var activitySequence = new MockSequence(); + + string[] expectedActivitySequence = + [ + ActivityNames.CopyCsvFiles, + ActivityNames.ImportMetadata, + ActivityNames.ImportData, + ActivityNames.WriteDataFiles, + ActivityNames.CompleteInitialDataSetVersionProcessing + ]; + + foreach (var activityName in expectedActivitySequence) + { + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => context.CallActivityAsync(activityName, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + } + + await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext, mockEntityFeature); + } + + [Fact] + public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + var activitySequence = new MockSequence(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.CopyCsvFiles, + mockOrchestrationContext.Object.InstanceId, + null)) + .Throws(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.HandleProcessingFailure, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + + await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + private static async Task ProcessInitialDataSetVersion(TaskOrchestrationContext orchestrationContext) + { + await ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion( + orchestrationContext, + new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); + } + + private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) + { + var mock = new Mock(); + + mock.Setup(context => + context.CreateReplaySafeLogger( + nameof(ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion))) + .Returns(NullLogger.Instance); + + mock.SetupGet(context => context.InstanceId) + .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); + + return mock; + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs index 77d1b1f78b4..688ac38c48b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs @@ -9,13 +9,8 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Utils; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; -using Microsoft.DurableTask; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; @@ -23,90 +18,6 @@ public abstract class ProcessNextDataSetVersionMappingsFunctionTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { - public class ProcessNextDataSetVersionMappingsTests( - ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessNextDataSetVersionMappingsFunctionTests(fixture) - { - [Fact] - public async Task Success() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - var activitySequence = new MockSequence(); - - string[] expectedActivitySequence = - [ - ActivityNames.CopyCsvFiles, - ActivityNames.CreateMappings, - ActivityNames.ApplyAutoMappings, - ActivityNames.CompleteNextDataSetVersionMappingProcessing, - ]; - - foreach (var activityName in expectedActivitySequence) - { - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => context.CallActivityAsync(activityName, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - } - - await ProcessNextDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - [Fact] - public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - var activitySequence = new MockSequence(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.CopyCsvFiles, - mockOrchestrationContext.Object.InstanceId, - null)) - .Throws(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.HandleProcessingFailure, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - - await ProcessNextDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - private async Task ProcessNextDataSetVersion(TaskOrchestrationContext orchestrationContext) - { - var function = GetRequiredService(); - await function.ProcessNextDataSetVersionMappings( - orchestrationContext, - new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); - } - - private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) - { - var mock = new Mock(MockBehavior.Strict); - - mock.Setup(context => context.CreateReplaySafeLogger( - nameof(ProcessNextDataSetVersionMappingsFunction.ProcessNextDataSetVersionMappings))) - .Returns(NullLogger.Instance); - - mock.SetupGet(context => context.InstanceId) - .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); - - return mock; - } - } - public abstract class CreateMappingsTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessNextDataSetVersionMappingsFunctionTests(fixture) @@ -115,7 +26,7 @@ public abstract class CreateMappingsTests( protected async Task CreateMappings(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CreateMappings(instanceId, CancellationToken.None); } } @@ -629,7 +540,7 @@ public abstract class ApplyAutoMappingsTests( protected async Task ApplyAutoMappings(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.ApplyAutoMappings(instanceId, CancellationToken.None); } } @@ -1790,7 +1701,7 @@ public async Task Success() private async Task CompleteProcessing(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CompleteNextDataSetVersionMappingProcessing(instanceId, CancellationToken.None); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs new file mode 100644 index 00000000000..252deb849d0 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs @@ -0,0 +1,92 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class ProcessNextDataSetVersionMappingsOrchestrationTests +{ + public class ProcessNextDataSetVersionMappingsTests + { + [Fact] + public async Task Success() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + var activitySequence = new MockSequence(); + + string[] expectedActivitySequence = + [ + ActivityNames.CopyCsvFiles, + ActivityNames.CreateMappings, + ActivityNames.ApplyAutoMappings, + ActivityNames.CompleteNextDataSetVersionMappingProcessing, + ]; + + foreach (var activityName in expectedActivitySequence) + { + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => context.CallActivityAsync(activityName, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + } + + await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + [Fact] + public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + var activitySequence = new MockSequence(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.CopyCsvFiles, + mockOrchestrationContext.Object.InstanceId, + null)) + .Throws(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.HandleProcessingFailure, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + + await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + private async Task ProcessNextDataSetVersion(TaskOrchestrationContext orchestrationContext) + { + await ProcessNextDataSetVersionMappingsFunctionOrchestration.ProcessNextDataSetVersionMappings( + orchestrationContext, + new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); + } + + private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) + { + var mock = new Mock(MockBehavior.Strict); + + mock.Setup(context => context.CreateReplaySafeLogger( + nameof(ProcessNextDataSetVersionMappingsFunctionOrchestration.ProcessNextDataSetVersionMappings))) + .Returns(NullLogger.Instance); + + mock.SetupGet(context => context.InstanceId) + .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); + + return mock; + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs index e9d51bad276..b15821ee4c5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs @@ -288,11 +288,11 @@ protected override IEnumerable GetFunctionTypes() return [ typeof(CreateDataSetFunction), - typeof(ProcessInitialDataSetVersionFunction), + typeof(CompleteInitialDataSetVersionProcessingFunction), typeof(CreateNextDataSetVersionMappingsFunction), - typeof(ProcessNextDataSetVersionMappingsFunction), + typeof(ProcessNextDataSetVersionMappingsFunctions), typeof(CompleteNextDataSetVersionImportFunction), - typeof(ProcessCompletionOfNextDataSetVersionFunction), + typeof(ProcessCompletionOfNextDataSetVersionFunctions), typeof(DeleteDataSetVersionFunction), typeof(CopyCsvFilesFunction), typeof(ImportMetadataFunction), diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs index b990d7e2b45..f1f002e15c4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs @@ -9,17 +9,17 @@ internal static class ActivityNames public const string HandleProcessingFailure = nameof(HandleProcessingFailureFunction.HandleProcessingFailure); public const string CompleteInitialDataSetVersionProcessing = - nameof(ProcessInitialDataSetVersionFunction.CompleteInitialDataSetVersionProcessing); + nameof(CompleteInitialDataSetVersionProcessingFunction.CompleteInitialDataSetVersionProcessing); - public const string CreateMappings = nameof(ProcessNextDataSetVersionMappingsFunction.CreateMappings); - public const string ApplyAutoMappings = nameof(ProcessNextDataSetVersionMappingsFunction.ApplyAutoMappings); + public const string CreateMappings = nameof(ProcessNextDataSetVersionMappingsFunctions.CreateMappings); + public const string ApplyAutoMappings = nameof(ProcessNextDataSetVersionMappingsFunctions.ApplyAutoMappings); public const string CompleteNextDataSetVersionMappingProcessing = - nameof(ProcessNextDataSetVersionMappingsFunction.CompleteNextDataSetVersionMappingProcessing); + nameof(ProcessNextDataSetVersionMappingsFunctions.CompleteNextDataSetVersionMappingProcessing); public const string CreateChanges = - nameof(ProcessCompletionOfNextDataSetVersionFunction.CreateChanges); + nameof(ProcessCompletionOfNextDataSetVersionFunctions.CreateChanges); public const string UpdateFileStoragePath = - nameof(ProcessCompletionOfNextDataSetVersionFunction.UpdateFileStoragePath); + nameof(ProcessCompletionOfNextDataSetVersionFunctions.UpdateFileStoragePath); public const string CompleteNextDataSetVersionImportProcessing = - nameof(ProcessCompletionOfNextDataSetVersionFunction.CompleteNextDataSetVersionImportProcessing); + nameof(ProcessCompletionOfNextDataSetVersionFunctions.CompleteNextDataSetVersionImportProcessing); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteInitialDataSetVersionProcessingFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteInitialDataSetVersionProcessingFunction.cs new file mode 100644 index 00000000000..b0d20ac0fbb --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteInitialDataSetVersionProcessingFunction.cs @@ -0,0 +1,30 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using Microsoft.Azure.Functions.Worker; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class CompleteInitialDataSetVersionProcessingFunction( + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver dataSetVersionPathResolver) : BaseProcessDataSetVersionFunction(publicDataDbContext) +{ + [Function(ActivityNames.CompleteInitialDataSetVersionProcessing)] + public async Task CompleteInitialDataSetVersionProcessing( + [ActivityTrigger] Guid instanceId, + CancellationToken cancellationToken) + { + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.Completing, cancellationToken); + + var dataSetVersion = dataSetVersionImport.DataSetVersion; + + // Delete the DuckDb database file as it is no longer needed + File.Delete(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); + + dataSetVersion.Status = DataSetVersionStatus.Draft; + + dataSetVersionImport.Completed = DateTimeOffset.UtcNow; + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs index 3351d89d3ec..8273ec0ef3e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs @@ -62,7 +62,7 @@ private async Task ProcessCompletionOfNextDataSetVersionImport( CancellationToken cancellationToken) { const string orchestratorName = - nameof(ProcessCompletionOfNextDataSetVersionFunction.ProcessCompletionOfNextDataSetVersion); + nameof(ProcessCompletionOfNextDataSetVersionOrchestration.ProcessCompletionOfNextDataSetVersionImport); var input = new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersionId }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs index aed6af84bed..5660fdfab27 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs @@ -59,7 +59,8 @@ private async Task ProcessInitialDataSetVersion( Guid instanceId, CancellationToken cancellationToken) { - const string orchestratorName = nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion); + const string orchestratorName = + nameof(ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion); var input = new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersionId }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs index 81a6b6058c7..f253abffc88 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs @@ -64,7 +64,7 @@ private async Task ProcessNextDataSetVersion( CancellationToken cancellationToken) { const string orchestratorName = - nameof(ProcessNextDataSetVersionMappingsFunction.ProcessNextDataSetVersionMappings); + nameof(ProcessNextDataSetVersionMappingsFunctionOrchestration.ProcessNextDataSetVersionMappings); var input = new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersionId }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs similarity index 64% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs index a71de9a718c..e9bae2c8a76 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs @@ -5,51 +5,15 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.Azure.Functions.Worker; -using Microsoft.DurableTask; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class ProcessCompletionOfNextDataSetVersionFunction( +public class ProcessCompletionOfNextDataSetVersionFunctions( PublicDataDbContext publicDataDbContext, IDataSetVersionPathResolver dataSetVersionPathResolver, IDataSetVersionChangeService dataSetVersionChangeService) : BaseProcessDataSetVersionFunction(publicDataDbContext) { - [Function(nameof(ProcessCompletionOfNextDataSetVersion))] - public async Task ProcessCompletionOfNextDataSetVersion( - [OrchestrationTrigger] TaskOrchestrationContext context, - ProcessDataSetVersionContext input) - { - var logger = context.CreateReplaySafeLogger(nameof(ProcessCompletionOfNextDataSetVersion)); - - logger.LogInformation( - "Processing completion of import for next data set version (InstanceId={InstanceId}, " + - "DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - try - { - await context.CallActivity(ActivityNames.UpdateFileStoragePath, logger, context.InstanceId); - await context.CallActivity(ActivityNames.ImportMetadata, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CreateChanges, logger, context.InstanceId); - await context.CallActivity(ActivityNames.ImportData, logger, context.InstanceId); - await context.CallActivity(ActivityNames.WriteDataFiles, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CompleteNextDataSetVersionImportProcessing, logger, - context.InstanceId); - } - catch (Exception e) - { - logger.LogError(e, - "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); - } - } - [Function(ActivityNames.CreateChanges)] public async Task CreateChanges( [ActivityTrigger] Guid instanceId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs new file mode 100644 index 00000000000..57db11c7180 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs @@ -0,0 +1,44 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public static class ProcessCompletionOfNextDataSetVersionOrchestration +{ + [Function(nameof(ProcessCompletionOfNextDataSetVersionImport))] + public static async Task ProcessCompletionOfNextDataSetVersionImport( + [OrchestrationTrigger] TaskOrchestrationContext context, + ProcessDataSetVersionContext input) + { + var logger = context.CreateReplaySafeLogger(nameof(ProcessCompletionOfNextDataSetVersionImport)); + + logger.LogInformation( + "Processing completion of import for next data set version (InstanceId={InstanceId}, " + + "DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + try + { + await context.CallActivity(ActivityNames.UpdateFileStoragePath, logger, context.InstanceId); + await context.CallActivityExclusively(ActivityNames.ImportMetadata, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CreateChanges, logger, context.InstanceId); + await context.CallActivity(ActivityNames.ImportData, logger, context.InstanceId); + await context.CallActivity(ActivityNames.WriteDataFiles, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CompleteNextDataSetVersionImportProcessing, logger, + context.InstanceId); + } + catch (Exception e) + { + logger.LogError(e, + "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionOrchestration.cs similarity index 57% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionOrchestration.cs index 9ba387dc7e9..f677a68986c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionOrchestration.cs @@ -1,20 +1,15 @@ -using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.Azure.Functions.Worker; using Microsoft.DurableTask; using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class ProcessInitialDataSetVersionFunction( - PublicDataDbContext publicDataDbContext, - IDataSetVersionPathResolver dataSetVersionPathResolver) : BaseProcessDataSetVersionFunction(publicDataDbContext) +public static class ProcessInitialDataSetVersionOrchestration { [Function(nameof(ProcessInitialDataSetVersion))] - public async Task ProcessInitialDataSetVersion( + public static async Task ProcessInitialDataSetVersion( [OrchestrationTrigger] TaskOrchestrationContext context, ProcessDataSetVersionContext input) { @@ -44,23 +39,4 @@ await context.CallActivity(ActivityNames.CompleteInitialDataSetVersionProcessing await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); } } - - [Function(ActivityNames.CompleteInitialDataSetVersionProcessing)] - public async Task CompleteInitialDataSetVersionProcessing( - [ActivityTrigger] Guid instanceId, - CancellationToken cancellationToken) - { - var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); - await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.Completing, cancellationToken); - - var dataSetVersion = dataSetVersionImport.DataSetVersion; - - // Delete the DuckDb database file as it is no longer needed - File.Delete(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); - - dataSetVersion.Status = DataSetVersionStatus.Draft; - - dataSetVersionImport.Completed = DateTimeOffset.UtcNow; - await publicDataDbContext.SaveChangesAsync(cancellationToken); - } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunctions.cs similarity index 56% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunctions.cs index c1507e6adc3..a24c1189444 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunctions.cs @@ -1,48 +1,14 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; using Microsoft.Azure.Functions.Worker; -using Microsoft.DurableTask; -using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class ProcessNextDataSetVersionMappingsFunction( +public class ProcessNextDataSetVersionMappingsFunctions( PublicDataDbContext publicDataDbContext, IDataSetVersionMappingService mappingService) : BaseProcessDataSetVersionFunction(publicDataDbContext) { - [Function(nameof(ProcessNextDataSetVersionMappings))] - public async Task ProcessNextDataSetVersionMappings([OrchestrationTrigger] TaskOrchestrationContext context, - ProcessDataSetVersionContext input) - { - var logger = context.CreateReplaySafeLogger(nameof(ProcessNextDataSetVersionMappings)); - - logger.LogInformation( - "Processing next data set version (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - try - { - await context.CallActivity(ActivityNames.CopyCsvFiles, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CreateMappings, logger, context.InstanceId); - await context.CallActivity(ActivityNames.ApplyAutoMappings, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CompleteNextDataSetVersionMappingProcessing, logger, - context.InstanceId); - } - catch (Exception e) - { - logger.LogError(e, - "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); - } - } - [Function(ActivityNames.CreateMappings)] public async Task CreateMappings( [ActivityTrigger] Guid instanceId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsOrchestration.cs new file mode 100644 index 00000000000..c2dc922c9d9 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsOrchestration.cs @@ -0,0 +1,40 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public static class ProcessNextDataSetVersionMappingsFunctionOrchestration +{ + [Function(nameof(ProcessNextDataSetVersionMappings))] + public static async Task ProcessNextDataSetVersionMappings([OrchestrationTrigger] TaskOrchestrationContext context, + ProcessDataSetVersionContext input) + { + var logger = context.CreateReplaySafeLogger(nameof(ProcessNextDataSetVersionMappings)); + + logger.LogInformation( + "Processing next data set version (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + try + { + await context.CallActivity(ActivityNames.CopyCsvFiles, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CreateMappings, logger, context.InstanceId); + await context.CallActivity(ActivityNames.ApplyAutoMappings, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CompleteNextDataSetVersionMappingProcessing, logger, + context.InstanceId); + } + catch (Exception e) + { + logger.LogError(e, + "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); + } + } +}