Skip to content

Commit

Permalink
Merge pull request #5275 from dfe-analytical-services/EES-5492-preven…
Browse files Browse the repository at this point in the history 
…t-exceptions-locking-importmetadata-stage

EES-5492 - broke orchestrations into their own simplified files and m…
  • Loading branch information
@duncan-at-hiveit
duncan-at-hiveit authored Oct 15, 2024
2 parents 0a93f98 + 93b5d5e commit 9eda67c
Show file tree
Hide file tree
Showing 22 changed files with 546 additions and 505 deletions.
86 changes: 86 additions & 0 deletions ...ic.Data.Processor.Tests/Functions/CompleteInitialDataSetVersionProcessingFunctionTests.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions;
using GovUk.Education.ExploreEducationStatistics.Public.Data.Model;
using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database;
using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables;
using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions;
using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces;
using Microsoft.EntityFrameworkCore;

namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions;

public abstract class CompleteInitialDataSetVersionProcessingFunctionTests(
ProcessorFunctionsIntegrationTestFixture fixture)
: ProcessorFunctionsIntegrationTest(fixture)
{
private static readonly string[] AllDataSetVersionFiles =
[
DataSetFilenames.CsvDataFile,
DataSetFilenames.CsvMetadataFile,
DataSetFilenames.DuckDbDatabaseFile,
DataSetFilenames.DuckDbLoadSqlFile,
DataSetFilenames.DuckDbSchemaSqlFile,
DataTable.ParquetFile,
FilterOptionsTable.ParquetFile,
IndicatorsTable.ParquetFile,
LocationOptionsTable.ParquetFile,
TimePeriodsTable.ParquetFile
];

public class CompleteInitialDataSetVersionProcessingProcessingTests(
ProcessorFunctionsIntegrationTestFixture fixture)
: CompleteInitialDataSetVersionProcessingFunctionTests(fixture)
{
private const DataSetVersionImportStage Stage = DataSetVersionImportStage.Completing;

[Fact]
public async Task Success()
{
var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage());

var dataSetVersionPathResolver = GetRequiredService<IDataSetVersionPathResolver>();
Directory.CreateDirectory(dataSetVersionPathResolver.DirectoryPath(dataSetVersion));

await CompleteInitialDataSetVersionProcessing(instanceId);

await using var publicDataDbContext = GetDbContext<PublicDataDbContext>();

var savedImport = await publicDataDbContext.DataSetVersionImports
.Include(i => i.DataSetVersion)
.SingleAsync(i => i.InstanceId == instanceId);

Assert.Equal(Stage, savedImport.Stage);
savedImport.Completed.AssertUtcNow();

Assert.Equal(DataSetVersionStatus.Draft, savedImport.DataSetVersion.Status);
}

[Fact]
public async Task DuckDbFileIsDeleted()
{
var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage());

// Create empty data set version files for all file paths
var dataSetVersionPathResolver = GetRequiredService<IDataSetVersionPathResolver>();
var directoryPath = dataSetVersionPathResolver.DirectoryPath(dataSetVersion);
Directory.CreateDirectory(directoryPath);
foreach (var filename in AllDataSetVersionFiles)
{
await File.Create(Path.Combine(directoryPath, filename)).DisposeAsync();
}

await CompleteInitialDataSetVersionProcessing(instanceId);

// Ensure the duck db database file is the only file that was deleted
AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion,
AllDataSetVersionFiles
.Where(file => file != DataSetFilenames.DuckDbDatabaseFile)
.ToArray());
}

private async Task CompleteInitialDataSetVersionProcessing(Guid instanceId)
{
var function = GetRequiredService<CompleteInitialDataSetVersionProcessingFunction>();
await function.CompleteInitialDataSetVersionProcessing(instanceId, CancellationToken.None);
}
}
}
4 changes: 2 additions & 2 deletions ...cs.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ public async Task Success()
StartOrchestrationOptions? startOrchestrationOptions = null;
durableTaskClientMock.Setup(client =>
client.ScheduleNewOrchestrationInstanceAsync(
nameof(ProcessCompletionOfNextDataSetVersionFunction
.ProcessCompletionOfNextDataSetVersion),
nameof(ProcessCompletionOfNextDataSetVersionOrchestration
.ProcessCompletionOfNextDataSetVersionImport),
It.IsAny<ProcessDataSetVersionContext>(),
It.IsAny<StartOrchestrationOptions>(),
It.IsAny<CancellationToken>()))
Expand Down
6 changes: 3 additions & 3 deletions ...reEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ await AddTestData<ContentDbContext>(context =>
StartOrchestrationOptions? startOrchestrationOptions = null;
durableTaskClientMock.Setup(client =>
client.ScheduleNewOrchestrationInstanceAsync(
nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion),
nameof(ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion),
It.IsAny<ProcessDataSetVersionContext>(),
It.IsAny<StartOrchestrationOptions>(),
It.IsAny<CancellationToken>()))
Expand Down Expand Up @@ -163,8 +163,8 @@ public async Task ReleaseFileIdHasDataSetVersion_ReturnsValidationProblem()
DataSet dataSet = DataFixture.DefaultDataSet();

DataSetVersion dataSetVersion = DataFixture.DefaultDataSetVersion()
.WithRelease(DataFixture.DefaultDataSetVersionRelease()
.WithReleaseFileId(releaseFile.Id))
.WithRelease(DataFixture.DefaultDataSetVersionRelease()
.WithReleaseFileId(releaseFile.Id))
.WithDataSet(dataSet);

await AddTestData<ContentDbContext>(context =>
Expand Down
3 changes: 2 additions & 1 deletion ...cs.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ public async Task Success()
StartOrchestrationOptions? startOrchestrationOptions = null;
durableTaskClientMock.Setup(client =>
client.ScheduleNewOrchestrationInstanceAsync(
nameof(ProcessNextDataSetVersionMappingsFunction.ProcessNextDataSetVersionMappings),
nameof(ProcessNextDataSetVersionMappingsFunctionOrchestration
.ProcessNextDataSetVersionMappings),
It.IsAny<ProcessDataSetVersionContext>(),
It.IsAny<StartOrchestrationOptions>(),
It.IsAny<CancellationToken>()))
Expand Down
112 changes: 10 additions & 102 deletions ...fNextDataSetVersionImportFunctionTests.cs → ...tionOfNextDataSetVersionFunctionsTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,14 @@
using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables;
using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures;
using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions;
using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model;
using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces;
using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils;
using Microsoft.DurableTask;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils;
using FilterMeta = GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta;

namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions;

public abstract class ProcessCompletionOfNextDataSetVersionImportFunctionTests(
public abstract class ProcessCompletionOfNextDataSetVersionFunctionsTests(
ProcessorFunctionsIntegrationTestFixture fixture)
: ProcessorFunctionsIntegrationTest(fixture)
{
Expand All @@ -37,102 +32,15 @@ public abstract class ProcessCompletionOfNextDataSetVersionImportFunctionTests(
TimePeriodsTable.ParquetFile
];

public class ProcessCompletionOfNextDataSetVersionImportTests(
ProcessorFunctionsIntegrationTestFixture fixture)
: ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture)
{
[Fact]
public async Task Success()
{
var mockOrchestrationContext = DefaultMockOrchestrationContext();
var activitySequence = new MockSequence();

string[] expectedActivitySequence =
[
ActivityNames.UpdateFileStoragePath,
ActivityNames.ImportMetadata,
ActivityNames.CreateChanges,
ActivityNames.ImportData,
ActivityNames.WriteDataFiles,
ActivityNames.CompleteNextDataSetVersionImportProcessing
];

foreach (var activityName in expectedActivitySequence)
{
mockOrchestrationContext
.InSequence(activitySequence)
.Setup(context => context.CallActivityAsync(activityName,
mockOrchestrationContext.Object.InstanceId,
null))
.Returns(Task.CompletedTask);
}

await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object);

VerifyAllMocks(mockOrchestrationContext);
}

[Fact]
public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity()
{
var mockOrchestrationContext = DefaultMockOrchestrationContext();

var activitySequence = new MockSequence();

mockOrchestrationContext
.InSequence(activitySequence)
.Setup(context =>
context.CallActivityAsync(ActivityNames.UpdateFileStoragePath,
mockOrchestrationContext.Object.InstanceId,
null))
.Throws<Exception>();

mockOrchestrationContext
.InSequence(activitySequence)
.Setup(context =>
context.CallActivityAsync(ActivityNames.HandleProcessingFailure,
mockOrchestrationContext.Object.InstanceId,
null))
.Returns(Task.CompletedTask);

await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object);

VerifyAllMocks(mockOrchestrationContext);
}

private async Task ProcessCompletionOfNextDataSetVersionImport(TaskOrchestrationContext orchestrationContext)
{
var function = GetRequiredService<ProcessCompletionOfNextDataSetVersionFunction>();
await function.ProcessCompletionOfNextDataSetVersion(
orchestrationContext,
new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() });
}

private static Mock<TaskOrchestrationContext> DefaultMockOrchestrationContext(Guid? instanceId = null)
{
var mock = new Mock<TaskOrchestrationContext>(MockBehavior.Strict);

mock.Setup(context =>
context.CreateReplaySafeLogger(
nameof(ProcessCompletionOfNextDataSetVersionFunction.ProcessCompletionOfNextDataSetVersion)))
.Returns(NullLogger.Instance);

mock.SetupGet(context => context.InstanceId)
.Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString());

return mock;
}
}

public abstract class CreateChangesTests(
ProcessorFunctionsIntegrationTestFixture fixture)
: ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture)
: ProcessCompletionOfNextDataSetVersionFunctionsTests(fixture)
{
protected const DataSetVersionImportStage Stage = DataSetVersionImportStage.CreatingChanges;

protected async Task CreateChanges(Guid instanceId)
{
var function = GetRequiredService<ProcessCompletionOfNextDataSetVersionFunction>();
var function = GetRequiredService<ProcessCompletionOfNextDataSetVersionFunctions>();
await function.CreateChanges(instanceId, CancellationToken.None);
}
}
Expand Down Expand Up @@ -3084,7 +2992,7 @@ .. DataFixture.DefaultTimePeriodMeta()

public class UpdateFileStoragePathTests(
ProcessorFunctionsIntegrationTestFixture fixture)
: ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture)
: ProcessCompletionOfNextDataSetVersionFunctionsTests(fixture)
{
private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ManualMapping;

Expand Down Expand Up @@ -3132,14 +3040,14 @@ public async Task Success_PathNotUpdated()

private async Task UpdateFileStoragePath(Guid instanceId)
{
var function = GetRequiredService<ProcessCompletionOfNextDataSetVersionFunction>();
var function = GetRequiredService<ProcessCompletionOfNextDataSetVersionFunctions>();
await function.UpdateFileStoragePath(instanceId, CancellationToken.None);
}
}

public class CompleteNextDataSetVersionImportProcessingTests(
ProcessorFunctionsIntegrationTestFixture fixture)
: ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture)
: ProcessCompletionOfNextDataSetVersionFunctionsTests(fixture)
{
private const DataSetVersionImportStage Stage = DataSetVersionImportStage.Completing;

Expand All @@ -3151,7 +3059,7 @@ public async Task Success()
var dataSetVersionPathResolver = GetRequiredService<IDataSetVersionPathResolver>();
Directory.CreateDirectory(dataSetVersionPathResolver.DirectoryPath(dataSetVersion));

await CompleteProcessing(instanceId);
await CompleteNextDataSetVersionImportProcessing(instanceId);

await using var publicDataDbContext = GetDbContext<PublicDataDbContext>();

Expand Down Expand Up @@ -3179,7 +3087,7 @@ public async Task DuckDbFileIsDeleted()
await File.Create(Path.Combine(directoryPath, filename)).DisposeAsync();
}

await CompleteProcessing(instanceId);
await CompleteNextDataSetVersionImportProcessing(instanceId);

// Ensure the duck db database file is the only file that was deleted
AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion,
Expand All @@ -3188,9 +3096,9 @@ public async Task DuckDbFileIsDeleted()
.ToArray());
}

private async Task CompleteProcessing(Guid instanceId)
private async Task CompleteNextDataSetVersionImportProcessing(Guid instanceId)
{
var function = GetRequiredService<ProcessCompletionOfNextDataSetVersionFunction>();
var function = GetRequiredService<ProcessCompletionOfNextDataSetVersionFunctions>();
await function.CompleteNextDataSetVersionImportProcessing(instanceId, CancellationToken.None);
}
}
Expand Down
Loading

0 comments on commit 9eda67c

Please sign in to comment.